Mainly, your step size is too big and you are plotting over too large a range.
X = -3:0.01:3;
[X,Y] = meshgrid(X);
Z =(3-X).^2 + 30*((Y-(X.^2)).^2);
surf(X,Y,Z,'FaceColor','c','FaceAlpha',0.3,'EdgeColor','none');
hold on
x(1) = 0; % initial value of x
y(1) = 0; % initial value of y
z(1) = ((3-x(1)).^2) + (30.*(y(1)-(x(1).^2)).^2);
stepsize = 0.01;
for i = 1:60
zx = 120*x(i)^3 + 2*x(i) - 120*x(i)*y(i) - 6;
zy = 60*y(i) - 60*x(i)^2;
x(i+1) = x(i) - stepsize*zx; %gradient descent
y(i+1) = y(i) - stepsize*zy;
z(i+1) = ((3-x(i+1)).^2) + (30.*(y(i+1)-(x(i+1).^2)).^2);
end
plot3(x,y,z,'o','Markersize',3,'Color','red')
hold off
axis([min(x),max(x), min(y),max(y), min(z), max(z)]);
view(154,34)
xlabel x; ylabel y; zlabel z;
