Here is an idea:
x0 = mean(x); % Choose some center point (maybe approximately)
y0 = mean(y);
[t,r] = cart2pol(x,y); % convert your data to polar system
[~,ix] = sort(t); % sort your data by angle
tol = 0.4; % some tolerance
ix1 = find(diff(r(ix) > tol); % find values where difference of radius is too big
ind = ix(ix1); % indices of outlier points
plot3(x(ind),y(ind),z(ind),'or')
