Something like this perhaps:
hs = scatter(randn(1,1000),randn(1,1000),'.');
hr = rectangle('Position',[.5 .5 .65 .5]);
![](https://www.mathworks.com/matlabcentral/answers/uploaded_files/174227/image.png)
minx = hr.Position(1);
maxx = hr.Position(1) + hr.Position(3);
miny = hr.Position(2);
maxy = hr.Position(2) + hr.Position(4);
mask = hs.XData >= minx & hs.XData <= maxx ...
& hs.YData >= miny & hs.YData <= maxy;
x = hs.XData(mask);
y = hs.YData(mask);
hold on
scatter(x,y,'filled')
![](https://www.mathworks.com/matlabcentral/answers/uploaded_files/174228/image.png)