Something like this perhaps:
hs = scatter(randn(1,1000),randn(1,1000),'.');
hr = rectangle('Position',[.5 .5 .65 .5]);
minx = hr.Position(1);
maxx = hr.Position(1) + hr.Position(3);
miny = hr.Position(2);
maxy = hr.Position(2) + hr.Position(4);
mask = hs.XData >= minx & hs.XData <= maxx ...
& hs.YData >= miny & hs.YData <= maxy;
x = hs.XData(mask);
y = hs.YData(mask);
hold on
scatter(x,y,'filled')