I'm pretty sure you can use hgtransform() on rectangle() objects.
hg = hgtransform;
rectangle('parent',hg,'position',[0 0 10 20],'curvature',1);
hg.Matrix = makehgtform('zrotate',pi/6);
axis equal
Alternatively, the following can be used to generate a vertex list that corresponds to the input syntax supported by rectangle(). You can use the vertex data to generate a patch or other type object if you don't want to deal with a rectangle() object.
pos = [0 0 10 20]; % [x y w h]
cur = [1 1]*0.25; % curvature
npts = 20;
npts = round(npts/4)*4 + 1;
th = linspace(0,2*pi,npts);
thm = reshape(th(2:end),[],4);
thm = [th(1) thm(end,1:3); thm];
rr = pos(3:4)/2;
er = cur.*rr;
x = er(1)*cos(thm) + pos(1) + rr(1) + (rr(1)-er(1))*[1 -1 -1 1];
y = er(2)*sin(thm) + pos(2) + rr(2) + (rr(2)-er(2))*[1 1 -1 -1];
x = [x(:); x(1)];
y = [y(:); y(1)];
% plot two identical rounded rectangles atop each other
rectangle('position',pos,'curvature',cur); hold on
plot(x,y,'--y')
axis equal
