Try it this way:
clc; % Clear the command window.
close all; % Close all figures (except those of imtool.)
imtool close all; % Close all imtool figures.
clear; % Erase all existing variables.
workspace; % Make sure the workspace panel is showing.
format longg;
format compact;
fontSize = 20;
rgbImage = imread('peppers.png');
reductionFactor = 0.5;
[rows columns numberOfColorChannels] = size(rgbImage);
if rows > 400 || columns > 400
rgbImage = imresize(rgbImage, reductionFactor);
end
subplot(2,2,1);
imshow(rgbImage);
title('Original image', 'FontSize', fontSize);
% Enlarge figure to full screen.
set(gcf, 'units','normalized','outerposition',[0 0 1 1]);
m = menu('Do you want the image be in a shape of a square?','yes','no');
[rows columns numberOfColorChannels] = size(rgbImage);
squareWidth = min([rows columns]);
if m == 1
rgbImage = imresize(rgbImage, [squareWidth, squareWidth]);
subplot(2,2,2);
imshow(rgbImage);
title('Square image', 'FontSize', fontSize);
end
[rows columns numberOfColorChannels] = size(rgbImage);
[columnsInImage rowsInImage] = meshgrid(1:columns, 1:rows);
centerX = columns/2;
centerY = rows/2;
% Ask user for a number.
defaultValue = 45;
titleBar = 'Enter a value';
userPrompt = sprintf('What should the radius of the circular frame be (< %d)? ', squareWidth);
caUserInput = inputdlg(userPrompt, titleBar, 1, {num2str(defaultValue)});
if isempty(caUserInput),return,end; % Bail out if they clicked Cancel.
radius = str2double(cell2mat(caUserInput));
% Check for a valid integer.
if isnan(radius)
% They didn't enter a number.
% They clicked Cancel, or entered a character, symbols, or something else not allowed.
radius = defaultValue;
message = sprintf('I said it had to be an integer.\nI will use %d and continue.', radius);
uiwait(warndlg(message));
end
if radius > min([rows columns]) / 2
uiwait(msgbox('NB! Radius have to be smaller or equal to the pictures dimensions'));
k = menu('NB! Radius have to be smaller or equal to the pictures dimensions. Try again?','Yes','No');
radius = min([rows columns]) / 2;
end
circlePixels = (rowsInImage - centerY).^2 ...
+ (columnsInImage - centerX).^2 <= radius.^2;
maskedRgbImage = bsxfun(@times, rgbImage, cast(circlePixels, class(rgbImage)));
subplot(2, 2, 3);
imshow(maskedRgbImage);
title('Circularly masked image', 'FontSize', fontSize);