Try something like this (untested)
% Get alert gray level from GUI
alertGrayLevel = app.edtAlert.Value;
% Make up colormap to show red pixels if image is above the alert gray level.
cmap = gray(256);
cmap(alertGrayLevel:end, :) = repmat([1,0,0], alertGrayLevel:end, 1);
% Get the current image.
thisFrame = getdata(videoObject)
if ndims == 3
% It's color so convert to gray scale.
thisFrame = rgb2gray(thisFrame);
end
if app.chkInvertGrayScale
% If they have checked the checkbox to invert the gray scale, do that.
thisFrame = 255 - thisFrame;
end
imshow(thisFrame, 'ColorMap', cmap);
% Display the colormap next to the image.
colormap(cmap);
colorbar;