You neglected to take the absolute value of ER/E before squaring. (Notice the warning you got that imaginary parts of complex values are ignored when plotting - this tells you the values being plotted were still complex.)
close all
clc
clear all
x=0:0.01:pi/2;
nR=3.7;
nI=5.4;
n=nR+nI*i;
mR=0.04;
mI=2.4;
m=mR+mI*i;
z=zeros(length(x),4);
for j=1:length(x);
A=cos(x(j));
B=sqrt((n^2)-((sin(x(j))^2)));
C=(n^2)*(cos(x(j)));
D=cos(x(j));
E=sqrt((m^2)-((sin(x(j))^2)));
F=(m^2)*(cos(x(j)));
rTE1(j)=(A-B)/(A+B);
rTM1(j)=(C-B)/(C+B);
rTE2(j)=(D-E)/(D+E);
rTM2(j)=(F-E)/(F+E);
% RTE1(j)= ((rTE1(j))^2);
% RTM1(j)= ((rTM1(j))^2);
% RTE2(j)= ((rTE2(j))^2);
% RTM2(j)= ((rTM2(j))^2);
RTE1(j) = abs(rTE1(j))^2;
RTM1(j) = abs(rTM1(j))^2;
RTE2(j) = abs(rTE2(j))^2;
RTM2(j) = abs(rTM2(j))^2;
z(j,1)=RTE1(j);
z(j,2)=RTM1(j);
z(j,3)=RTE2(j);
z(j,4)=RTM2(j);
end
t=x*180/pi;
plot(t,z(:,2),t,z(:,1),t,z(:,4),t,z(:,3))
