What about interpolation?
clc,clear
XYColor = [
1 3 1 1 1
1 2 1 0 1
1 1 0 1 0
2 1 0 0 1
2 2 1 1 1
2 3 1 0 0
3 1 1 0 0
3 2 1 0 0
3 3 1 1 0];
x = XYColor(:,1);
y = XYColor(:,2);
r = XYColor(:,3);
g = XYColor(:,4);
b = XYColor(:,5);
FR = scatteredInterpolant(x,y,r);
FG = scatteredInterpolant(x,y,g);
FB = scatteredInterpolant(x,y,b);
[x1,y1] = meshgrid(1:0.2:3);
r1 = FR(x1,y1);
surf(x1,y1,r1)