使用深度学习的神经样式迁移

加载数据

imshow(imtile({styleImage,contentImage},BackgroundColor="w"));

加载特征提取网络

net = vgg19;

lastFeatureLayerIdx = 38;
layers = net.Layers;
layers = layers(1:lastFeatureLayerIdx);

VGG-19 网络的最大池化层会造成衰落效应。要降低衰落效应并增加梯度流，请将所有最大池化层替换为平均池化层 [1]

for l = 1:lastFeatureLayerIdx
layer = layers(l);
if isa(layer,"nnet.cnn.layer.MaxPooling2DLayer")
layers(l) = averagePooling2dLayer( ...
layer.PoolSize,Stride=layer.Stride,Name=layer.Name);
end
end

lgraph = layerGraph(layers);

plot(lgraph)
title("Feature Extraction Network")

dlnet = dlnetwork(lgraph);

预处理数据

imageSize = [384,512];
styleImg = imresize(styleImage,imageSize);
contentImg = imresize(contentImage,imageSize);

imgInputLayer = lgraph.Layers(1);
meanVggNet = imgInputLayer.Mean(1,1,:);

styleImg = rescale(single(styleImg),0,255) - meanVggNet;
contentImg = rescale(single(contentImg),0,255) - meanVggNet;

初始化迁移图像

noiseRatio = 0.7;
randImage = randi([-20,20],[imageSize 3]);
transferImage = noiseRatio.*randImage + (1-noiseRatio).*contentImg;

定义损失函数和样式迁移参数

内容损失

${L}_{content}=\sum _{l}{W}_{c}^{l}×\frac{1}{HWC}\sum _{i,j}\left({\underset{}{\overset{ˆ}{Y}}}_{i,j}^{l}-{Y}_{i,j}^{l}{\right)}^{2}$

styleTransferOptions.contentFeatureLayerNames = "conv4_2";

styleTransferOptions.contentFeatureLayerWeights = 1;

样式损失

${G}_{\underset{}{\overset{ˆ}{Z}}}=\sum _{i,j}{\underset{}{\overset{ˆ}{Z}}}_{i,j}×{\underset{}{\overset{ˆ}{Z}}}_{j,i}$

${G}_{Z}=\sum _{i,j}{Z}_{i,j}×{Z}_{j,i}$

${L}_{style}=\sum _{l}{W}_{s}^{l}×\frac{1}{\left(2HWC{\right)}^{2}}\sum \left({G}_{\underset{}{\overset{ˆ}{Z}}}^{l}-{G}_{Z}^{l}{\right)}^{2}$

styleTransferOptions.styleFeatureLayerNames = [ ...
"conv1_1","conv2_1","conv3_1","conv4_1","conv5_1"];

styleTransferOptions.styleFeatureLayerWeights = [0.5,1.0,1.5,3.0,4.0];

总损失

${L}_{total}=\alpha ×{L}_{content}+\beta ×{L}_{style}$

styleTransferOptions.alpha = 1;
styleTransferOptions.beta = 1e3;

指定训练选项

numIterations = 2500;

learningRate = 2;
trailingAvg = [];
trailingAvgSq = [];

训练网络

dlStyle = dlarray(styleImg,"SSC");
dlContent = dlarray(contentImg,"SSC");
dlTransfer = dlarray(transferImage,"SSC");

if canUseGPU
dlContent = gpuArray(dlContent);
dlStyle = gpuArray(dlStyle);
dlTransfer = gpuArray(dlTransfer);
end

numContentFeatureLayers = numel(styleTransferOptions.contentFeatureLayerNames);
contentFeatures = cell(1,numContentFeatureLayers);
[contentFeatures{:}] = forward(dlnet,dlContent,Outputs=styleTransferOptions.contentFeatureLayerNames);

numStyleFeatureLayers = numel(styleTransferOptions.styleFeatureLayerNames);
styleFeatures = cell(1,numStyleFeatureLayers);
[styleFeatures{:}] = forward(dlnet,dlStyle,Outputs=styleTransferOptions.styleFeatureLayerNames);

• 选择最佳样式迁移图像作为最终输出图像。

figure

minimumLoss = inf;

for iteration = 1:numIterations
% Evaluate the transfer image gradients and state using dlfeval and the
% imageGradients function listed at the end of the example
contentFeatures,styleFeatures,styleTransferOptions);

if losses.totalLoss < minimumLoss
minimumLoss = losses.totalLoss;
dlOutput = dlTransfer;
end

% Display the transfer image on the first iteration and after every 50
% iterations. The postprocessing steps are described in the "Postprocess
% Transfer Image for Display" section of this example
if mod(iteration,50) == 0 || (iteration == 1)

transferImage = gather(extractdata(dlTransfer));
transferImage = transferImage + meanVggNet;
transferImage = uint8(transferImage);
transferImage = imresize(transferImage,size(contentImage,[1 2]));

image(transferImage)
title(["Transfer Image After Iteration ",num2str(iteration)])
axis off image
drawnow
end

end

后处理迁移图像以用于显示

transferImage = gather(extractdata(dlOutput));

transferImage = transferImage + meanVggNet;

transferImage = uint8(transferImage);

transferImage = imresize(transferImage,size(contentImage,[1 2]));

imshow(imtile({contentImage,transferImage,styleImage}, ...
GridSize=[1 3],BackgroundColor="w"));

支持函数

计算图像损失和梯度

contentFeatures,styleFeatures,params)

% Initialize transfer image feature containers
numContentFeatureLayers = numel(params.contentFeatureLayerNames);
numStyleFeatureLayers = numel(params.styleFeatureLayerNames);

transferContentFeatures = cell(1,numContentFeatureLayers);
transferStyleFeatures = cell(1,numStyleFeatureLayers);

% Extract content features of transfer image
[transferContentFeatures{:}] = forward(dlnet,dlTransfer, ...
Outputs=params.contentFeatureLayerNames);

% Extract style features of transfer image
[transferStyleFeatures{:}] = forward(dlnet,dlTransfer, ...
Outputs=params.styleFeatureLayerNames);

% Calculate content loss
cLoss = contentLoss(transferContentFeatures,contentFeatures, ...
params.contentFeatureLayerWeights);

% Calculate style loss
sLoss = styleLoss(transferStyleFeatures,styleFeatures, ...
params.styleFeatureLayerWeights);

% Calculate final loss as weighted combination of content and style loss
loss = (params.alpha * cLoss) + (params.beta * sLoss);

% Calculate gradient with respect to transfer image

% Extract various losses
losses.totalLoss = gather(extractdata(loss));
losses.contentLoss = gather(extractdata(cLoss));
losses.styleLoss = gather(extractdata(sLoss));

end

计算内容损失

contentLoss 辅助函数计算内容图像特征和迁移图像特征之间的加权均方差。

function loss = contentLoss(transferContentFeatures,contentFeatures,contentWeights)

loss = 0;
for i=1:numel(contentFeatures)
temp = 0.5 .* mean((transferContentFeatures{1,i}-contentFeatures{1,i}).^2,"all");
loss = loss + (contentWeights(i)*temp);
end
end

计算样式损失

styleLoss 辅助函数计算样式图像特征的 Gram 矩阵和迁移图像特征的 Gram 矩阵之间的加权均方差。

function loss = styleLoss(transferStyleFeatures,styleFeatures,styleWeights)

loss = 0;
for i=1:numel(styleFeatures)

tsf = transferStyleFeatures{1,i};
sf = styleFeatures{1,i};
[h,w,c] = size(sf);

gramStyle = calculateGramMatrix(sf);
gramTransfer = calculateGramMatrix(tsf);
sLoss = mean((gramTransfer - gramStyle).^2,"all") / ((h*w*c)^2);

loss = loss + (styleWeights(i)*sLoss);
end
end

计算 Gram 矩阵

styleLoss 辅助函数使用 calculateGramMatrix 辅助函数来计算特征图的 Gram 矩阵。

function gramMatrix = calculateGramMatrix(featureMap)
[H,W,C] = size(featureMap);
reshapedFeatures = reshape(featureMap,H*W,C);
gramMatrix = reshapedFeatures' * reshapedFeatures;
end

参考资料

[1] Leon A. Gatys, Alexander S. Ecker, and Matthias Bethge."A Neural Algorithm of Artistic Style."Preprint, submitted September 2, 2015. https://arxiv.org/abs/1508.06576