Please ignore the Above code;
The main core of the loop keeps running slower after many iterations. as an example it starts off by executing the inner part in 3 secs initially and after few hours it goes to 20 secs. Since I need the code to run for couple of days. I need to fix this issue. I feel it's the two parfeval's that are being called is creating an issue.
for epoch_loop = START_EPOCHS:TOTAL_EPOCHS
itr_no_train_per_epoch=0;
for mini_batch_itr_per_epoch_no=1:NO_OF_MINI_BATCH_ITRS_PER_EPOCH
for mini_batch_no=1:NO_OF_MINI_BATCHES_TRAIN+2
% track loop count
itr_no_train=itr_no_train+1;
itr_no_train_per_epoch=itr_no_train_per_epoch+1;
% Select buffer
[set_read_num, set_cpu_num, set_gpu_num] = buffer_select(mini_batch_no, MAX_SET_NO, set_read_num, set_cpu_num);
% Launch parallel asynchronous jobs
if (set_read_num~=0)
job1 = parfeval(POOL_val,@read_dataset_from_hdd, 1, mini_batch_no, CONST_DATA, 1);
end
if (set_cpu_num~=0)
job2 = parfeval(POOL_val,@compute_cpu_task, 1, BUFF_DATA_TRAIN.batch_file_read(:,:,:,:,set_cpu_num), CONST_DATA);
end
if (set_gpu_num~=0)
compute_gpu_task({BUFF_DATA_TRAIN.batch_file_process(:,:,:,:,set_gpu_num), BUFF_DATA_TRAIN.batch_file_label_read(:,:,:,:,set_gpu_num)});
end
% Collect result from parallel jobs
if (set_read_num~=0)
result{1} = fetchOutputs(job1);
BUFF_DATA_TRAIN.batch_file_read(:,:,:,:,set_read_num) = result{1}{1};
BUFF_DATA_TRAIN.batch_file_label_read(:,:,:,:,set_read_num) = result{1}{2};
end
if (set_cpu_num~=0)
result{2} = fetchOutputs(job2);
BUFF_DATA_TRAIN.batch_file_process(:,:,:,:,set_cpu_num) = result{2};
end
end
end
end
