function evaluate_ref_models()
% Evaluate MatConvNet reference models to validate them

addpath(fullfile(fileparts(mfilename('fullpath')), '..','examples', 'imagenet')) ;

models = {...
  'resnet-50-dag', 2015, ...
  'resnet-101-dag', 2015, ...
  'resnet-152-dag', 2015, ...
  'matconvnet-alex', 2012, ...
  'matconvnet-vgg-s', 2013, ...
  'matconvnet-vgg-m', 2013, ...
  'matconvnet-vgg-f', 2013, ...
  'matconvnet-vgg-verydeep-16', 2014, ...
  'caffe-ref', 2012, ...
  'caffe-alex', 2012, ...
  'vgg-s', 2013, ...
  'vgg-m', 2013, ...
  'vgg-f', 2013, ...
  'vgg-m-128', 2013, ...
  'vgg-m-1024', 2013, ...
  'vgg-m-2048', 2013, ...
  'vgg-verydeep-19', 2014, ...
  'vgg-verydeep-16', 2014, ...
  'googlenet-dag', 2014} ;

for i = 1:2:numel(models)
  opts.dataDir = fullfile('data', 'imagenet12-s256') ;
  opts.expDir = fullfile('data','models-eval', models{i}) ;
  opts.imdbPath = fullfile(opts.expDir, 'imdb.mat');
  opts.modelPath = fullfile('data', 'models', ...
    sprintf('imagenet-%s.mat', models{i})) ;
  opts.lite = false ;
  opts.numFetchThreads = 12 ;
  opts.train.batchSize = 128 ;
  opts.train.numEpochs = 1 ;
  opts.train.gpus = [] ;
  opts.train.prefetch = true ;
  opts.train.expDir = opts.expDir ;

  resultPath = fullfile(opts.expDir, 'results.mat') ;
  if ~exist(resultPath)
    results = cnn_imagenet_evaluate(opts) ;
    save(fullfile(opts.expDir, 'results.mat'), 'results') ;
  end
end

fprintf('|%30s|%10s|%10s|%10s|%10s|\n', 'model', 'introduced', ...
        'top-1 err.', 'top-5 err.', 'images/s') ;
fprintf(strrep(sprintf('|%30s|%10s|%10s|%10s|%10s|\n', '', '', '', '', ''),' ','-')) ;

years = horzcat(models{2:2:end}) ;
[~,perm] = sort(years, 'descend') ;

for i = perm
  i = 2*(i - 1) + 1 ;
  opts.expDir = fullfile('data', 'models-eval', models{i}) ;
  resultPath = fullfile(opts.expDir, 'results.mat') ;
  load(resultPath, 'results') ;

  if isfield(results.val, 'error')
    top5 = results.val.error(2,end) ;
    top1 = results.val.error(1,end) ;
    speed = results.val.speed(end) ;
  else
    top5 = results.val.top5err(1,end) ;
    top1 = results.val.top1err(1,end) ;
    speed = results.val.num / results.val.time ;
  end

  fprintf('|%30s|%10d|%10s|%10s|%10s|\n', ...
    models{i}, ...
    models{i+1}, ...
    sprintf('%5.1f',top1*100), ...
    sprintf('%5.1f',top5*100), ...
    sprintf('%5.1f',speed)) ;
end

% (cd data/models ; md5sum *.mat | xargs  printf '| %-33s| %-40s|\n')
