% define datasets of experiment
myDatasets = {
    'cocomo81o','cocomo81s','cocomo81','cocomo81e',...
    'desharnaisL1','desharnais','desharnaisL2',...
    'desharnaisL3',...
    'nasa93','nasa93_center_1','nasa93_center_2','nasa93_center_5',...
    'sdr'...
    'albrecht',...
    'finnish','kemerer','maxwell','miyazaki94','telecom1',...
    %'china'
    };

% sum of win values for each error measure separately, over all datasets
allWinValues3Way = -1 * ones(7,90);
allLossValues3Way = -1 * ones(7,90);
allWinMinusLossValues3Way = -1 * ones(7,90);
allWinValuesLOO = -1 * ones(7,90);
allLossValuesLOO = -1 * ones(7,90);
allWinMinusLossValuesLOO = -1 * ones(7,90);

datasetSize = size(myDatasets,2);
for datasetCounter = 1:datasetSize
    
    % load win and loss values for 3 way
    eval(['load(''workspaces\3Way\',char(myDatasets(datasetCounter)),'.mat'');']);
    winValues3Way = winValues';
    lossValues3Way = lossValues';
    winMinusLossValues3Way = (winValues-lossValues)';
    % then sum them up
    allWinValues3Way = allWinValues3Way + winValues3Way;
    allLossValues3Way = allLossValues3Way + lossValues3Way;
    allWinMinusLossValues3Way = allWinMinusLossValues3Way + winMinusLossValues3Way;
    
    % now do the same for LOO
    % load win and loss values
    eval(['load(''workspaces\LOO\',char(myDatasets(datasetCounter)),'.mat'');']);
    winValuesLOO = winValues';
    lossValuesLOO = lossValues';
    winMinusLossValuesLOO = (winValues-lossValues)';
    % then sum them up
    allWinValuesLOO = allWinValuesLOO + winValuesLOO;
    allLossValuesLOO = allLossValuesLOO + lossValuesLOO;
    allWinMinusLossValuesLOO = allWinMinusLossValuesLOO + winMinusLossValuesLOO;
    
end

% now start calcuating the absolute rank difference w.r.t icse paper
orderInIcse = [41	51	3	4	17	20	27	36	48	65	80	5	6	67	66	12	39	86	81	89	60	61	78	76	59	42	87	74	82	30	31	79	73	55	23	88	63	75	44	45	68	72	69	47	85	49	77	32	33	64	70	50	52	83	40	58	1	2	54	71	26	34	84	37	57	7	8	24	16	11	13	56	35	90	21	22	15	19	62	43	46	38	25	9	10	29	28 18 14 53];
myOrigin = [1:90];
allWinValues3Way = allWinValues3Way(:,orderInIcse);
allLossValues3Way = allLossValues3Way(:,orderInIcse);
allWinMinusLossValues3Way = allWinMinusLossValues3Way(:,orderInIcse);
allWinValuesLOO = allWinValuesLOO(:,orderInIcse);
allLossValuesLOO = allLossValuesLOO(:,orderInIcse);
allWinMinusLossValuesLOO = allWinMinusLossValuesLOO(:,orderInIcse);

% below variables will keep the abs-rank change for different error
% measures w.r.t. win, loss and (win-loss)
stabilityMAR3Way = zeros(90,3);
stabilityMMRE3Way = zeros(90,3);
stabilityMMER3Way = zeros(90,3);
stabilityMBRE3Way = zeros(90,3);
stabilityMIBRE3Way = zeros(90,3);
stabilityMDMRE3Way = zeros(90,3);
stabilityPRED3Way = zeros(90,3);

stabilityMARLOO = zeros(90,3);
stabilityMMRELOO = zeros(90,3);
stabilityMMERLOO = zeros(90,3);
stabilityMBRELOO = zeros(90,3);
stabilityMIBRELOO = zeros(90,3);
stabilityMDMRELOO = zeros(90,3);
stabilityPREDLOO = zeros(90,3);

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


% now get sorted indices for win, loss and (win-loss)
%%%% MAR 3-Way %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
winIndices = []; lossIndices = []; winMinusLossIndices = [];
[C winIndices] = sort(allWinValues3Way(1,:), 'descend');
stabilityMAR3Way(:,1) = abs(winIndices-myOrigin)';
[C lossIndices] = sort(allLossValues3Way(1,:), 'ascend');
stabilityMAR3Way(:,2) = abs(lossIndices-myOrigin)';
[C winMinusLossIndices] = sort(allWinMinusLossValues3Way(1,:), 'descend');
stabilityMAR3Way(:,3) = abs(winMinusLossIndices-myOrigin)';
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

%%%% MMRE 3-Way %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
winIndices = []; lossIndices = []; winMinusLossIndices = [];
[C winIndices] = sort(allWinValues3Way(2,:), 'descend');
stabilityMMRE3Way(:,1) = abs(winIndices-myOrigin)';
[C lossIndices] = sort(allLossValues3Way(2,:), 'ascend');
stabilityMMRE3Way(:,2) = abs(lossIndices-myOrigin)';
[C winMinusLossIndices] = sort(allWinMinusLossValues3Way(2,:), 'descend');
stabilityMMRE3Way(:,3) = abs(winMinusLossIndices-myOrigin)';
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

%%%% MMER 3-Way %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
winIndices = []; lossIndices = []; winMinusLossIndices = [];
[C winIndices] = sort(allWinValues3Way(3,:), 'descend');
stabilityMMER3Way(:,1) = abs(winIndices-myOrigin)';
[C lossIndices] = sort(allLossValues3Way(3,:), 'ascend');
stabilityMMER3Way(:,2) = abs(lossIndices-myOrigin)';
[C winMinusLossIndices] = sort(allWinMinusLossValues3Way(3,:), 'descend');
stabilityMMER3Way(:,3) = abs(winMinusLossIndices-myOrigin)';
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

%%%% MBRE 3-Way %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
winIndices = []; lossIndices = []; winMinusLossIndices = [];
[C winIndices] = sort(allWinValues3Way(4,:), 'descend');
stabilityMBRE3Way(:,1) = abs(winIndices-myOrigin)';
[C lossIndices] = sort(allLossValues3Way(4,:), 'ascend');
stabilityMBRE3Way(:,2) = abs(lossIndices-myOrigin)';
[C winMinusLossIndices] = sort(allWinMinusLossValues3Way(4,:), 'descend');
stabilityMBRE3Way(:,3) = abs(winMinusLossIndices-myOrigin)';
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

%%%% MIBRE 3-Way %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
winIndices = []; lossIndices = []; winMinusLossIndices = [];
[C winIndices] = sort(allWinValues3Way(5,:), 'descend');
stabilityMIBRE3Way(:,1) = abs(winIndices-myOrigin)';
[C lossIndices] = sort(allLossValues3Way(5,:), 'ascend');
stabilityMIBRE3Way(:,2) = abs(lossIndices-myOrigin)';
[C winMinusLossIndices] = sort(allWinMinusLossValues3Way(5,:), 'descend');
stabilityMIBRE3Way(:,3) = abs(winMinusLossIndices-myOrigin)';
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

%%%% MDMRE 3-Way %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
winIndices = []; lossIndices = []; winMinusLossIndices = [];
[C winIndices] = sort(allWinValues3Way(6,:), 'descend');
stabilityMDMRE3Way(:,1) = abs(winIndices-myOrigin)';
[C lossIndices] = sort(allLossValues3Way(6,:), 'ascend');
stabilityMDMRE3Way(:,2) = abs(lossIndices-myOrigin)';
[C winMinusLossIndices] = sort(allWinMinusLossValues3Way(6,:), 'descend');
stabilityMDMRE3Way(:,3) = abs(winMinusLossIndices-myOrigin)';
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

%%%% PRED 3-Way %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
winIndices = []; lossIndices = []; winMinusLossIndices = [];
[C winIndices] = sort(allWinValues3Way(7,:), 'descend');
stabilityPRED3Way(:,1) = abs(winIndices-myOrigin)';
[C lossIndices] = sort(allLossValues3Way(7,:), 'ascend');
stabilityPRED3Way(:,2) = abs(lossIndices-myOrigin)';
[C winMinusLossIndices] = sort(allWinMinusLossValues3Way(7,:), 'descend');
stabilityPRED3Way(:,3) = abs(winMinusLossIndices-myOrigin)';
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%% NOW START DOING THE STABILITY CALCULATIONS %%%%%%
%%%%%% FOR LOO BELOW BASICALLY DO THE SAME STUFF AGAIN %
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

%%%% MAR 3-Way %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
winIndices = []; lossIndices = []; winMinusLossIndices = [];
[C winIndices] = sort(allWinValuesLOO(1,:), 'descend');
stabilityMARLOO(:,1) = abs(winIndices(myOrigin))';
[C lossIndices] = sort(allLossValuesLOO(1,:), 'ascend');
stabilityMARLOO(:,2) = abs(lossIndices-myOrigin)';
[C winMinusLossIndices] = sort(allWinMinusLossValuesLOO(1,:), 'descend');
stabilityMARLOO(:,3) = abs(winMinusLossIndices-myOrigin)';
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

%%%% MMRE 3-Way %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
winIndices = []; lossIndices = []; winMinusLossIndices = [];
[C winIndices] = sort(allWinValuesLOO(2,:), 'descend');
stabilityMMRELOO(:,1) = abs(winIndices-myOrigin)';
[C lossIndices] = sort(allLossValuesLOO(2,:), 'ascend');
stabilityMMRELOO(:,2) = abs(lossIndices-myOrigin)';
[C winMinusLossIndices] = sort(allWinMinusLossValuesLOO(2,:), 'descend');
stabilityMMRELOO(:,3) = abs(winMinusLossIndices-myOrigin)';
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

%%%% MMER 3-Way %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
winIndices = []; lossIndices = []; winMinusLossIndices = [];
[C winIndices] = sort(allWinValuesLOO(3,:), 'descend');
stabilityMMERLOO(:,1) = abs(winIndices-myOrigin)';
[C lossIndices] = sort(allLossValuesLOO(3,:), 'ascend');
stabilityMMERLOO(:,2) = abs(lossIndices-myOrigin)';
[C winMinusLossIndices] = sort(allWinMinusLossValuesLOO(3,:), 'descend');
stabilityMMERLOO(:,3) = abs(winMinusLossIndices-myOrigin)';
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

%%%% MBRE 3-Way %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
winIndices = []; lossIndices = []; winMinusLossIndices = [];
[C winIndices] = sort(allWinValuesLOO(4,:), 'descend');
stabilityMBRELOO(:,1) = abs(winIndices-myOrigin)';
[C lossIndices] = sort(allLossValuesLOO(4,:), 'ascend');
stabilityMBRELOO(:,2) = abs(lossIndices-myOrigin)';
[C winMinusLossIndices] = sort(allWinMinusLossValuesLOO(4,:), 'descend');
stabilityMBRELOO(:,3) = abs(winMinusLossIndices-myOrigin)';
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

%%%% MIBRE 3-Way %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
winIndices = []; lossIndices = []; winMinusLossIndices = [];
[C winIndices] = sort(allWinValuesLOO(5,:), 'descend');
stabilityMIBRELOO(:,1) = abs(winIndices-myOrigin)';
[C lossIndices] = sort(allLossValuesLOO(5,:), 'ascend');
stabilityMIBRELOO(:,2) = abs(lossIndices-myOrigin)';
[C winMinusLossIndices] = sort(allWinMinusLossValuesLOO(5,:), 'descend');
stabilityMIBRELOO(:,3) = abs(winMinusLossIndices-myOrigin)';
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

%%%% MDMRE 3-Way %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
winIndices = []; lossIndices = []; winMinusLossIndices = [];
[C winIndices] = sort(allWinValuesLOO(6,:), 'descend');
stabilityMDMRELOO(:,1) = abs(winIndices-myOrigin)';
[C lossIndices] = sort(allLossValuesLOO(6,:), 'ascend');
stabilityMDMRELOO(:,2) = abs(lossIndices-myOrigin)';
[C winMinusLossIndices] = sort(allWinMinusLossValuesLOO(6,:), 'descend');
stabilityMDMRELOO(:,3) = abs(winMinusLossIndices-myOrigin)';
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

%%%% PRED 3-Way %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
winIndices = []; lossIndices = []; winMinusLossIndices = [];
[C winIndices] = sort(allWinValuesLOO(7,:), 'descend');
stabilityPREDLOO(:,1) = abs(winIndices-myOrigin)';
[C lossIndices] = sort(allLossValuesLOO(7,:), 'ascend');
stabilityPREDLOO(:,2) = abs(lossIndices-myOrigin)';
[C winMinusLossIndices] = sort(allWinMinusLossValuesLOO(7,:), 'descend');
stabilityPREDLOO(:,3) = abs(winMinusLossIndices-myOrigin)';
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

% and now plot for each error measure
% how much the absolute rank difference is
h=figure;
subplot(3,1,1); hold on; plot(stabilityMAR3Way(:,1),'r');plot(stabilityMARLOO(:,1),'g'); hold off;
title('Win');ylabel('MAR');legend('3Way','LOO');
subplot(3,1,2); hold on; plot(stabilityMAR3Way(:,2),'r');plot(stabilityMARLOO(:,2),'g'); hold off;
title('Loss');ylabel('MAR');legend('3Way','LOO');
subplot(3,1,3); hold on; plot(stabilityMAR3Way(:,3),'r');plot(stabilityMARLOO(:,3),'g'); hold off;
title('Win-Loss');ylabel('MAR');legend('3Way','LOO');
eval(['print(h,''-dpng'', ''plots\absRanks\MAR-Abs-Rank.png'');']);
eval(['print(h,''-dpdf'', ''plots\absRanks\MAR-Abs-Rank.pdf'');']);
close all;

h=figure;
subplot(3,1,1); hold on; plot(stabilityMMRE3Way(:,1),'r');plot(stabilityMMRELOO(:,1),'g'); hold off;
title('Win');ylabel('MMRE');legend('3Way','LOO');
subplot(3,1,2); hold on; plot(stabilityMMRE3Way(:,2),'r');plot(stabilityMMRELOO(:,2),'g'); hold off;
title('Loss');ylabel('MMRE');legend('3Way','LOO');
subplot(3,1,3); hold on; plot(stabilityMMRE3Way(:,3),'r');plot(stabilityMMRELOO(:,3),'g'); hold off;
title('Win-Loss');ylabel('MMRE');legend('3Way','LOO');
eval(['print(h,''-dpng'', ''plots\absRanks\MMRE-Abs-Rank.png'');']);
eval(['print(h,''-dpdf'', ''plots\absRanks\MMRE-Abs-Rank.pdf'');']);
close all;

h=figure;
subplot(3,1,1); hold on; plot(stabilityMMER3Way(:,1),'r');plot(stabilityMMERLOO(:,1),'g'); hold off;
title('Win');ylabel('MMER');legend('3Way','LOO');
subplot(3,1,2); hold on; plot(stabilityMMER3Way(:,2),'r');plot(stabilityMMERLOO(:,2),'g'); hold off;
title('Loss');ylabel('MMER');legend('3Way','LOO');
subplot(3,1,3); hold on; plot(stabilityMMER3Way(:,3),'r');plot(stabilityMMERLOO(:,3),'g'); hold off;
title('Win-Loss');ylabel('MMER');legend('3Way','LOO');
eval(['print(h,''-dpng'', ''plots\absRanks\MMER-Abs-Rank.png'');']);
eval(['print(h,''-dpdf'', ''plots\absRanks\MMER-Abs-Rank.pdf'');']);
close all;

h=figure;
subplot(3,1,1); hold on; plot(stabilityMBRE3Way(:,1),'r');plot(stabilityMBRELOO(:,1),'g'); hold off;
title('Win');ylabel('MBRE');legend('3Way','LOO');
subplot(3,1,2); hold on; plot(stabilityMBRE3Way(:,2),'r');plot(stabilityMBRELOO(:,2),'g'); hold off;
title('Loss');ylabel('MBRE');legend('3Way','LOO');
subplot(3,1,3); hold on; plot(stabilityMBRE3Way(:,3),'r');plot(stabilityMBRELOO(:,3),'g'); hold off;
title('Win-Loss');ylabel('MBRE');legend('3Way','LOO');
eval(['print(h,''-dpng'', ''plots\absRanks\MBRE-Abs-Rank.png'');']);
eval(['print(h,''-dpdf'', ''plots\absRanks\MBRE-Abs-Rank.pdf'');']);
close all;

h=figure;
subplot(3,1,1); hold on; plot(stabilityMIBRE3Way(:,1),'r');plot(stabilityMIBRELOO(:,1),'g'); hold off;
title('Win');ylabel('MIBRE');legend('3Way','LOO');
subplot(3,1,2); hold on; plot(stabilityMIBRE3Way(:,2),'r');plot(stabilityMIBRELOO(:,2),'g'); hold off;
title('Loss');ylabel('MIBRE');legend('3Way','LOO');
subplot(3,1,3); hold on; plot(stabilityMIBRE3Way(:,3),'r');plot(stabilityMIBRELOO(:,3),'g'); hold off;
title('Win-Loss');ylabel('MIBRE');legend('3Way','LOO');
eval(['print(h,''-dpng'', ''plots\absRanks\MIBRE-Abs-Rank.png'');']);
eval(['print(h,''-dpdf'', ''plots\absRanks\MIBRE-Abs-Rank.pdf'');']);
close all;

h=figure;
subplot(3,1,1); hold on; plot(stabilityMDMRE3Way(:,1),'r');plot(stabilityMDMRELOO(:,1),'g'); hold off;
title('Win');ylabel('MDMRE');legend('3Way','LOO');
subplot(3,1,2); hold on; plot(stabilityMDMRE3Way(:,2),'r');plot(stabilityMDMRELOO(:,2),'g'); hold off;
title('Loss');ylabel('MDMRE');legend('3Way','LOO');
subplot(3,1,3); hold on; plot(stabilityMDMRE3Way(:,3),'r');plot(stabilityMDMRELOO(:,3),'g'); hold off;
title('Win-Loss');ylabel('MDMRE');legend('3Way','LOO');
eval(['print(h,''-dpng'', ''plots\absRanks\MDMRE-Abs-Rank.png'');']);
eval(['print(h,''-dpdf'', ''plots\absRanks\MDMRE-Abs-Rank.pdf'');']);
close all;

h=figure;
subplot(3,1,1); hold on; plot(stabilityPRED3Way(:,1),'r');plot(stabilityPREDLOO(:,1),'g'); hold off;
title('Win');ylabel('PRED');legend('3Way','LOO');
subplot(3,1,2); hold on; plot(stabilityPRED3Way(:,2),'r');plot(stabilityPREDLOO(:,2),'g'); hold off;
title('Loss');ylabel('PRED');legend('3Way','LOO');
subplot(3,1,3); hold on; plot(stabilityPRED3Way(:,3),'r');plot(stabilityPREDLOO(:,3),'g'); hold off;
title('Win-Loss');ylabel('PRED');legend('3Way','LOO');
eval(['print(h,''-dpng'', ''plots\absRanks\PRED-Abs-Rank.png'');']);
eval(['print(h,''-dpdf'', ''plots\absRanks\PRED-Abs-Rank.pdf'');']);
close all;
