clear;clc;
biasVarFile = fopen('biasVar.txt', 'w');
fclose(biasVarFile);

% define datasets of experiment
myDatasets = {
    'cocomo81o','cocomo81s','cocomo81','cocomo81e',...
    'desharnaisL1','desharnais','desharnaisL2',...
    'desharnaisL3',...
    'nasa93','nasa93_center_1','nasa93_center_2','nasa93_center_5',...
    'sdr'...
    'albrecht',...
    'finnish','kemerer','maxwell','miyazaki94','telecom1',...
    %'china'
    };

datasetSize = size(myDatasets,2);
for datasetCounter = 1:datasetSize
    
    % convert the order to icse
    orderInIcse = [41	51	3	4	17	20	27	36	48	65	80	5	6	67	66	12	39	86	81	89	60	61	78	76	59	42	87	74	82	30	31	79	73	55	23	88	63	75	44	45	68	72	69	47	85	49	77	32	33	64	70	50	52	83	40	58	1	2	54	71	26	34	84	37	57	7	8	24	16	11	13	56	35	90	21	22	15	19	62	43	46	38	25	9	10	29	28 18 14 53];
    
    eval(['load(''workspaces\3way\',char(myDatasets(datasetCounter)),'.mat'');']);
    actuals3Way = actual10by3values;
    algorithmResults3Way = algorithmResults(orderInIcse,:);
    biasValues3Way = -1*ones(size(algorithmResults,1),1);
    varValues3Way = -1*ones(size(algorithmResults,1),1);
    
    eval(['load(''workspaces\LOO\',char(myDatasets(datasetCounter)),'.mat'');']);
    actualsLOO = actualsToSave';
    algorithmResultsLOO = algorithmResults(orderInIcse,:);
    biasValuesLOO = -1*ones(size(algorithmResults3Way,1),1);
    varValuesLOO = -1*ones(size(algorithmResults3Way,1),1);
    
    % below variables keep track of statistical difference
    biasDifference = -1 * ones(size(varValues3Way,1),1);
    varDifference = -1 * ones(size(varValues3Way,1),1);
    
    % calculate bias and variance
    labels = {};
    labelsBias = {}; % this one keeps the labels of statistically different guys
    labelsVar = {}; % this one keeps the labels of statistically different guys
    for i = 1:size(varValues3Way,1)
        labels =[labels i];
        fHatMean3Way = mean(algorithmResults3Way(i,:));
        biasSeparate3Way = ((fHatMean3Way - actuals3Way).^2);
        biasValues3Way(i) = mean(biasSeparate3Way);
        varSeparate3Way = ((fHatMean3Way - algorithmResults3Way(i,:)).^2);
        varValues3Way(i) = mean(varSeparate3Way);
        
        fHatMeanLOO = mean(algorithmResultsLOO(i,:));
        biasSeparateLOO = ((fHatMeanLOO - actualsLOO).^2);
        biasValuesLOO(i) = mean(biasSeparateLOO);
        varSeparateLOO = ((fHatMeanLOO - algorithmResultsLOO(i,:)).^2);
        varValuesLOO(i) = mean(varSeparateLOO);
        
        % if there is a statistical difference in bias then mark it with a 1
        [P,H] = ranksum(biasSeparate3Way,biasSeparateLOO);
        if H == 0   % if they are the same
            biasDifference(i) = 0;
        else
            biasDifference(i) = 1;
            labelsBias =[labelsBias i];
        end
        
        % if there is a statistical difference in bias then mark it with a 1
        [P,H] = ranksum(varSeparate3Way,varSeparateLOO);
        if H == 0   % if they are the same
            varDifference(i) = 0;
        else
            varDifference(i) = 1;
            labelsVar =[labelsBias i];
        end
    end
    
    
    % now start plotting all bias-var plots without statistical check
    h = figure;hold on;
    set(gca,'FontSize',20);
    plot(log(biasValues3Way), log(varValues3Way),'.');
    axis([0 20 0 20]);
    text(log(biasValues3Way),log(varValues3Way), labels,'EdgeColor','green','LineStyle',':','FontSize',16);
    plot(log(biasValuesLOO), log(varValuesLOO),'.');
    text(log(biasValuesLOO),log(varValuesLOO), labels,'EdgeColor','red','LineStyle','-.','FontSize',16);
    xlabel('Logged Bias Values');
    ylabel('Logged Variance Values');
    title([char(myDatasets(datasetCounter)) ', ' num2str(size(biasValues3Way,1)) ' Methods Shown.' ...
        ' Green=3Way and Red=LOO.']);
    eval(['print(h,''-dpng'', ''biasVarPlots\',char(myDatasets(datasetCounter)),'.png'');']);
    hold off;close all;
    %     % then start plotting all bias-var plots this time only those that are
    %     % statistically different
    %     h = figure;hold on;
    %     set(gca,'FontSize',20);
    %     biasValues3Way = biasValues3Way(find(biasDifference == 1));
    %     varValues3Way = varValues3Way(find(varDifference == 1));
    %     biasValuesLOO = biasValuesLOO(find(biasDifference == 1));
    %     varValuesLOO = varValuesLOO(find(varDifference == 1));
    %     plot(log(biasValues3Way), log(varValues3Way),'.');
    %     axis([0 20 0 20]);
    %     text(log(biasValues3Way),log(varValues3Way), labelsStats,'EdgeColor','green','LineStyle',':','FontSize',16);
    %     plot(log(biasValuesLOO), log(varValuesLOO),'.');
    %     text(log(biasValuesLOO),log(varValuesLOO), labelsStats,'EdgeColor','red','LineStyle','-.','FontSize',16);
    %     xlabel('Logged Bias Values');
    %     ylabel('Logged Variance Values');
    %     title([char(myDatasets(datasetCounter)) ', ' num2str(size(biasValues3Way,1)) ' Methods Shown.' ...
    %         ' Green=3Way and Red=LOO.']);
    %     eval(['print(h,''-dpng'', ''biasVarPlots\withStat',char(myDatasets(datasetCounter)),'.png'');']);
    %     hold off;close all;
    biasVarFile = fopen('biasVar.txt', 'a');
    fprintf(biasVarFile, [char(myDatasets(datasetCounter)) '\n']);
    fprintf(biasVarFile,  'biasValues3Way');
    if size(labelsBias,2) > 0
        for tmpCounter = 1:(size(labelsBias,2)-1)
            fprintf(biasVarFile,  [',' num2str((biasValues3Way(cell2mat(labelsBias(tmpCounter)))))]);
        end
        fprintf(biasVarFile,  [',' num2str((biasValues3Way(cell2mat(labelsBias(size(labelsBias,2)))))) '\n']);
        
        fprintf(biasVarFile,  'biasValuesLOO');
        for tmpCounter = 1:(size(labelsBias,2)-1)
            fprintf(biasVarFile,  [',' num2str(biasValuesLOO(cell2mat(labelsBias(tmpCounter))))]);
        end
        fprintf(biasVarFile,  [',' num2str(biasValuesLOO(cell2mat(labelsBias(size(labelsBias,2))))) '\n']);
    else
        fprintf(biasVarFile,  '\nbiasValuesLOO\n');
    end
    
    fprintf(biasVarFile,  'varValues3Way');
    if size(labelsVar,2) > 0
        for tmpCounter = 1:(size(labelsVar,2)-1)
            fprintf(biasVarFile,  [',' num2str(varValues3Way(cell2mat(labelsVar(tmpCounter))))]);
        end
        fprintf(biasVarFile,  [',' num2str(varValues3Way(cell2mat(labelsVar(size(labelsVar,2))))) '\n']);
        
        fprintf(biasVarFile,  'varValuesLOO');
        for tmpCounter = 1:(size(labelsVar,2)-1)
            fprintf(biasVarFile,  [',' num2str(varValuesLOO(cell2mat(labelsVar(tmpCounter))))]);
        end
        fprintf(biasVarFile,  [',' num2str(varValuesLOO(cell2mat(labelsVar(size(labelsVar,2))))) '\n\n']);
    else
        fprintf(biasVarFile,  '\nvarValuesLOO\n');
    end
end
fclose(biasVarFile);