;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
; This file is part of "NOVA":  NOVA = search + COCOMO tools
; Copyright, 2008, Tim Menzies tim@menzies.us
;
; NOVA is free software: you can redistribute it and/or modify
; it under the terms of the GNU General Public License as published by
; the Free Software Foundation, either version 3 of the License, or
; (at your option) any later version.
;
; NOVA is distributed in the hope that it will be useful,
; but WITHOUT ANY WARRANTY; without even the implied warranty of
; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
; GNU General Public License for more details.
; You should have received a copy of the GNU General Public License
; a long with NOVA.  If not, see <http://www.gnu.org/licenses/>.
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;

(defvar *engines* '((isamp      "isamp"      "ISAMP")
		    (maxwalksat "maxwalksat" "MaxWalkSat")
		    (sa         "sa"         "SA")
		    (a-star     "a-star"     "A*")
		    (beam       "beam"       "Beam")
		    (keys       "keys"       "KEYS")
		    (seesaw     "seesaw"     "SEESAW-R")
		    (seesaw2    "seesaw2"    "SEESAW-D")))

(defvar *case-studies* '((default "default" "Default" default-case-study)
			 (flight  "flight"  "Flight"  flight-case-study)
			 (ground  "ground"  "Ground"  ground-case-study)
			 (osp     "osp"     "OSP"     osp-case-study)
			 (osp2    "osp2"    "OSP2"    osp2-case-study)
                         (tps10   "tps10"   "TPS10"   tps10-case-study)
                         (tps100  "tps100"  "TPS100"  tps100-case-study)
                         (tps1000 "tps1000" "TPS1000" tps1000-case-study)))

(defvar *policies* '((strategic "strategic" "Strategic" (strategic))
		     (tactical  "tactical"  "Tactical"  (tactical))
		     (nil       "all"       "All"       nil)))

(defun write-paths (engine case-study-name policy-name paths &optional (calculate-p t))
  "Writes paths to the results folder."
  (when calculate-p
    (dolist (path paths)
      (dolist (state (states? path))
	(effort? state)
	(months? state)
	(defects? state)
	(threat? state)
	(energy? state))))
  (let ((e  (assoc engine *engines*))
	(cs (assoc case-study-name *case-studies*))
	(p  (assoc policy-name *policies*)))     
    (with-open-file (str (concatenate 'string "../results/" (second cs) "/" (second p) "/" (second e) ".out")
			 :direction :output :if-exists :supersede)
      (format str "~a~%" paths))))

(defun read-paths (engine case-study-name policy-name)
  "Reads paths from the results folder."
  (let ((e  (assoc engine *engines*))
	(cs (assoc case-study-name *case-studies*))
	(p  (assoc policy-name *policies*)))     
    (with-open-file (str (concatenate 'string "../results/" (second cs) "/" (second p) "/" (second e) ".out"))
      (read str))))

(defun search-and-write-paths (engine)
  "Searches all combinations of *case-studies* and *policies* and writes the paths."
  (dolist (cs *case-studies*)
    (dolist (p *policies*)
      (format t "~a/~a~%" (third cs) (third p))
      (write-paths engine (first cs) (first p)
		   (funcall engine :case-study (funcall (fourth cs)) :policies (fourth p))))))

(defun search-rig ()
  "Searches everything."
  (dolist (e *engines*)
    (format t "~a~%" (third e))
    (search-and-write-paths (first e))))

(defun score-engine (engine score-fn &key (case-study-name 'default) (policy-name nil))
  "Scores the given engine."
  (let ((e     (assoc engine *engines*))
        (paths (read-paths engine case-study-name policy-name)))
    (mapcar #'(lambda (path) (funcall score-fn (min-energy-ttest? path))) paths)))

(defun score-baseline (engine score-fn &key (case-study-name 'default) (policy-name nil))
  "Generates a baseline score."
  (let ((e     (assoc engine *engines*))
        (paths (read-paths engine case-study-name policy-name)))
    (mapcar #'(lambda (path) (funcall score-fn (first (path-states path)))) paths)))

(defun score-engines (score-fn &key (case-study-name 'default) (policy-name nil) (baseline nil))
  "Scores all engines for the given case study and policy."
  (let (output)
    (push (list "Baseline" (score-baseline (first (first *engines*))
                                           score-fn
                                           :case-study-name case-study-name
                                           :policy-name policy-name))
          output)
    (dolist (engine *engines* (nreverse output))
      (push (list (third engine) (score-engine (first engine)
                                               score-fn
                                               :case-study-name case-study-name
                                               :policy-name policy-name))
            output))))

(defun write-lc-data (state file &optional (num-simulations 1000))
  "Writes local calibration data for the given state into a CSV file."
  (init-db)
  (kloc! (state-kloc-min state) (state-kloc-max state))
  (apply-treatments (append (state-ignore state)
			    (state-open   state)
			    (state-closed state)))
  (apply-treatments (state-open state))
  (let ((sfs '(prec flex resl team pmat))
	(ems '(time stor data pvol ruse rely docu acap pcap pcon aexp plex ltex tool sced cplx site)))
    (with-open-file (str file :direction :output :if-exists :supersede)
      (format str "~{~a,~}" sfs)
      (format str "~{~a,~}" ems)
      (format str "A,B,KSLOC,EFFORT~%")
      (dotimes (i num-simulations)
	(zaps)
	(format str "~{~,12f,~}" (mapcar #'sf2effort sfs))
	(format str "~{~,12f,~}" (mapcar #'em2effort ems))
	(format str "~,12f,~,12f,~,12f,~,12f~%" (! 'a) (! 'b) (! 'kloc) (effort))))))

(defun search-and-write-lc-data (engine)
  "Searches all combinations of *case-studies* and *policies* and writes LC data."
  (let ((e (assoc engine *engines*)))
    (dolist (cs *case-studies*)
      (dolist (p *policies*)
	(format t "~a ~a~%" (third cs) (third p))
	(write-lc-data (min-energy-ttest? (funcall engine :case-study (funcall (fourth cs)) :policies (fourth p)))
		       (concatenate 'string "../results/lc/" (second e) "/" (second cs) "-" (second p) ".csv"))))))

(defun create-win-loss-table (>wins data)
  "Compares data using Mann-Whitney and creates a win-loss table."
  (let (table)
    (dotimes (i (length data) (nreverse table))
      (let ((wins   0)
	    (losses 0)
	    (ties   0))
	(dotimes (j (length data))
	  (when (not (eq i j))
	    (case (mann-whitney (second (elt data i)) (second (elt data j)))
	      (-1 (if >wins (incf losses) (incf wins)))
	      ( 0 (incf ties))
	      ( 1 (if >wins (incf wins) (incf losses))))))
	(push (list (first (elt data i)) wins losses ties (second (elt data i))) table)))))

(defun print-win-loss-table (str table)
  "Prints a win-loss table to the specified stream."
  (labels ((repeat-string (string n)
	     (let ((output ""))
	       (dotimes (i n output)
		 (setf output (concatenate 'string output string))))))
    (let* ((label-width (apply #'max (mapcar #'(lambda (row) (length (first row))) table)))
	   (wlt-max     (apply #'max (mapcar #'(lambda (row) (length (format nil "~a" (max (second row)
                                                                                           (third  row)
                                                                                           (fourth row)))))
					     table)))
	   (wlt-width   (max wlt-max 3)))
      (format str "~vt ~v<W~> ~:*~v<L~> ~:*~v<T~>~%" label-width wlt-width)
      (format str "~a ~a ~:*~a ~:*~a~%" (repeat-string "-" label-width) (repeat-string "-" wlt-width))
      (dolist (row (sort (copy-tree table) #'(lambda (row1 row2)
					       (if (eql (third row1) (third row2))
						   (> (second row1) (second row2))
						   (< (third row1) (third row2))))))
	(format str "~va" label-width (first row))
	(format str " ~v<~a~>" wlt-width (second row))
	(format str " ~v<~a~>" wlt-width (third row))
	(format str " ~v<~a~>" wlt-width (fourth row))
	(format str "~%")))))

(defun print-win-loss-tables (str tables)
  "Prints multiple win-loss tables to the specified stream."
  (let (totals)
    (dolist (table tables)
      (format str "~a:~%" (first table))
      (print-win-loss-table str (second table))
      (format str "~%")
      (dolist (row (second table))
        (let ((totals-row (assoc (first row) totals)))
          (when (not totals-row)
            (push (list (first row) 0 0 0) totals)
            (setf totals-row (assoc (first row) totals)))
          (incf (second totals-row) (second row))
          (incf (third  totals-row) (third  row))
          (incf (fourth totals-row) (fourth row)))))
    (format str "Total:~%")
    (print-win-loss-table str totals)))

(defun print-all-win-loss-tables (str)
  "Prints win-loss tables for all engines to the specified stream."
  (let ((tables nil))
    (dolist (cs *case-studies*)
      (dolist (p *policies*)
        (push (list (format nil "~a/~a" (third cs) (third p))
                    (create-win-loss-table nil (score-engines #'energy?
                                                              :case-study-name (first cs)
                                                              :policy-name (first p)
                                                              :baseline t)))
              tables)))
    (print-win-loss-tables str tables)))

(defun print-boxplot (str >wins data title &optional (precision 0))
  "Compares data using Mann-Whitney and prints a LaTeX boxplot to the specified stream."
  (format str "{\\scriptsize\\begin{center}~%")
  (format str "\\begin{tabular}{l@{~~} c@{~~} l@{~~} r@{~~} r@{~~} r@{~~} r@{~~} r@{~~} c}~%")
  (format str "& & & \\multicolumn{5}{c}{quartiles} \\\\~%")
  (format str "\\cline{4-8}~%")
  (format str "& rank & & 0 & 25 & 50 & 75 & 100 \\\\~%")
  (format str "\\hline~%")
  (format str "~a " title)
  (let* ((table    (create-win-loss-table >wins data))
         (all-data (apply #'append (mapcar #'fifth table)))
         previous-row
         rank)
    (dolist (row (sort (copy-tree table) #'(lambda (row1 row2)
                                             (if (eql (third row1) (third row2))
                                                 (> (second row1) (second row2))
                                                 (< (third row1) (third row2))))))
      (let ((min (apply #'min (fifth row)))
            (max (apply #'max (fifth row))))
        (multiple-value-bind (med spread upper lower) (median (fifth row))
          (if (not previous-row)
              (setf rank 1)
              (if (> (third row)
                     (third previous-row))
                  (setf rank (1+ rank))))
          (if (zerop precision)
              (format str "& ~d & ~a & ~d & ~d & ~d & ~d & ~d & "
                      rank
                      (first row)
                      (round min)
                      (round lower)
                      (round med)
                      (round upper)
                      (round max))
              (format str "& ~d & ~a & ~,vf & ~,vf & ~,vf & ~,vf & ~,vf & "
                      rank
                      (first row)
                      precision min
                      precision lower
                      precision med
                      precision upper
                      precision max))
          (let ((min%   (percentile min   all-data))
                (lower% (percentile lower all-data))
                (med%   (percentile med   all-data))
                (upper% (percentile upper all-data))
                (max%   (percentile max   all-data)))
            (format str "\\boxplot{~f}{~f}{~f}{~f}{~f} \\\\~%"
                    lower%
                    (- med% lower%)
                    med%
                    (- upper% med%)
                    upper%))))
      (setf previous-row row)))
  (format str "\\hline~%")
  (format str "\\multicolumn{8}{c}{~~} & 50\\%~%")
  (format str "\\end{tabular}~%")
  (format str "\\end{center}}~%"))

(defun print-simple-boxplot (str >wins data title &optional (precision 0))
  "Compares data using Mann-Whitney and prints a LaTeX boxplot to the specified stream."
  (format str "{\\scriptsize\\begin{center}~%")
  (format str "\\begin{tabular}{c l @{} c @{} c @{} c @{}}~%")
  (format str "\\multicolumn{5}{l}{~a} \\\\~%" title)
  (format str "Rank & Change \\\\~%")
  (format str "\\hline~%")
  (let* ((table    (create-win-loss-table >wins data))
         (all-data (apply #'append (mapcar #'fifth table)))
         previous-row
         rank)
    (dolist (row (sort (copy-tree table) #'(lambda (row1 row2)
                                             (if (eql (third row1) (third row2))
                                                 (> (second row1) (second row2))
                                                 (< (third row1) (third row2))))))
      (let ((min (apply #'min (fifth row)))
            (max (apply #'max (fifth row))))
        (multiple-value-bind (med spread upper lower) (median (fifth row))
          (if (not previous-row)
              (setf rank 1)
              (if (> (third row)
                     (third previous-row))
                  (setf rank (1+ rank))))
          (format str "~d & ~a & \\vline & "
                  rank
                  (first row))
          (let ((min%   (percentile min   all-data))
                (lower% (percentile lower all-data))
                (med%   (percentile med   all-data))
                (upper% (percentile upper all-data))
                (max%   (percentile max   all-data)))
            (format str "\\boxplot{~f}{~f}{~f}{~f}{~f} & \\vline \\\\~%"
                    lower%
                    (- med% lower%)
                    med%
                    (- upper% med%)
                    upper%))))
      (setf previous-row row)))
  (format str "\\hline~%")
  (format str "& & 0\\% & 50\\% & 100\\%~%")
  (format str "\\end{tabular}~%")
  (format str "\\end{center}}~%"))

(defun print-boxplots (str engine-score-fn strategy-score-fn title &optional (precision 0))
  "Compares data using Mann-Whitney and prints LaTeX boxplots to the specified stream."
  (format str "\\documentclass{article}~%~%")
  (format str "\\newcommand{\\boxplot}[5]{~%")
  (format str "\\begin{picture}(100, 10)~%")
  (format str "\\put(#1, 1){\\line(0, 1){6}}~%")
  (format str "\\put(#1, 4){\\line(1, 0){#2}}~%")
  (format str "\\put(#3, 4){\\circle*{3}}~%")
  (format str "\\put(#3, 4){\\line(1, 0){#4}}~%")
  (format str "\\put(#5, 1){\\line(0, 1){6}}~%")
  (format str "\\put(50, 2){\\line(0, 1){4}}~%")
  (format str "\\end{picture}~%")
  (format str "}~%~%")
  (format str "\\title{~a}~%" title)
  (format str "\\author{}~%")
  (format str "\\date{}~%~%")
  (format str "\\begin{document}~%")
  (format str "\\maketitle~%")
  (dolist (cs *case-studies*)
    (dolist (p *policies*)
      (print-simple-boxplot str nil (append (score-engines engine-score-fn :case-study-name (first cs) :policy-name (first p))
                                     (score-strategies strategy-score-fn :case-study (funcall (fourth cs))))
                     (concatenate 'string (third cs) "/" (third p))
                     precision)))
  (format str "\\end{document}~%"))

(defun print-boxplots-only-engines (str engine-score-fn title &optional (precision 0))
  "Compares data using Mann-Whitney and prints LaTeX boxplots to the specified stream."
  (format str "\\documentclass{article}~%~%")
  (format str "\\newcommand{\\boxplot}[5]{~%")
  (format str "\\begin{picture}(100, 10)~%")
  (format str "\\put(#1, 1){\\line(0, 1){6}}~%")
  (format str "\\put(#1, 4){\\line(1, 0){#2}}~%")
  (format str "\\put(#3, 4){\\circle*{3}}~%")
  (format str "\\put(#3, 4){\\line(1, 0){#4}}~%")
  (format str "\\put(#5, 1){\\line(0, 1){6}}~%")
  (format str "\\put(50, 2){\\line(0, 1){4}}~%")
  (format str "\\end{picture}~%")
  (format str "}~%~%")
  (format str "\\title{~a}~%" title)
  (format str "\\author{}~%")
  (format str "\\date{}~%~%")
  (format str "\\begin{document}~%")
  (format str "\\maketitle~%")
  (dolist (cs *case-studies*)
    (dolist (p *policies*)
      (print-simple-boxplot str nil (score-engines engine-score-fn :case-study-name (first cs) :policy-name (first p))
                            (concatenate 'string (third cs) "/" (third p))
                            precision)))
  (format str "\\end{document}~%"))

(defun write-all-boxplots ()
  "Prints LaTeX boxplots for all four dimensions to the results folder."
  (dolist (score-fn '((effort?  effort  "effort")
                      (months?  months  "months")
                      (defects? defects "defects")
                      (threat?  threat  "threat")))
    (with-open-file (str (concatenate 'string "../results/boxplots/" (third score-fn) ".tex") :direction :output :if-exists :supersede)
      (print-boxplots-only-engines str
                      (first score-fn)
                      ; (second score-fn)
                      (string-capitalize (third score-fn))
                      (if (eq (first score-fn) 'threat?)
                          2
                          0)))))

(defun write-multi-plot (case-study-name policy-name)
  "Writes a gnuplot script for constraints vs. energy multiplots.  Assumes six engines."
  (let* ((cs   (assoc case-study-name *case-studies*))
         (p    (assoc policy-name *policies*))
         (file (concatenate 'string "multi-" (second cs) "-" (second p)))
         (plt  (concatenate 'string "../results/plots/" file ".plt"))
         (png  (concatenate 'string file ".png")))
    (with-open-file (str plt :direction :output :if-exists :supersede)
      (format str "set terminal png font '/usr/share/fonts/truetype/ttf-bitstream-vera/Vera.ttf' 8~%")
      (format str "set output '~a'~%" png)
      (format str "set origin 0.0, 0.0~%")
      (format str "set size 1.0, 1.0~%")
      (format str "set multiplot~%")
      (dotimes (i (length *engines*))
        (let ((e (elt *engines* i)))
	  (format str "set origin ~,1f, ~,3f~%"
                  (if (zerop (mod i 2)) 0 0.5)
                  (if (> (1+ i) 2) 0.5 0))
	  (format str "set size 0.5, 0.5~%")
	  (format str "set title '~a'~%" (third e))
	  (format str "set xlabel 'Constraints'~%")
	  (format str "set ylabel 'Energy'~%")
	  (format str "unset key~%")
	  (format str "plot '-' with points pointsize 0 linetype 0, ")
	  (format str "'-' with points pointtype 5 pointsize 0.25 linetype -1~%")
          (dolist (e2 *engines*)
            (dolist (path (read-paths (first e2) case-study-name policy-name))
              (let ((state (min-energy-ttest? path)))
                (format str "~a ~f~%" (constraint-count state) (energy? state)))))
          (format str "end~%")
          (dolist (path (read-paths (first e) case-study-name policy-name))
            (let ((state (min-energy-ttest? path)))
              (format str "~a ~f~%" (constraint-count state) (energy? state))))
          (format str "end~%"))))))

(defun write-scatter-plots (case-study-name policy-name)
  "Writes a gnuplot script for constraints vs. energy scatterplots."
  (let ((cs   (assoc case-study-name *case-studies*))
        (p    (assoc policy-name *policies*)))
    (dolist (e *engines*)
      (let* ((file (concatenate 'string "scatter-" (second cs) "-" (second p) "-" (second e)))
             (plt  (concatenate 'string "../results/plots/" file ".plt"))
             (png  (concatenate 'string file ".png")))
        (with-open-file (str plt :direction :output :if-exists :supersede)
          (format str "set terminal png font '/usr/share/fonts/truetype/ttf-bitstream-vera/Vera.ttf' 8~%")
          (format str "set output '~a'~%" png)
	  (format str "set title '~a'~%" (third e))
	  (format str "set xlabel 'Constraints'~%")
	  (format str "set ylabel 'Energy'~%")
	  (format str "unset key~%")
	  (format str "plot '-' with points pointtype 6 pointsize 1.25 linetype 0, ")
	  (format str "'-' with points pointtype 7 pointsize 1.25 linetype -1~%")
          (dolist (e2 *engines*)
            (dolist (path (read-paths (first e2) case-study-name policy-name))
              (let ((state (min-energy-ttest? path)))
                (format str "~a ~f~%" (constraint-count state) (energy? state)))))
          (format str "end~%")
          (dolist (path (read-paths (first e) case-study-name policy-name))
            (let ((state (min-energy-ttest? path)))
              (format str "~a ~f~%" (constraint-count state) (energy? state))))
          (format str "end~%"))))))

(defun write-single-run-plot (engine case-study-name policy-name &optional (index 0))
  "Writes a gnuplot script for a constraints vs. (en/ef/mo/de/th) single run plot."
  (dolist (which '((energy? "energy" "Energy")
                   (effort? "effort" "Effort")
                   (months? "months" "Months")
                   (defects? "defects" "Defects")
                   (threat? "threat" "Threat")))
    (let* ((e    (assoc engine *engines*))
           (cs   (assoc case-study-name *case-studies*))
           (p    (assoc policy-name *policies*))
           (file (concatenate 'string "single-" (second e) "-" (second cs) "-" (second p) "-" (second which)))
           (plt  (concatenate 'string "../results/plots/" file ".plt"))
           (png  (concatenate 'string file ".png")))
      (with-open-file (str plt :direction :output :if-exists :supersede)
        (format str "set terminal png font '/usr/share/fonts/truetype/ttf-bitstream-vera/Vera.ttf' 8~%")
        (format str "set output '~a'~%" png)
        (format str "set xlabel 'Constraints'~%")
        (format str "set ylabel '~a'~%" (third which))
        (format str "unset key~%")
        (format str "plot '-' with linespoints pointtype 7 pointsize 1 linetype -1, '-' with points pointtype 6 pointsize 2 linetype -1~%")
        (let ((path (elt (read-paths (first e) case-study-name policy-name) index)))
          (dolist (state (states? path))
            (format str "~a ~f~%"
                    (constraint-count state)
                    (funcall (first which) state)))
          (format str "end~%")
          (let ((decision-point (min-energy-ttest? path)))
            (format str "~a ~f~%"
                    (constraint-count decision-point)
                    (funcall (first which) decision-point)))
          (format str "end~%"))))))

(defun print-constraint-frequency (str case-study-name policy-name)
  "Prints the number of times each constraint is seen in the minimum energy state."
  (let (constraints
        (num-paths 0))
    (dolist (e *engines*)
      (dolist (path (read-paths (first e) case-study-name policy-name))
        (incf num-paths)
        (dolist (constraint (constraints? (min-energy-ttest? path)))
          (when (not (assoc constraint constraints :test #'equal))
            (push (cons constraint 0) constraints))
          (incf (rest (assoc constraint constraints :test #'equal))))))
    (dolist (constraint (sort constraints #'(lambda (x y) (> (rest x) (rest y)))))
      (format str "~8a ~3<~a~>%~%" (first constraint) (round (* (/ (rest constraint) num-paths) 100))))))

(defun write-all-constraint-frequencies ()
  "Writes all constraint frequencies to the results folder."
  (dolist (cs *case-studies*)
    (dolist (p *policies*)
      (with-open-file (str (concatenate 'string "../results/freq/" (second cs) "-" (second p) ".txt")
                           :direction :output :if-exists :supersede)
        (print-constraint-frequency str (first cs) (first p))))))

(defun print-runtimes (engine)
  "Prints average runtimes for the given engine."
  (dolist (cs *case-studies*)
    (dolist (p *policies*)
      (let (runtimes)
        (dolist (path (read-paths engine (first cs) (first p)))
          (push (- (path-end path) (path-start path)) runtimes))
        (format t "~a/~a ~20t~,1f~%" (third cs) (third p) (mean runtimes))))))