;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ; This file is part of "NOVA": NOVA = search + COCOMO tools ; Copyright, 2008, Tim Menzies tim@menzies.us ; ; NOVA is free software: you can redistribute it and/or modify ; it under the terms of the GNU General Public License as published by ; the Free Software Foundation, either version 3 of the License, or ; (at your option) any later version. ; ; NOVA is distributed in the hope that it will be useful, ; but WITHOUT ANY WARRANTY; without even the implied warranty of ; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ; GNU General Public License for more details. ; You should have received a copy of the GNU General Public License ; a long with NOVA. If not, see . ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; (in-package :xomo.experiments) (defun create-report-pdf (experiment-directory names search-engine-id policy-id case-study-id) (with-open-file (fstream (merge-pathnames "energy-vs-risk-exposure-compare.tex" experiment-directory) :direction :output :if-exists :supersede :if-does-not-exist :create) (macrolet ((out (control-string &rest args) `(format fstream ,control-string ,@args))) (out "~&\\documentclass[letterpaper,10pt]{article}") (out "~&\\usepackage{graphicx}") (out "~&% pagelayout") (out "~&%header") (out "~&\\setlength{\\topmargin}{-0.5in}") (out "~&\\setlength{\\headheight}{0in}") (out "~&\\setlength{\\headsep}{0in}") (out "~&%side margin") (out "~&\\setlength{\\oddsidemargin}{0in}") (out "~&\\setlength{\\evensidemargin}{0in}") (out "~&% text size") (out "~&\\setlength{\\textwidth}{6.5in}") (out "~&\\setlength{\\textheight}{9in}") (out "~&% footer") (out "~&\\setlength{\\footskip}{.25in}") (out "~&% define the title") (out "~&\\title{Energy vs Risk Exposure}") (out "~&\\begin{document}") (out "~&Left column was searched based on energy, right was searched based on risk-exposure\\\\") (out "~&Search Engine: ~a\\\\" search-engine-id) (out "~&Policy: ~a\\\\" policy-id) (out "~&Case Study: ~a\\\\" case-study-id) (dolist (name names) (out "~&\\includegraphics[width=3.25in]{energy/decisions-plot-~a.png}" name) (out "~&\\includegraphics[width=3.25in]{risk-exposure/decisions-plot-~a.png}" name)) (out "~&\\begin{minipage}[t]{3.5in}~%") (out "~&\\input{energy/decisions}") (out "~&\\end{minipage}~%") (out "~&\\begin{minipage}[t]{3.5in}~%") (out "~&\\input{risk-exposure/decisions}") (out "~&\\end{minipage}~%") (out "~&\\end{document}") (terpri fstream)))) (defun create-run-all-script (experiment-directory names) ;;TODO pathname of compare report needs moved into a variable (with-open-file (fstream (merge-pathnames "run-all.sh" experiment-directory) :direction :output :if-exists :supersede :if-does-not-exist :create) (macrolet ((out (control-string &rest args) `(format fstream ,control-string ,@args))) (dolist (subdir '("energy" "risk-exposure")) (out "~&(") (out "~&cd ~a" subdir) (dolist (name names) (out "~&bash decisions-plot-~a-gn.sh" name)) (out "~&)")) (out "~&pdflatex energy-vs-risk-exposure-compare.tex") (terpri fstream)))) (defun run-energy-vs-risk-exposure-decisions-experiment (&key (search-engine-id 'nova.impl.search-engines:seesaw-r) (policy-id 'xomo.policies:all) (case-study-id 'nova.impl.case-studies:default) (find-best-state-fn #'min-score-state-of-path-by-ttest?) (runs 100) (base-output-directory nova.settings:*nova-output-dir*)) (with-new-db (init-db) (let* ((experiment-output-directory (merge-pathnames "energy-vs-risk-exposure/" base-output-directory)) (score-name-fn-list (list (cons "energy" 'xomo.evaluation-methods:energy) (cons "risk-exposure" 'xomo.evaluation-methods:risk-exposure) (cons "effort" 'xomo.model:effort) (cons "months" 'xomo.model:months) (cons "defects" 'xomo.model:defects) (cons "threat" 'xomo.model:threat))) ;;search engine must add only one singleton per decision (search-engine (lookup-search-engine search-engine-id)) (policy (lookup-policy policy-id)) (case-study (lookup-case-study case-study-id)) (learn-by-energy-result (nova.apps.learning:nova-learn (lookup-evaluation-method 'xomo.evaluation-methods:energy) search-engine find-best-state-fn :case-study case-study :policy policy)) (learn-by-risk-exposure-result (nova.apps.learning:nova-learn (lookup-evaluation-method 'xomo.evaluation-methods:risk-exposure) search-engine find-best-state-fn :case-study case-study :policy policy))) (let ((learn-by-energy-output-subdirectory (merge-pathnames "energy/" experiment-output-directory)) (learn-by-risk-exposure-output-subdirectory (merge-pathnames "risk-exposure/" experiment-output-directory)) (learn-by-energy-decisions-scores (generate-decisions-scores learn-by-energy-result (mapcar #'cdr score-name-fn-list) :runs runs)) (learn-by-risk-exposure-decisions-scores (generate-decisions-scores learn-by-risk-exposure-result (mapcar #'cdr score-name-fn-list) :runs runs)) (learn-by-energy-best-decision (position (result-best-state learn-by-energy-result) (states? (result-path learn-by-energy-result)))) (learn-by-risk-exposure-best-decision (position (result-best-state learn-by-risk-exposure-result) (states? (result-path learn-by-risk-exposure-result))))) (ensure-directories-exist learn-by-energy-output-subdirectory :verbose t) (ensure-directories-exist learn-by-risk-exposure-output-subdirectory :verbose t) (dolist (score-name-fn score-name-fn-list) (let* ((lbe-scores (geta (cdr score-name-fn) learn-by-energy-decisions-scores)) (lbre-scores (geta (cdr score-name-fn) learn-by-risk-exposure-decisions-scores)) (miny (format-decisions-num 0)) (maxy (format-decisions-num (* 1.05 (apply #'max (append (mapcar #'second lbe-scores) (mapcar #'second lbre-scores) )))))) (output-decisions-plot (car score-name-fn) "evaluation by energy" (format-decisions-scores lbe-scores) learn-by-energy-best-decision miny maxy learn-by-energy-output-subdirectory) (output-decisions-plot (car score-name-fn) "evaluation by risk-exposure" (format-decisions-scores lbre-scores) learn-by-risk-exposure-best-decision miny maxy learn-by-risk-exposure-output-subdirectory))) (output-decisions "decisions based on evaluation by energy" learn-by-energy-result learn-by-energy-best-decision learn-by-energy-output-subdirectory) (output-decisions "decisions based on evaluation by risk-exposure" learn-by-risk-exposure-result learn-by-risk-exposure-best-decision learn-by-risk-exposure-output-subdirectory) (create-report-pdf experiment-output-directory (mapcar #'car score-name-fn-list) search-engine-id policy-id case-study-id) (create-run-all-script experiment-output-directory (mapcar #'car score-name-fn-list)) t))))