;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
; This file is part of "NOVA":  NOVA = search + COCOMO tools
; Copyright, 2008, Tim Menzies tim@menzies.us
;
; NOVA is free software: you can redistribute it and/or modify
; it under the terms of the GNU General Public License as published by
; the Free Software Foundation, either version 3 of the License, or
; (at your option) any later version.
;
; NOVA is distributed in the hope that it will be useful,
; but WITHOUT ANY WARRANTY; without even the implied warranty of
; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
; GNU General Public License for more details.
; You should have received a copy of the GNU General Public License
; a long with NOVA.  If not, see <http://www.gnu.org/licenses/>.
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(in-package :xomo.experiments)

(register-experiment-by-parameters 'test-experiment
                                   :name "test-experiment"
                                   :treatments (list
                                                (make-experiment-treatment-by-ids "energy-seesaw-r"
                                                                                  'xomo.evaluation-methods:energy
                                                                                  'nova.impl.search-engines:seesaw-r
                                                                                  'min-score-state-of-path-by-ttest)
                                                (make-experiment-treatment-by-ids "energy-seesaw-d"
                                                                                  'xomo.evaluation-methods:energy
                                                                                  'nova.impl.search-engines:seesaw-d
                                                                                  'min-score-state-of-path-by-ttest))
                                   :scenarios (subseq (all-experiment-scenarios) 0 (min 2 (length (all-experiment-scenarios)))))
(export 'test-experiment)

(register-experiment-by-parameters 'seesaw-compare
                                   :name "seesaw-compare"
                                   :treatments (list
                                                (lookup-experiment-treatment 'nova.experiments:none)
                                                (make-experiment-treatment-by-ids "energy-seesaw-r"
                                                                                  'xomo.evaluation-methods:energy
                                                                                  'nova.impl.search-engines:seesaw-r
                                                                                  'min-score-state-of-path-by-ttest)
                                                (make-experiment-treatment-by-ids "energy-seesaw-d"
                                                                                  'xomo.evaluation-methods:energy
                                                                                  'nova.impl.search-engines:seesaw-d
                                                                                  'min-score-state-of-path-by-ttest)
                                                (make-experiment-treatment-by-ids "risk-exposure-seesaw-r"
                                                                                  'xomo.evaluation-methods:risk-exposure
                                                                                  'nova.impl.search-engines:seesaw-r
                                                                                  'min-score-state-of-path-by-ttest)
                                                (make-experiment-treatment-by-ids "risk-exposure-seesaw-d"
                                                                                  'xomo.evaluation-methods:risk-exposure
                                                                                  'nova.impl.search-engines:seesaw-d
                                                                                  'min-score-state-of-path-by-ttest)))
(export 'seesaw-compare)

(labels ((generate-energy-evaluation-method (name mode)
           (make-evaluation-method-descriptor
            name
            (augment-evaluate-state-function-by-ids
             (lambda ()
               (let ((old-energy-score-mode (retrieve-energy-score-mode))
                     score)
                 (set-energy-score-mode mode)
                 (setf score (energy))
                 (set-energy-score-mode old-energy-score-mode)
                 score))
:store-evaluation-score :with-constraints :xomo-scores :ignore-state)
            #'<)))
  (let* ((search-engine-id 'nova.impl.search-engines:seesaw-r)
         (search-engine (lookup-search-engine search-engine-id))
         (search-engine-name (string-downcase (symbol-name search-engine-id)))
         (find-best-state-method-id 'min-score-state-of-path-by-ttest)
         (find-best-state-method (lookup-find-best-state-method find-best-state-method-id)))
    (register-experiment-by-parameters 'evaluation-ss-r
                                       :name "evaluation seesaw-r"
                                       :treatments (list
                                                    (make-experiment-treatment (concatenate 'string "bfc-" search-engine-name)
                                                                               (generate-energy-evaluation-method "bfc" 'bfc)
                                                                               search-engine
                                                                               find-best-state-method)
                                                    (make-experiment-treatment (concatenate 'string "bf-" search-engine-name)
                                                                               (generate-energy-evaluation-method "bf" 'bf)
                                                                               search-engine
                                                                               find-best-state-method)
                                                    (make-experiment-treatment (concatenate 'string "fc-" search-engine-name)
                                                                               (generate-energy-evaluation-method "fc" 'fc)
                                                                               search-engine
                                                                               find-best-state-method)
                                                    (make-experiment-treatment (concatenate 'string "bc-" search-engine-name)
                                                                               (generate-energy-evaluation-method "bc" 'bc)
                                                                               search-engine
                                                                               find-best-state-method)
                                                    (make-experiment-treatment (concatenate 'string "risk-exposure-" search-engine-name)
                                                                               (lookup-evaluation-method 'xomo.evaluation-methods:risk-exposure)
                                                                               search-engine
                                                                               find-best-state-method)))))

(export 'evaluation-ss-r)