(defun GenRuleNew4 (ruleset avoid matcho)
  (let ((counts (list))
	(match (cross-index (table-copy (xindex-table matcho)))))
    (maphash #'(lambda (k v) (setf (gethash k (xindex-counts match)) (counted-n v))) (xindex-counts match))
    (maphash #'(lambda (k v)
		 (if (/= (second k) (xindex-classi match))
		     (push (cons k (/ (* v (/ v (* v .95))) (gethash (list (second k) (third k)) (xindex-uniques match) 0))) counts)))
	     (xindex-counts match))
    (setf counts (stable-sort counts '> :key #'(lambda (x) (cdr x))))
    (if (= (length counts) 0)
	(make-RDRule :fn #'(lambda (x) nil) :class 'Unknown :comment (list 'Bottom) :parent ruleset)
	(progn
	  (setf counts (car (first counts)))
	  (incf *TREESIZE*)
	  (let ((fn (MakePredicate (second counts) (third counts))))
	    (make-RDRule :fn fn :class (first counts) :comment counts :parent ruleset))))))

(defun GenRuleNew3 (ruleset avoid matcho)
  (let ((counts (list))
	(match (cross-index (table-copy (xindex-table matcho)))))
    (maphash #'(lambda (k v) (setf (gethash k (xindex-counts match)) (counted-n v))) (xindex-counts match))
    (maphash #'(lambda (k v)
		 (if (/= (second k) (xindex-classi match))
		     (push (cons k (/ (* v (/ v (* v .9))) (* (sqrt (gethash (list (second k) (third k)) (xindex-uniques match) 0)) (+ (log (gethash (list (second k) (third k)) (xindex-uniques match) 0)) 0.0001)))) counts)))
;		     (push (cons k (/ v (gethash (list (second k) (third k)) (xindex-uniques match) 0))) counts)))
	     (xindex-counts match))
    (setf counts (stable-sort counts '> :key #'(lambda (x) (cdr x))))
    (if (= (length counts) 0)
	(make-RDRule :fn #'(lambda (x) nil) :class 'Unknown :comment (list 'Bottom) :parent ruleset)
	(progn
	  (setf counts (car (first counts)))
	  (incf *TREESIZE*)
	  (let ((fn (MakePredicate (second counts) (third counts))))
	    (make-RDRule :fn fn :class (first counts) :comment counts :parent ruleset))))))

(defun GenRuleNew2 (ruleset avoid matcho)
  (let ((counts (list))
	(match (cross-index (table-copy (xindex-table matcho)))))
    (maphash #'(lambda (k v) (setf (gethash k (xindex-counts match)) (counted-n v))) (xindex-counts match))
    (maphash #'(lambda (k v)
		 (if (/= (second k) (xindex-classi match))
		     (push (cons k (/ (* v (/ v (* v .85))) (sqrt (gethash (list (second k) (third k)) (xindex-uniques match) 0)))) counts)))
;		     (push (cons k v) counts)))
	     (xindex-counts match))
    (setf counts (stable-sort counts '> :key #'(lambda (x) (cdr x))))
    (if (= (length counts) 0)
	(make-RDRule :fn #'(lambda (x) nil) :class 'Unknown :comment (list 'Bottom) :parent ruleset)
	(progn
	  (setf counts (car (first counts)))
	  (incf *TREESIZE*)
	  (let ((fn (MakePredicate (second counts) (third counts))))
	    (make-RDRule :fn fn :class (first counts) :comment counts :parent ruleset))))))

(defun GenRuleNew (ruleset avoid matcho)
  (let ((counts (list))
	(match (cross-index (table-copy (xindex-table matcho)))))
    (maphash #'(lambda (k v) (setf (gethash k (xindex-counts match)) (counted-n v))) (xindex-counts match))
    (maphash #'(lambda (k v)
		 (if (/= (second k) (xindex-classi match))
		     (push (cons k (/ (* v (/ v (* v .8))) (+ (log (gethash (list (second k) (third k)) (xindex-uniques match) 0)) .0001))) counts)))
;		     (push (cons k (/ v (- (xindex-ns match) v))) counts)))
	     (xindex-counts match))
    (setf counts (stable-sort counts '> :key #'(lambda (x) (cdr x))))
    (if (= (length counts) 0)
	(make-RDRule :fn #'(lambda (x) nil) :class 'Unknown :comment (list 'Bottom) :parent ruleset)
	(progn
	  (setf counts (car (first counts)))
	  (incf *TREESIZE*)
	  (let ((fn (MakePredicate (second counts) (third counts))))
	    (make-RDRule :fn fn :class (first counts) :comment counts :parent ruleset))))))

(defun GenRuleBase (ruleset avoid matcho)
  (let ((counts (list))
	(match (cross-index (table-copy (xindex-table matcho)))))
    (maphash #'(lambda (k v) (setf (gethash k (xindex-counts match)) (counted-n v))) (xindex-counts match))
    (maphash #'(lambda (k v)
		 (if (/= (second k) (xindex-classi match))
		     (push (cons k (GetNbProb match (first k) (second k) (third k))) counts)))
	     (xindex-counts match))
    (setf counts (stable-sort counts '> :key #'(lambda (x) (cdr x))))
    (if (= (length counts) 0)
	(make-RDRule :fn #'(lambda (x) nil) :class 'Unknown :comment (list 'Bottom) :parent ruleset)
	(progn
	  (setf counts (car (first counts)))
	  (incf *TREESIZE*)
	  (let ((fn (MakePredicate (second counts) (third counts))))
	    (make-RDRule :fn fn :class (first counts) :comment counts :parent ruleset))))))

(defun GenRuleLwl (ruleset avoid matcho)
  (let ((counts (list))
	(match (cross-index (table-copy (xindex-table matcho)))))

    (maphash #'(lambda (k v) (setf (gethash k (xindex-counts match)) (counted-n v))) (xindex-counts match))

    (let ((limit *LIMIT*)
	  (tmprule ruleset)) ;the slope will be 1/4
      (dotimes (x limit)
	(if (and (not (null (RDRule-parent tmprule))) (eql ruleset (RDRule-true (RDRule-parent tmprule))))
	    (progn
	      (setf tmprule (RDRule-parent tmprule))
	      (setf match (weightedMergeHT match (RDRule-bad tmprule) (/ (- limit x) (+ limit 1))))))))

    (maphash #'(lambda (k v)
		 (if (/= (second k) (xindex-classi match))
		     (push (cons k (GetNbProb match (first k) (second k) (third k))) counts)))
	     (xindex-counts match))
    (setf counts (stable-sort counts '> :key #'(lambda (x) (cdr x))))
    (if (= (length counts) 0)
	(make-RDRule :fn #'(lambda (x) nil) :class 'Unknown :comment (list 'Bottom) :parent ruleset)
	(progn
	  (incf *TREESIZE*)
	  (setf counts (car (first counts)))
	  (make-RDRule :fn (MakePredicate (second counts) (third counts)) :class (first counts) :comment counts :parent ruleset)))))

(defun weightedMergeHT (parent child weight)
  (maphash #'(lambda (k v)
	       (setf (gethash k (xindex-counts parent)) (+ (gethash k (xindex-counts parent) 0) (* weight (counted-n v)))))
	   (xindex-counts child))
  parent)


(defun GetNbProb (cross class col val)
  (let ((likeh (make-hash-table)))
    (maphash #'(lambda (k v)
		 (let (P_C F_XC L_XC)
		   (setf P_C (/ (+ v 0) (* (xindex-ns cross) (hash-table-count (xindex-class-counts cross)))))
		   (setf F_XC (log (/ (+ (gethash (list k col val) (xindex-counts cross) 0) (* P_C 2)) (+ v 0))))
		   (setf L_XC (* F_XC P_C))
		   (setf (gethash k likeh) L_XC)))
	     (xindex-class-counts cross))
    (/ (+ (* (gethash class likeh 0) (gethash class likeh 0)) 1) (+ (sumhash likeh) 2))))

(defun sumhash (ht)
  (let ((total 0))
    (maphash #'(lambda (k v)
		 (setf total (+ total v)))
	     ht)
    total))

(defun LimitChoices (ruleset counts)
  ;(if (and (not (null (RDRule-parent ruleset))) (not (eq ruleset (RDRule-true (RDRule-parent ruleset)))))
   (let ((tmpruleset (copy-RDRule ruleset)))
    (while (and (not (null tmpruleset)) (listp (RDRule-comment tmpruleset)))
	;(print (length counts))
      (if (listp (RDRule-comment tmpruleset))
	  (setf counts (remove-if #'(lambda (x) (and (equal (second (car x)) (second (RDRule-comment ruleset))) (equal (third (car x)) (third (RDRule-comment tmpruleset))))) counts)))
      (setf tmpruleset (RDRule-parent tmpruleset))))
;  (let ((tcounts nil))
;    (mapcar #'(lambda (x)
;		(let ((comment (car x))
;		      (footer (rest x)))
;					;(print comment)
;		  (if (equal nil (assoc (list (second comment) (third comment)) tcounts :test #'equal))
;		      (push (cons (list (second comment) (third comment)) footer) tcounts)
;		      (rplacd (assoc (list (second comment) (third comment)) tcounts :test #'equal) (cons (list (second comment) (third comment)) footer))))) 
;	    counts)
;    (let ((temp nil))
;      (mapcar #'(lambda (x)
;		  (let ((comment (car x))
;			(footer (rest x)))
;					;(print (format nil "~A=~A" comment (assoc (list (second comment) (third comment)) tcounts)))
;		    (if (equal footer (cdr (assoc (list (second comment) (third comment)) tcounts )));:test #'equal)))
;			(push x temp))))
;	      counts)
;      (setf tcounts nil)
;      (mapcar #'(lambda (x)
;		  (let ((comment (car x))
;			(footer (rest x)))
;		    (if (or (not (equal (first comment) (first (RDRule-comment ruleset)))) t)
;			(push x tcounts))))
;	      temp)
;      (setf counts tcounts)))
  counts)