package de.ugoe.cs.cpdp.eval;

import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.PrintWriter;
import java.util.ArrayList;
import java.util.LinkedList;
import java.util.List;

import de.ugoe.cs.cpdp.training.ITrainer;
import de.ugoe.cs.cpdp.training.IWekaCompatibleTrainer;
import de.ugoe.cs.util.StringTools;
import weka.classifiers.Classifier;
import weka.classifiers.Evaluation;
import weka.core.Attribute;
import weka.core.Instances;

/**
 * Base class for the evaluation of results of classifiers compatible with the {@link Classifier} interface.
 * For each classifier, the following metrics are calculated:
 * <ul>
 *  <li>Success with recall>0.7, precision>0.5</li>
 *  <li>Success with recall>0.7, precision>0.7</li>
 *  <li>Success with gscore>0.75</li>
 *  <li>Success with gscore>0.6</li>
 *  <li>error rate</li>
 *  <li>recall</li>
 *  <li>precision</li>
 *  <li>fscore</li>
 *  <li>gscore</li>
 *  <li>AUC</li>
 *  <li>AUCEC (weighted by LOC, if applicable; 0.0 if LOC not available)</li>
 *  <li>true positive rate</li>
 *  <li>true negative rate</li>
 *  <li>true positives</li>
 *  <li>false positives</li>
 *  <li>true negatives</li>
 *  <li>false negatives</li>
 *  <li>training error</li>
 *  <li>training recall</li>
 *  <li>training precision</li>
 *  <li>training success with recall>0.7 and precision>0.5
 * </ul> 
 * @author Steffen Herbold
 */
public abstract class AbstractWekaEvaluation implements IEvaluationStrategy {

	/**
	 * writer for the evaluation results
	 */
	private PrintWriter output = new PrintWriter(System.out);
	
	private boolean outputIsSystemOut = true;
	
	/**
	 * Creates the weka evaluator. Allows the creation of the evaluator in different ways, e.g., for cross-validation
	 * or evaluation on the test data.
	 * @param testdata test data
	 * @param classifier classifier used
	 * @return evaluator
	 */
	protected abstract Evaluation createEvaluator(Instances testdata, Classifier classifier);
	
	/*
	 * (non-Javadoc)
	 * @see de.ugoe.cs.cpdp.eval.EvaluationStrategy#apply(weka.core.Instances, weka.core.Instances, java.util.List, boolean)
	 */
	@Override
	public void apply(Instances testdata, Instances traindata, List<ITrainer> trainers,
			boolean writeHeader) {
		final List<Classifier> classifiers = new LinkedList<Classifier>();
		for( ITrainer trainer : trainers ) {
			if( trainer instanceof IWekaCompatibleTrainer ) {
				classifiers.add(((IWekaCompatibleTrainer) trainer).getClassifier());
			} else {
				throw new RuntimeException("The selected evaluator only support Weka classifiers");
			}
		}
		
		if( writeHeader ) {
			output.append("version,size_test,size_training");
			for( ITrainer trainer : trainers ) {
				output.append(",succHe_" + ((IWekaCompatibleTrainer) trainer).getName());
				output.append(",succZi_" + ((IWekaCompatibleTrainer) trainer).getName());
				output.append(",succG75_" + ((IWekaCompatibleTrainer) trainer).getName());
				output.append(",succG60_" + ((IWekaCompatibleTrainer) trainer).getName());
				output.append(",error_" + ((IWekaCompatibleTrainer) trainer).getName());
				output.append(",recall_" + ((IWekaCompatibleTrainer) trainer).getName());
				output.append(",precision_" + ((IWekaCompatibleTrainer) trainer).getName());
				output.append(",fscore_" + ((IWekaCompatibleTrainer) trainer).getName());
				output.append(",gscore_" + ((IWekaCompatibleTrainer) trainer).getName());
				output.append(",mcc_" + ((IWekaCompatibleTrainer) trainer).getName());
				output.append(",auc_" + ((IWekaCompatibleTrainer) trainer).getName());
				output.append(",aucec_" + ((IWekaCompatibleTrainer) trainer).getName());
				output.append(",tpr_" + ((IWekaCompatibleTrainer) trainer).getName());
				output.append(",tnr_" + ((IWekaCompatibleTrainer) trainer).getName());
				output.append(",tp_" + ((IWekaCompatibleTrainer) trainer).getName());
				output.append(",fn_" + ((IWekaCompatibleTrainer) trainer).getName());
				output.append(",tn_" + ((IWekaCompatibleTrainer) trainer).getName());
				output.append(",fp_" + ((IWekaCompatibleTrainer) trainer).getName());
				output.append(",trainerror_" + ((IWekaCompatibleTrainer) trainer).getName());
				output.append(",trainrecall_" + ((IWekaCompatibleTrainer) trainer).getName());
				output.append(",trainprecision_" + ((IWekaCompatibleTrainer) trainer).getName());
				output.append(",trainsuccHe_" + ((IWekaCompatibleTrainer) trainer).getName());
			}
			output.append(StringTools.ENDLINE);
		}
		
		output.append(testdata.relationName());
		output.append("," + testdata.numInstances());	
		output.append("," + traindata.numInstances());
		
		Evaluation eval = null;
		Evaluation evalTrain = null;
		for( Classifier classifier : classifiers ) {
			eval = createEvaluator(testdata, classifier);
			evalTrain = createEvaluator(traindata, classifier);
			
			double pf = eval.numFalsePositives(1)/(eval.numFalsePositives(1)+eval.numTrueNegatives(1));
			double gmeasure = 2*eval.recall(1)*(1.0-pf)/(eval.recall(1)+(1.0-pf));
			double mcc = (eval.numTruePositives(1)*eval.numTrueNegatives(1)-eval.numFalsePositives(1)*eval.numFalseNegatives(1))/Math.sqrt((eval.numTruePositives(1)+eval.numFalsePositives(1))*(eval.numTruePositives(1)+eval.numFalseNegatives(1))*(eval.numTrueNegatives(1)+eval.numFalsePositives(1))*(eval.numTrueNegatives(1)+eval.numFalseNegatives(1)));
			double aucec = calculateReviewEffort(testdata, classifier);
			
			if( eval.recall(1)>=0.7 && eval.precision(1) >= 0.5 ) {
				output.append(",1");
			} else {
				output.append(",0");
			}
			
			if( eval.recall(1)>=0.7 && eval.precision(1) >= 0.7 ) {
				output.append(",1");
			} else {
				output.append(",0");
			}
			
			if( gmeasure>0.75 ) {
				output.append(",1");
			} else {
				output.append(",0");
			}
			
			if( gmeasure>0.6 ) {
				output.append(",1");
			} else {
				output.append(",0");
			}
			
			output.append("," + eval.errorRate());
			output.append("," + eval.recall(1));
			output.append("," + eval.precision(1));
			output.append("," + eval.fMeasure(1));
			output.append("," + gmeasure);
			output.append("," + mcc);
			output.append("," + eval.areaUnderROC(1));
			output.append("," + aucec);
			output.append("," + eval.truePositiveRate(1));
			output.append("," + eval.trueNegativeRate(1));
			output.append("," + eval.numTruePositives(1));
			output.append("," + eval.numFalseNegatives(1));
			output.append("," + eval.numTrueNegatives(1));
			output.append("," + eval.numFalsePositives(1));
			output.append("," + evalTrain.errorRate());
			output.append("," + evalTrain.recall(1));
			output.append("," + evalTrain.precision(1));
			if( evalTrain.recall(1)>=0.7 && evalTrain.precision(1) >= 0.5 ) {
				output.append(",1");
			} else {
				output.append(",0");
			}
		}
		
		output.append(StringTools.ENDLINE);
		output.flush();
	}
	
	private double calculateReviewEffort(Instances testdata, Classifier classifier) {
		
		final Attribute loc = testdata.attribute("loc");
		if( loc==null ) {
			return 0.0;
		}
				
		final List<Integer> bugPredicted = new ArrayList<>();
		final List<Integer> nobugPredicted = new ArrayList<>(); 
		double totalLoc = 0.0d;
		int totalBugs = 0;
		for( int i=0 ; i<testdata.numInstances() ; i++ ) {
			try {
				if( Double.compare(classifier.classifyInstance(testdata.instance(i)),0.0d)==0 ) {
					nobugPredicted.add(i);
				} else {
					bugPredicted.add(i);
				}
			} catch (Exception e) {
				throw new RuntimeException("unexpected error during the evaluation of the review effort", e);
			}
			if(Double.compare(testdata.instance(i).classValue(),1.0d)==0) {
				totalBugs++;
			}
			totalLoc += testdata.instance(i).value(loc);
		}
		
		final List<Double> reviewLoc = new ArrayList<>(testdata.numInstances());
		final List<Double> bugsFound = new ArrayList<>(testdata.numInstances());
		
		double currentBugsFound = 0;
		
		while( !bugPredicted.isEmpty() ) {
			double minLoc = Double.MAX_VALUE;
			int minIndex = -1;
			for( int i=0 ; i<bugPredicted.size() ; i++ ) {
				double currentLoc = testdata.instance(bugPredicted.get(i)).value(loc);
				if( currentLoc<minLoc ) {
					minIndex = i;
					minLoc = currentLoc;
				}
			}
			if( minIndex!=-1 ) {
				reviewLoc.add(minLoc/totalLoc);
				
				currentBugsFound += testdata.instance(bugPredicted.get(minIndex)).classValue();
				bugsFound.add(currentBugsFound);
				
				bugPredicted.remove(minIndex);
			} else {
				throw new RuntimeException("Shouldn't happen!");
			}
		}
		
		while( !nobugPredicted.isEmpty() ) {
			double minLoc = Double.MAX_VALUE;
			int minIndex = -1;
			for( int i=0 ; i<nobugPredicted.size() ; i++ ) {
				double currentLoc = testdata.instance(nobugPredicted.get(i)).value(loc);
				if( currentLoc<minLoc ) {
					minIndex = i;
					minLoc = currentLoc;
				}
			}
			if( minIndex!=-1 ) {				
				reviewLoc.add(minLoc/totalLoc);
				
				currentBugsFound += testdata.instance(nobugPredicted.get(minIndex)).classValue();
				bugsFound.add(currentBugsFound);
				nobugPredicted.remove(minIndex);
			} else {
				throw new RuntimeException("Shouldn't happen!");
			}
		}
		
		double auc = 0.0;
		for( int i=0 ; i<bugsFound.size() ; i++ ) {
			auc += reviewLoc.get(i)*bugsFound.get(i)/totalBugs;
		}
		
		return auc;
	}

	/*
	 * (non-Javadoc)
	 * @see de.ugoe.cs.cpdp.Parameterizable#setParameter(java.lang.String)
	 */
	@Override
	public void setParameter(String parameters) {
		if( output!=null && !outputIsSystemOut ) {
			output.close();
		}
		if( "system.out".equals(parameters) || "".equals(parameters) ) {
			output = new PrintWriter(System.out);
			outputIsSystemOut = true;
		} else {
			try {
				output = new PrintWriter(new FileOutputStream(parameters));
				outputIsSystemOut = false;
			} catch (FileNotFoundException e) {
				throw new RuntimeException(e);
			}
		}
	}
}