Index: /trunk/CrossPare/src/de/ugoe/cs/cpdp/eval/AbstractWekaEvaluation.java =================================================================== --- /trunk/CrossPare/src/de/ugoe/cs/cpdp/eval/AbstractWekaEvaluation.java (revision 131) +++ /trunk/CrossPare/src/de/ugoe/cs/cpdp/eval/AbstractWekaEvaluation.java (revision 132) @@ -92,4 +92,5 @@ Instances traindata, List trainers, + List efforts, boolean writeHeader, List storages) @@ -150,5 +151,5 @@ eval.numFalsePositives(1) / (eval.numFalsePositives(1) + eval.numTrueNegatives(1)); double gmeasure = 2 * eval.recall(1) * (1.0 - pf) / (eval.recall(1) + (1.0 - pf)); - double aucec = calculateReviewEffort(testdata, classifier); + double aucec = calculateReviewEffort(testdata, classifier, efforts); double succHe = eval.recall(1) >= 0.7 && eval.precision(1) >= 0.5 ? 1.0 : 0.0; double succZi = eval.recall(1) >= 0.7 && eval.precision(1) >= 0.7 ? 1.0 : 0.0; @@ -208,5 +209,94 @@ output.flush(); } - + + private double calculateReviewEffort(Instances testdata, Classifier classifier, List efforts) { + if( efforts==null ) { + return 0; + } + + final List bugPredicted = new ArrayList<>(); + final List nobugPredicted = new ArrayList<>(); + double totalLoc = 0.0d; + int totalBugs = 0; + for (int i = 0; i < testdata.numInstances(); i++) { + try { + if (Double.compare(classifier.classifyInstance(testdata.instance(i)), 0.0d) == 0) { + nobugPredicted.add(i); + } + else { + bugPredicted.add(i); + } + } + catch (Exception e) { + throw new RuntimeException( + "unexpected error during the evaluation of the review effort", + e); + } + if (Double.compare(testdata.instance(i).classValue(), 1.0d) == 0) { + totalBugs++; + } + totalLoc += efforts.get(i); + } + + final List reviewLoc = new ArrayList<>(testdata.numInstances()); + final List bugsFound = new ArrayList<>(testdata.numInstances()); + + double currentBugsFound = 0; + + while (!bugPredicted.isEmpty()) { + double minLoc = Double.MAX_VALUE; + int minIndex = -1; + for (int i = 0; i < bugPredicted.size(); i++) { + double currentLoc = efforts.get(bugPredicted.get(i)); + if (currentLoc < minLoc) { + minIndex = i; + minLoc = currentLoc; + } + } + if (minIndex != -1) { + reviewLoc.add(minLoc / totalLoc); + + currentBugsFound += testdata.instance(bugPredicted.get(minIndex)).classValue(); + bugsFound.add(currentBugsFound); + + bugPredicted.remove(minIndex); + } + else { + throw new RuntimeException("Shouldn't happen!"); + } + } + + while (!nobugPredicted.isEmpty()) { + double minLoc = Double.MAX_VALUE; + int minIndex = -1; + for (int i = 0; i < nobugPredicted.size(); i++) { + double currentLoc = efforts.get(nobugPredicted.get(i)); + if (currentLoc < minLoc) { + minIndex = i; + minLoc = currentLoc; + } + } + if (minIndex != -1) { + reviewLoc.add(minLoc / totalLoc); + + currentBugsFound += testdata.instance(nobugPredicted.get(minIndex)).classValue(); + bugsFound.add(currentBugsFound); + nobugPredicted.remove(minIndex); + } + else { + throw new RuntimeException("Shouldn't happen!"); + } + } + + double auc = 0.0; + for (int i = 0; i < bugsFound.size(); i++) { + auc += reviewLoc.get(i) * bugsFound.get(i) / totalBugs; + } + + return auc; + } + + @SuppressWarnings("unused") + @Deprecated private double calculateReviewEffort(Instances testdata, Classifier classifier) { Index: /trunk/CrossPare/src/de/ugoe/cs/cpdp/eval/IEvaluationStrategy.java =================================================================== --- /trunk/CrossPare/src/de/ugoe/cs/cpdp/eval/IEvaluationStrategy.java (revision 131) +++ /trunk/CrossPare/src/de/ugoe/cs/cpdp/eval/IEvaluationStrategy.java (revision 132) @@ -46,4 +46,5 @@ Instances traindata, List trainers, + List efforts, boolean writeHeader, List storages); Index: /trunk/CrossPare/src/de/ugoe/cs/cpdp/execution/AbstractCrossProjectExperiment.java =================================================================== --- /trunk/CrossPare/src/de/ugoe/cs/cpdp/execution/AbstractCrossProjectExperiment.java (revision 131) +++ /trunk/CrossPare/src/de/ugoe/cs/cpdp/execution/AbstractCrossProjectExperiment.java (revision 132) @@ -178,4 +178,5 @@ // Setup testdata and training data Instances testdata = testVersion.getInstances(); + List efforts = testVersion.getEfforts(); SetUniqueList traindataSet = SetUniqueList.setUniqueList(new LinkedList()); @@ -311,5 +312,5 @@ config.getExperimentName() + ".csv"); } - evaluator.apply(testdata, traindata, allTrainers, writeHeader, + evaluator.apply(testdata, traindata, allTrainers, efforts, writeHeader, config.getResultStorages()); writeHeader = false; Index: /trunk/CrossPare/src/de/ugoe/cs/cpdp/execution/ClassifierCreationExperiment.java =================================================================== --- /trunk/CrossPare/src/de/ugoe/cs/cpdp/execution/ClassifierCreationExperiment.java (revision 131) +++ /trunk/CrossPare/src/de/ugoe/cs/cpdp/execution/ClassifierCreationExperiment.java (revision 132) @@ -101,4 +101,5 @@ Instances testdata = testVersion.getInstances(); Instances traindata = new Instances(testdata); + List efforts = testVersion.getEfforts(); // Give the dataset a new name @@ -168,5 +169,5 @@ config.getExperimentName() + ".csv"); } - evaluator.apply(testdata, traindata, allTrainers, writeHeader, config.getResultStorages()); + evaluator.apply(testdata, traindata, allTrainers, efforts, writeHeader, config.getResultStorages()); writeHeader = false; } Index: /trunk/CrossPare/src/de/ugoe/cs/cpdp/execution/CrossValidationExperiment.java =================================================================== --- /trunk/CrossPare/src/de/ugoe/cs/cpdp/execution/CrossValidationExperiment.java (revision 131) +++ /trunk/CrossPare/src/de/ugoe/cs/cpdp/execution/CrossValidationExperiment.java (revision 132) @@ -166,4 +166,5 @@ // Setup testdata and training data Instances testdata = testVersion.getInstances(); + List efforts = testVersion.getEfforts(); for (ITrainingStrategy trainer : config.getTrainers()) { @@ -205,5 +206,5 @@ config.getExperimentName() + ".csv"); } - evaluator.apply(testdata, testdata, allTrainers, writeHeader, + evaluator.apply(testdata, testdata, allTrainers, efforts, writeHeader, config.getResultStorages()); writeHeader = false; Index: /trunk/CrossPare/src/de/ugoe/cs/cpdp/loader/AbstractFolderLoader.java =================================================================== --- /trunk/CrossPare/src/de/ugoe/cs/cpdp/loader/AbstractFolderLoader.java (revision 131) +++ /trunk/CrossPare/src/de/ugoe/cs/cpdp/loader/AbstractFolderLoader.java (revision 132) @@ -16,7 +16,9 @@ import java.io.File; +import java.util.ArrayList; import java.util.LinkedList; import java.util.List; +import weka.core.Attribute; import weka.core.Instances; @@ -66,5 +68,6 @@ Instances data = instancesLoader.load(versionFile); String versionName = data.relationName(); - versions.add(new SoftwareVersion(projectName, versionName, data)); + List efforts = getEfforts(data); + versions.add(new SoftwareVersion(projectName, versionName, data, efforts)); } } @@ -73,4 +76,29 @@ } return versions; + } + + private List getEfforts(Instances data) { + // attribute in the JURECZKO data and default + Attribute effortAtt = data.attribute("loc"); + if (effortAtt == null) { + // attribute in the NASA/SOFTMINE/MDP data + effortAtt = data.attribute("LOC_EXECUTABLE"); + } + if (effortAtt == null) { + // attribute in the AEEEM data + effortAtt = data.attribute("numberOfLinesOfCode"); + } + if (effortAtt == null) { + // attribute in the RELINK data + effortAtt = data.attribute("CountLineCodeExe"); + } + if( effortAtt == null ) { + return null; + } + List efforts = new ArrayList<>(data.size()); + for( int i=0; i efforts; /** @@ -49,10 +56,11 @@ * data of the version */ - public SoftwareVersion(String project, String version, Instances instances) { + public SoftwareVersion(String project, String version, Instances instances, List efforts) { this.project = project; this.version = version; this.instances = instances; + this.efforts = efforts; } - + /** * returns the project name @@ -81,4 +89,15 @@ return new Instances(instances); } + + /** + *

+ * returns the review effort of the version + *

+ * + * @return + */ + public List getEfforts() { + return efforts; + } /**