Changeset 132 for trunk

Timestamp:

06/28/16 12:01:39 (8 years ago)

Author:

sherbold

Message:

• rather intrusive and large change to correctly evaluate AUCEC in case metrics are modified. The effort is now stored directly with a software version and it is the duty of the loader to specify the review effort for each instance. This required changes to the execution strategiey, data loading, and evaluation process.

Location:

trunk/CrossPare/src/de/ugoe/cs/cpdp

Files:

• eval/AbstractWekaEvaluation.java (modified) (3 diffs)
• eval/IEvaluationStrategy.java (modified) (1 diff)
• execution/AbstractCrossProjectExperiment.java (modified) (2 diffs)
• execution/ClassifierCreationExperiment.java (modified) (2 diffs)
• execution/CrossValidationExperiment.java (modified) (2 diffs)
• loader/AbstractFolderLoader.java (modified) (3 diffs)
• loader/DecentFolderLoader.java (modified) (1 diff)
• versions/SoftwareVersion.java (modified) (4 diffs)

trunk/CrossPare/src/de/ugoe/cs/cpdp/eval/AbstractWekaEvaluation.java

@@ -92 +92 @@
                       Instances traindata,
                       List<ITrainer> trainers,
+                      List<Double> efforts, 
                       boolean writeHeader,
                       List<IResultStorage> storages)
@@ -150 +151 @@
                 eval.numFalsePositives(1) / (eval.numFalsePositives(1) + eval.numTrueNegatives(1));
             double gmeasure = 2 * eval.recall(1) * (1.0 - pf) / (eval.recall(1) + (1.0 - pf));
-            double aucec = calculateReviewEffort(testdata, classifier);
+            double aucec = calculateReviewEffort(testdata, classifier, efforts);
             double succHe = eval.recall(1) >= 0.7 && eval.precision(1) >= 0.5 ? 1.0 : 0.0;
             double succZi = eval.recall(1) >= 0.7 && eval.precision(1) >= 0.7 ? 1.0 : 0.0;
@@ -208 +209 @@
         output.flush();
     }
-
+    
+    private double calculateReviewEffort(Instances testdata, Classifier classifier, List<Double> efforts) {
+        if( efforts==null ) {
+            return 0;
+        }
+        
+        final List<Integer> bugPredicted = new ArrayList<>();
+        final List<Integer> nobugPredicted = new ArrayList<>();
+        double totalLoc = 0.0d;
+        int totalBugs = 0;
+        for (int i = 0; i < testdata.numInstances(); i++) {
+            try {
+                if (Double.compare(classifier.classifyInstance(testdata.instance(i)), 0.0d) == 0) {
+                    nobugPredicted.add(i);
+                }
+                else {
+                    bugPredicted.add(i);
+                }
+            }
+            catch (Exception e) {
+                throw new RuntimeException(
+                                           "unexpected error during the evaluation of the review effort",
+                                           e);
+            }
+            if (Double.compare(testdata.instance(i).classValue(), 1.0d) == 0) {
+                totalBugs++;
+            }
+            totalLoc += efforts.get(i);
+        }
+
+        final List<Double> reviewLoc = new ArrayList<>(testdata.numInstances());
+        final List<Double> bugsFound = new ArrayList<>(testdata.numInstances());
+
+        double currentBugsFound = 0;
+
+        while (!bugPredicted.isEmpty()) {
+            double minLoc = Double.MAX_VALUE;
+            int minIndex = -1;
+            for (int i = 0; i < bugPredicted.size(); i++) {
+                double currentLoc = efforts.get(bugPredicted.get(i));
+                if (currentLoc < minLoc) {
+                    minIndex = i;
+                    minLoc = currentLoc;
+                }
+            }
+            if (minIndex != -1) {
+                reviewLoc.add(minLoc / totalLoc);
+
+                currentBugsFound += testdata.instance(bugPredicted.get(minIndex)).classValue();
+                bugsFound.add(currentBugsFound);
+
+                bugPredicted.remove(minIndex);
+            }
+            else {
+                throw new RuntimeException("Shouldn't happen!");
+            }
+        }
+
+        while (!nobugPredicted.isEmpty()) {
+            double minLoc = Double.MAX_VALUE;
+            int minIndex = -1;
+            for (int i = 0; i < nobugPredicted.size(); i++) {
+                double currentLoc = efforts.get(nobugPredicted.get(i));
+                if (currentLoc < minLoc) {
+                    minIndex = i;
+                    minLoc = currentLoc;
+                }
+            }
+            if (minIndex != -1) {
+                reviewLoc.add(minLoc / totalLoc);
+
+                currentBugsFound += testdata.instance(nobugPredicted.get(minIndex)).classValue();
+                bugsFound.add(currentBugsFound);
+                nobugPredicted.remove(minIndex);
+            }
+            else {
+                throw new RuntimeException("Shouldn't happen!");
+            }
+        }
+
+        double auc = 0.0;
+        for (int i = 0; i < bugsFound.size(); i++) {
+            auc += reviewLoc.get(i) * bugsFound.get(i) / totalBugs;
+        }
+
+        return auc;
+    }
+
+    @SuppressWarnings("unused")
+    @Deprecated
     private double calculateReviewEffort(Instances testdata, Classifier classifier) {
 

trunk/CrossPare/src/de/ugoe/cs/cpdp/eval/IEvaluationStrategy.java

@@ -46 +46 @@
                Instances traindata,
                List<ITrainer> trainers,
+               List<Double> efforts,
                boolean writeHeader,
                List<IResultStorage> storages);

trunk/CrossPare/src/de/ugoe/cs/cpdp/execution/AbstractCrossProjectExperiment.java

@@ -178 +178 @@
                 // Setup testdata and training data
                 Instances testdata = testVersion.getInstances();
+                List<Double> efforts = testVersion.getEfforts();
                 SetUniqueList<Instances> traindataSet =
                     SetUniqueList.setUniqueList(new LinkedList<Instances>());
@@ -311 +312 @@
                             config.getExperimentName() + ".csv");
                     }
-                    evaluator.apply(testdata, traindata, allTrainers, writeHeader,
+                    evaluator.apply(testdata, traindata, allTrainers, efforts, writeHeader,
                                     config.getResultStorages());
                     writeHeader = false;

trunk/CrossPare/src/de/ugoe/cs/cpdp/execution/ClassifierCreationExperiment.java

@@ -101 +101 @@
             Instances testdata = testVersion.getInstances();
             Instances traindata = new Instances(testdata);
+            List<Double> efforts = testVersion.getEfforts();
 
             // Give the dataset a new name
@@ -168 +169 @@
                         config.getExperimentName() + ".csv");
                 }
-                evaluator.apply(testdata, traindata, allTrainers, writeHeader, config.getResultStorages());
+                evaluator.apply(testdata, traindata, allTrainers, efforts, writeHeader, config.getResultStorages());
                 writeHeader = false;
             }

trunk/CrossPare/src/de/ugoe/cs/cpdp/execution/CrossValidationExperiment.java

@@ -166 +166 @@
                 // Setup testdata and training data
                 Instances testdata = testVersion.getInstances();
+                List<Double> efforts = testVersion.getEfforts();
                 
                 for (ITrainingStrategy trainer : config.getTrainers()) {
@@ -205 +206 @@
                             config.getExperimentName() + ".csv");
                     }
-                    evaluator.apply(testdata, testdata, allTrainers, writeHeader,
+                    evaluator.apply(testdata, testdata, allTrainers, efforts, writeHeader,
                                     config.getResultStorages());
                     writeHeader = false;

trunk/CrossPare/src/de/ugoe/cs/cpdp/loader/AbstractFolderLoader.java

@@ -16 +16 @@
 
 import java.io.File;
+import java.util.ArrayList;
 import java.util.LinkedList;
 import java.util.List;
 
+import weka.core.Attribute;
 import weka.core.Instances;
 
@@ -66 +68 @@
                             Instances data = instancesLoader.load(versionFile);
                             String versionName = data.relationName();
-                            versions.add(new SoftwareVersion(projectName, versionName, data));
+                            List<Double> efforts = getEfforts(data); 
+                            versions.add(new SoftwareVersion(projectName, versionName, data, efforts));
                         }
                     }
@@ -73 +76 @@
         }
         return versions;
+    }
+    
+    private List<Double> getEfforts(Instances data) {
+        // attribute in the JURECZKO data and default
+        Attribute effortAtt = data.attribute("loc");
+        if (effortAtt == null) {
+            // attribute in the NASA/SOFTMINE/MDP data
+            effortAtt = data.attribute("LOC_EXECUTABLE");
+        }
+        if (effortAtt == null) {
+            // attribute in the AEEEM data
+            effortAtt = data.attribute("numberOfLinesOfCode");
+        }
+        if (effortAtt == null) {
+            // attribute in the RELINK data
+            effortAtt = data.attribute("CountLineCodeExe");
+        }
+        if( effortAtt == null ) {
+            return null;
+        }
+        List<Double> efforts = new ArrayList<>(data.size());
+        for( int i=0; i<data.size(); i++ ) {
+            efforts.add(data.get(i).value(effortAtt));
+        }
+        return efforts;
     }
 

trunk/CrossPare/src/de/ugoe/cs/cpdp/loader/DecentFolderLoader.java

@@ -104 +104 @@
             String versionName = versionFile.getName();
             Instances data = instancesLoader.load(versionFile);
-            versions.add(new SoftwareVersion(projectName, versionName, data));
+            versions.add(new SoftwareVersion(projectName, versionName, data, null));
         }
     }

trunk/CrossPare/src/de/ugoe/cs/cpdp/versions/SoftwareVersion.java

@@ -14 +14 @@
 
 package de.ugoe.cs.cpdp.versions;
+
+import java.util.List;
 
 import weka.core.Instances;
@@ -38 +40 @@
      */
     private final Instances instances;
+    
+    /**
+     * Review effort per instance. 
+     */
+    private final List<Double> efforts;
 
     /**
@@ -49 +56 @@
      *            data of the version
      */
-    public SoftwareVersion(String project, String version, Instances instances) {
+    public SoftwareVersion(String project, String version, Instances instances, List<Double> efforts) {
         this.project = project;
         this.version = version;
         this.instances = instances;
+        this.efforts = efforts;
     }
-
+    
     /**
      * returns the project name
@@ -81 +89 @@
         return new Instances(instances);
     }
+    
+    /**
+     * <p>
+     * returns the review effort of the version
+     * </p>
+     *
+     * @return
+     */
+    public List<Double> getEfforts() {
+        return efforts;
+    }
 
     /**