source: trunk/CrossPare/src/de/ugoe/cs/cpdp/eval/AbstractWekaEvaluation.java @ 132

Last change on this file since 132 was 132, checked in by sherbold, 8 years ago
  • rather intrusive and large change to correctly evaluate AUCEC in case metrics are modified. The effort is now stored directly with a software version and it is the duty of the loader to specify the review effort for each instance. This required changes to the execution strategiey, data loading, and evaluation process.
  • Property svn:mime-type set to text/plain
File size: 17.2 KB
Line 
1// Copyright 2015 Georg-August-Universität Göttingen, Germany
2//
3//   Licensed under the Apache License, Version 2.0 (the "License");
4//   you may not use this file except in compliance with the License.
5//   You may obtain a copy of the License at
6//
7//       http://www.apache.org/licenses/LICENSE-2.0
8//
9//   Unless required by applicable law or agreed to in writing, software
10//   distributed under the License is distributed on an "AS IS" BASIS,
11//   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12//   See the License for the specific language governing permissions and
13//   limitations under the License.
14
15package de.ugoe.cs.cpdp.eval;
16
17import java.io.FileNotFoundException;
18import java.io.FileOutputStream;
19import java.io.PrintWriter;
20import java.util.ArrayList;
21import java.util.Iterator;
22import java.util.LinkedList;
23import java.util.List;
24
25import de.ugoe.cs.cpdp.training.ITrainer;
26import de.ugoe.cs.cpdp.training.IWekaCompatibleTrainer;
27import de.ugoe.cs.util.StringTools;
28import weka.classifiers.Classifier;
29import weka.classifiers.Evaluation;
30import weka.core.Attribute;
31import weka.core.Instances;
32
33/**
34 * Base class for the evaluation of results of classifiers compatible with the {@link Classifier}
35 * interface. For each classifier, the following metrics are calculated:
36 * <ul>
37 * <li>succHe: Success with recall>0.7, precision>0.5</li>
38 * <li>succZi: Success with recall>0.7, precision>0.7</li>
39 * <li>succG75: Success with gscore>0.75</li>
40 * <li>succG60: Success with gscore>0.6</li>
41 * <li>error</li>
42 * <li>recall</li>
43 * <li>precision</li>
44 * <li>fscore</li>
45 * <li>gscore</li>
46 * <li>MCC</li>
47 * <li>AUC</li>
48 * <li>AUCEC (weighted by LOC, if applicable; 0.0 if LOC not available)</li>
49 * <li>tpr: true positive rate</li>
50 * <li>tnr: true negative rate</li>
51 * <li>fpr: false positive rate</li>
52 * <li>fnr: false negative rate</li>
53 * <li>tp: true positives</li>
54 * <li>fp: false positives</li>
55 * <li>tn: true negatives</li>
56 * <li>fn: false negatives</li>
57 * </ul>
58 *
59 * @author Steffen Herbold
60 */
61public abstract class AbstractWekaEvaluation implements IEvaluationStrategy {
62
63    /**
64     * writer for the evaluation results
65     */
66    private PrintWriter output = new PrintWriter(System.out);
67
68    private boolean outputIsSystemOut = true;
69   
70    private String configurationName = "default";
71
72    /**
73     * Creates the Weka evaluator. Allows the creation of the evaluator in different ways, e.g., for
74     * cross-validation or evaluation on the test data.
75     *
76     * @param testdata
77     *            test data
78     * @param classifier
79     *            classifier used
80     * @return evaluator
81     */
82    protected abstract Evaluation createEvaluator(Instances testdata, Classifier classifier);
83
84    /*
85     * (non-Javadoc)
86     *
87     * @see de.ugoe.cs.cpdp.eval.EvaluationStrategy#apply(weka.core.Instances, weka.core.Instances,
88     * java.util.List, boolean)
89     */
90    @Override
91    public void apply(Instances testdata,
92                      Instances traindata,
93                      List<ITrainer> trainers,
94                      List<Double> efforts,
95                      boolean writeHeader,
96                      List<IResultStorage> storages)
97    {
98        final List<Classifier> classifiers = new LinkedList<>();
99        final List<ExperimentResult> experimentResults = new LinkedList<>();
100        String productName = testdata.relationName();
101       
102        for (ITrainer trainer : trainers) {
103            if (trainer instanceof IWekaCompatibleTrainer) {
104                classifiers.add(((IWekaCompatibleTrainer) trainer).getClassifier());
105                experimentResults.add(new ExperimentResult(configurationName, productName, ((IWekaCompatibleTrainer) trainer).getName()));
106            }
107            else {
108                throw new RuntimeException("The selected evaluator only support Weka classifiers");
109            }
110        }
111
112        if (writeHeader) {
113            output.append("version,size_test,size_training");
114            for (ITrainer trainer : trainers) {
115                output.append(",succHe_" + ((IWekaCompatibleTrainer) trainer).getName());
116                output.append(",succZi_" + ((IWekaCompatibleTrainer) trainer).getName());
117                output.append(",succG75_" + ((IWekaCompatibleTrainer) trainer).getName());
118                output.append(",succG60_" + ((IWekaCompatibleTrainer) trainer).getName());
119                output.append(",error_" + ((IWekaCompatibleTrainer) trainer).getName());
120                output.append(",recall_" + ((IWekaCompatibleTrainer) trainer).getName());
121                output.append(",precision_" + ((IWekaCompatibleTrainer) trainer).getName());
122                output.append(",fscore_" + ((IWekaCompatibleTrainer) trainer).getName());
123                output.append(",gscore_" + ((IWekaCompatibleTrainer) trainer).getName());
124                output.append(",mcc_" + ((IWekaCompatibleTrainer) trainer).getName());
125                output.append(",auc_" + ((IWekaCompatibleTrainer) trainer).getName());
126                output.append(",aucec_" + ((IWekaCompatibleTrainer) trainer).getName());
127                output.append(",tpr_" + ((IWekaCompatibleTrainer) trainer).getName());
128                output.append(",tnr_" + ((IWekaCompatibleTrainer) trainer).getName());
129                output.append(",fpr_" + ((IWekaCompatibleTrainer) trainer).getName());
130                output.append(",fnr_" + ((IWekaCompatibleTrainer) trainer).getName());
131                output.append(",tp_" + ((IWekaCompatibleTrainer) trainer).getName());
132                output.append(",fn_" + ((IWekaCompatibleTrainer) trainer).getName());
133                output.append(",tn_" + ((IWekaCompatibleTrainer) trainer).getName());
134                output.append(",fp_" + ((IWekaCompatibleTrainer) trainer).getName());
135            }
136            output.append(StringTools.ENDLINE);
137        }
138
139        output.append(productName);
140        output.append("," + testdata.numInstances());
141        output.append("," + traindata.numInstances());
142
143        Evaluation eval = null;
144        Iterator<Classifier> classifierIter = classifiers.iterator();
145        Iterator<ExperimentResult> resultIter = experimentResults.iterator();
146        while (classifierIter.hasNext()) {
147            Classifier classifier = classifierIter.next();
148            eval = createEvaluator(testdata, classifier);
149
150            double pf =
151                eval.numFalsePositives(1) / (eval.numFalsePositives(1) + eval.numTrueNegatives(1));
152            double gmeasure = 2 * eval.recall(1) * (1.0 - pf) / (eval.recall(1) + (1.0 - pf));
153            double aucec = calculateReviewEffort(testdata, classifier, efforts);
154            double succHe = eval.recall(1) >= 0.7 && eval.precision(1) >= 0.5 ? 1.0 : 0.0;
155            double succZi = eval.recall(1) >= 0.7 && eval.precision(1) >= 0.7 ? 1.0 : 0.0;
156            double succG75 = gmeasure > 0.75 ? 1.0 : 0.0;
157            double succG60 = gmeasure > 0.6 ? 1.0 : 0.0;
158           
159            output.append("," + succHe);
160            output.append("," + succZi);
161            output.append("," + succG75);
162            output.append("," + succG60);           
163            output.append("," + eval.errorRate());
164            output.append("," + eval.recall(1));
165            output.append("," + eval.precision(1));
166            output.append("," + eval.fMeasure(1));
167            output.append("," + gmeasure);
168            output.append("," + eval.matthewsCorrelationCoefficient(1));
169            output.append("," + eval.areaUnderROC(1));
170            output.append("," + aucec);
171            output.append("," + eval.truePositiveRate(1));
172            output.append("," + eval.trueNegativeRate(1));
173            output.append("," + eval.falsePositiveRate(1));
174            output.append("," + eval.falseNegativeRate(1));
175            output.append("," + eval.numTruePositives(1));
176            output.append("," + eval.numFalseNegatives(1));
177            output.append("," + eval.numTrueNegatives(1));
178            output.append("," + eval.numFalsePositives(1));
179           
180            ExperimentResult result = resultIter.next();
181            result.setSizeTestData(testdata.numInstances());
182            result.setSizeTrainingData(traindata.numInstances());
183            result.setSuccHe(succHe);
184            result.setSuccZi(succZi);
185            result.setSuccG75(succG75);
186            result.setSuccG60(succG60);
187            result.setError(eval.errorRate());
188            result.setRecall(eval.recall(1));
189            result.setPrecision(eval.precision(1));
190            result.setFscore(eval.fMeasure(1));
191            result.setGscore(gmeasure);
192            result.setMcc(eval.matthewsCorrelationCoefficient(1));
193            result.setAuc(eval.areaUnderROC(1));
194            result.setAucec(aucec);
195            result.setTpr(eval.truePositiveRate(1));
196            result.setTnr(eval.trueNegativeRate(1));
197            result.setFpr(eval.falsePositiveRate(1));
198            result.setFnr(eval.falseNegativeRate(1));
199            result.setTp(eval.numTruePositives(1));
200            result.setFn(eval.numFalseNegatives(1));
201            result.setTn(eval.numTrueNegatives(1));
202            result.setFp(eval.numFalsePositives(1));
203            for( IResultStorage storage : storages ) {
204                storage.addResult(result);
205            }
206        }
207
208        output.append(StringTools.ENDLINE);
209        output.flush();
210    }
211   
212    private double calculateReviewEffort(Instances testdata, Classifier classifier, List<Double> efforts) {
213        if( efforts==null ) {
214            return 0;
215        }
216       
217        final List<Integer> bugPredicted = new ArrayList<>();
218        final List<Integer> nobugPredicted = new ArrayList<>();
219        double totalLoc = 0.0d;
220        int totalBugs = 0;
221        for (int i = 0; i < testdata.numInstances(); i++) {
222            try {
223                if (Double.compare(classifier.classifyInstance(testdata.instance(i)), 0.0d) == 0) {
224                    nobugPredicted.add(i);
225                }
226                else {
227                    bugPredicted.add(i);
228                }
229            }
230            catch (Exception e) {
231                throw new RuntimeException(
232                                           "unexpected error during the evaluation of the review effort",
233                                           e);
234            }
235            if (Double.compare(testdata.instance(i).classValue(), 1.0d) == 0) {
236                totalBugs++;
237            }
238            totalLoc += efforts.get(i);
239        }
240
241        final List<Double> reviewLoc = new ArrayList<>(testdata.numInstances());
242        final List<Double> bugsFound = new ArrayList<>(testdata.numInstances());
243
244        double currentBugsFound = 0;
245
246        while (!bugPredicted.isEmpty()) {
247            double minLoc = Double.MAX_VALUE;
248            int minIndex = -1;
249            for (int i = 0; i < bugPredicted.size(); i++) {
250                double currentLoc = efforts.get(bugPredicted.get(i));
251                if (currentLoc < minLoc) {
252                    minIndex = i;
253                    minLoc = currentLoc;
254                }
255            }
256            if (minIndex != -1) {
257                reviewLoc.add(minLoc / totalLoc);
258
259                currentBugsFound += testdata.instance(bugPredicted.get(minIndex)).classValue();
260                bugsFound.add(currentBugsFound);
261
262                bugPredicted.remove(minIndex);
263            }
264            else {
265                throw new RuntimeException("Shouldn't happen!");
266            }
267        }
268
269        while (!nobugPredicted.isEmpty()) {
270            double minLoc = Double.MAX_VALUE;
271            int minIndex = -1;
272            for (int i = 0; i < nobugPredicted.size(); i++) {
273                double currentLoc = efforts.get(nobugPredicted.get(i));
274                if (currentLoc < minLoc) {
275                    minIndex = i;
276                    minLoc = currentLoc;
277                }
278            }
279            if (minIndex != -1) {
280                reviewLoc.add(minLoc / totalLoc);
281
282                currentBugsFound += testdata.instance(nobugPredicted.get(minIndex)).classValue();
283                bugsFound.add(currentBugsFound);
284                nobugPredicted.remove(minIndex);
285            }
286            else {
287                throw new RuntimeException("Shouldn't happen!");
288            }
289        }
290
291        double auc = 0.0;
292        for (int i = 0; i < bugsFound.size(); i++) {
293            auc += reviewLoc.get(i) * bugsFound.get(i) / totalBugs;
294        }
295
296        return auc;
297    }
298
299    @SuppressWarnings("unused")
300    @Deprecated
301    private double calculateReviewEffort(Instances testdata, Classifier classifier) {
302
303        // attribute in the JURECZKO data and default
304        Attribute loc = testdata.attribute("loc");
305        if (loc == null) {
306            // attribute in the NASA/SOFTMINE/MDP data
307            loc = testdata.attribute("LOC_EXECUTABLE");
308        }
309        if (loc == null) {
310            // attribute in the AEEEM data
311            loc = testdata.attribute("numberOfLinesOfCode");
312        }
313        if (loc == null) {
314            // attribute in the RELINK data
315            loc = testdata.attribute("CountLineCodeExe");
316        }
317        if( loc == null ) {
318            return 0.0;
319        }
320
321        final List<Integer> bugPredicted = new ArrayList<>();
322        final List<Integer> nobugPredicted = new ArrayList<>();
323        double totalLoc = 0.0d;
324        int totalBugs = 0;
325        for (int i = 0; i < testdata.numInstances(); i++) {
326            try {
327                if (Double.compare(classifier.classifyInstance(testdata.instance(i)), 0.0d) == 0) {
328                    nobugPredicted.add(i);
329                }
330                else {
331                    bugPredicted.add(i);
332                }
333            }
334            catch (Exception e) {
335                throw new RuntimeException(
336                                           "unexpected error during the evaluation of the review effort",
337                                           e);
338            }
339            if (Double.compare(testdata.instance(i).classValue(), 1.0d) == 0) {
340                totalBugs++;
341            }
342            totalLoc += testdata.instance(i).value(loc);
343        }
344
345        final List<Double> reviewLoc = new ArrayList<>(testdata.numInstances());
346        final List<Double> bugsFound = new ArrayList<>(testdata.numInstances());
347
348        double currentBugsFound = 0;
349
350        while (!bugPredicted.isEmpty()) {
351            double minLoc = Double.MAX_VALUE;
352            int minIndex = -1;
353            for (int i = 0; i < bugPredicted.size(); i++) {
354                double currentLoc = testdata.instance(bugPredicted.get(i)).value(loc);
355                if (currentLoc < minLoc) {
356                    minIndex = i;
357                    minLoc = currentLoc;
358                }
359            }
360            if (minIndex != -1) {
361                reviewLoc.add(minLoc / totalLoc);
362
363                currentBugsFound += testdata.instance(bugPredicted.get(minIndex)).classValue();
364                bugsFound.add(currentBugsFound);
365
366                bugPredicted.remove(minIndex);
367            }
368            else {
369                throw new RuntimeException("Shouldn't happen!");
370            }
371        }
372
373        while (!nobugPredicted.isEmpty()) {
374            double minLoc = Double.MAX_VALUE;
375            int minIndex = -1;
376            for (int i = 0; i < nobugPredicted.size(); i++) {
377                double currentLoc = testdata.instance(nobugPredicted.get(i)).value(loc);
378                if (currentLoc < minLoc) {
379                    minIndex = i;
380                    minLoc = currentLoc;
381                }
382            }
383            if (minIndex != -1) {
384                reviewLoc.add(minLoc / totalLoc);
385
386                currentBugsFound += testdata.instance(nobugPredicted.get(minIndex)).classValue();
387                bugsFound.add(currentBugsFound);
388                nobugPredicted.remove(minIndex);
389            }
390            else {
391                throw new RuntimeException("Shouldn't happen!");
392            }
393        }
394
395        double auc = 0.0;
396        for (int i = 0; i < bugsFound.size(); i++) {
397            auc += reviewLoc.get(i) * bugsFound.get(i) / totalBugs;
398        }
399
400        return auc;
401    }
402
403    /*
404     * (non-Javadoc)
405     *
406     * @see de.ugoe.cs.cpdp.Parameterizable#setParameter(java.lang.String)
407     */
408    @Override
409    public void setParameter(String parameters) {
410        if (output != null && !outputIsSystemOut) {
411            output.close();
412        }
413        if ("system.out".equals(parameters) || "".equals(parameters)) {
414            output = new PrintWriter(System.out);
415            outputIsSystemOut = true;
416        }
417        else {
418            try {
419                output = new PrintWriter(new FileOutputStream(parameters));
420                outputIsSystemOut = false;
421                int filenameStart = parameters.lastIndexOf('/')+1;
422                int filenameEnd = parameters.lastIndexOf('.');
423                configurationName = parameters.substring(filenameStart, filenameEnd);
424            }
425            catch (FileNotFoundException e) {
426                throw new RuntimeException(e);
427            }
428        }
429    }
430}
Note: See TracBrowser for help on using the repository browser.