source: trunk/CrossPare/src/de/ugoe/cs/cpdp/eval/AbstractWekaEvaluation.java

Last change on this file was 135, checked in by sherbold, 8 years ago
  • code documentation and formatting
  • Property svn:mime-type set to text/plain
File size: 17.8 KB
Line 
1// Copyright 2015 Georg-August-Universität Göttingen, Germany
2//
3//   Licensed under the Apache License, Version 2.0 (the "License");
4//   you may not use this file except in compliance with the License.
5//   You may obtain a copy of the License at
6//
7//       http://www.apache.org/licenses/LICENSE-2.0
8//
9//   Unless required by applicable law or agreed to in writing, software
10//   distributed under the License is distributed on an "AS IS" BASIS,
11//   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12//   See the License for the specific language governing permissions and
13//   limitations under the License.
14
15package de.ugoe.cs.cpdp.eval;
16
17import java.io.FileNotFoundException;
18import java.io.FileOutputStream;
19import java.io.PrintWriter;
20import java.util.ArrayList;
21import java.util.Iterator;
22import java.util.LinkedList;
23import java.util.List;
24
25import de.ugoe.cs.cpdp.training.ITrainer;
26import de.ugoe.cs.cpdp.training.IWekaCompatibleTrainer;
27import de.ugoe.cs.util.StringTools;
28import weka.classifiers.Classifier;
29import weka.classifiers.Evaluation;
30import weka.core.Attribute;
31import weka.core.Instances;
32
33/**
34 * Base class for the evaluation of results of classifiers compatible with the {@link Classifier}
35 * interface. For each classifier, the following metrics are calculated:
36 * <ul>
37 * <li>succHe: Success with recall>0.7, precision>0.5</li>
38 * <li>succZi: Success with recall>=0.75, precision>=0.7, and error<=0.25</li>
39 * <li>succG75: Success with gscore>0.75</li>
40 * <li>succG60: Success with gscore>0.6</li>
41 * <li>error</li>
42 * <li>recall</li>
43 * <li>precision</li>
44 * <li>fscore</li>
45 * <li>gscore</li>
46 * <li>MCC</li>
47 * <li>AUC</li>
48 * <li>AUCEC (weighted by LOC, if applicable; 0.0 if LOC not available)</li>
49 * <li>tpr: true positive rate</li>
50 * <li>tnr: true negative rate</li>
51 * <li>fpr: false positive rate</li>
52 * <li>fnr: false negative rate</li>
53 * <li>tp: true positives</li>
54 * <li>fp: false positives</li>
55 * <li>tn: true negatives</li>
56 * <li>fn: false negatives</li>
57 * </ul>
58 *
59 * @author Steffen Herbold
60 */
61public abstract class AbstractWekaEvaluation implements IEvaluationStrategy {
62
63    /**
64     * writer for the evaluation results
65     */
66    private PrintWriter output = new PrintWriter(System.out);
67
68    /**
69     * flag that defines if the output is the system out
70     */
71    private boolean outputIsSystemOut = true;
72
73    /**
74     * name of the configuration
75     */
76    private String configurationName = "default";
77
78    /**
79     * Creates the Weka evaluator. Allows the creation of the evaluator in different ways, e.g., for
80     * cross-validation or evaluation on the test data.
81     *
82     * @param testdata
83     *            test data
84     * @param classifier
85     *            classifier used
86     * @return evaluator
87     */
88    protected abstract Evaluation createEvaluator(Instances testdata, Classifier classifier);
89
90    /*
91     * (non-Javadoc)
92     *
93     * @see de.ugoe.cs.cpdp.eval.EvaluationStrategy#apply(weka.core.Instances, weka.core.Instances,
94     * java.util.List, boolean)
95     */
96    @Override
97    public void apply(Instances testdata,
98                      Instances traindata,
99                      List<ITrainer> trainers,
100                      List<Double> efforts,
101                      boolean writeHeader,
102                      List<IResultStorage> storages)
103    {
104        final List<Classifier> classifiers = new LinkedList<>();
105        final List<ExperimentResult> experimentResults = new LinkedList<>();
106        String productName = testdata.relationName();
107
108        for (ITrainer trainer : trainers) {
109            if (trainer instanceof IWekaCompatibleTrainer) {
110                classifiers.add(((IWekaCompatibleTrainer) trainer).getClassifier());
111                experimentResults
112                    .add(new ExperimentResult(configurationName, productName,
113                                              ((IWekaCompatibleTrainer) trainer).getName()));
114            }
115            else {
116                throw new RuntimeException("The selected evaluator only support Weka classifiers");
117            }
118        }
119
120        if (writeHeader) {
121            output.append("version,size_test,size_training");
122            for (ITrainer trainer : trainers) {
123                output.append(",succHe_" + ((IWekaCompatibleTrainer) trainer).getName());
124                output.append(",succZi_" + ((IWekaCompatibleTrainer) trainer).getName());
125                output.append(",succG75_" + ((IWekaCompatibleTrainer) trainer).getName());
126                output.append(",succG60_" + ((IWekaCompatibleTrainer) trainer).getName());
127                output.append(",error_" + ((IWekaCompatibleTrainer) trainer).getName());
128                output.append(",recall_" + ((IWekaCompatibleTrainer) trainer).getName());
129                output.append(",precision_" + ((IWekaCompatibleTrainer) trainer).getName());
130                output.append(",fscore_" + ((IWekaCompatibleTrainer) trainer).getName());
131                output.append(",gscore_" + ((IWekaCompatibleTrainer) trainer).getName());
132                output.append(",mcc_" + ((IWekaCompatibleTrainer) trainer).getName());
133                output.append(",auc_" + ((IWekaCompatibleTrainer) trainer).getName());
134                output.append(",aucec_" + ((IWekaCompatibleTrainer) trainer).getName());
135                output.append(",tpr_" + ((IWekaCompatibleTrainer) trainer).getName());
136                output.append(",tnr_" + ((IWekaCompatibleTrainer) trainer).getName());
137                output.append(",fpr_" + ((IWekaCompatibleTrainer) trainer).getName());
138                output.append(",fnr_" + ((IWekaCompatibleTrainer) trainer).getName());
139                output.append(",tp_" + ((IWekaCompatibleTrainer) trainer).getName());
140                output.append(",fn_" + ((IWekaCompatibleTrainer) trainer).getName());
141                output.append(",tn_" + ((IWekaCompatibleTrainer) trainer).getName());
142                output.append(",fp_" + ((IWekaCompatibleTrainer) trainer).getName());
143            }
144            output.append(StringTools.ENDLINE);
145        }
146
147        output.append(productName);
148        output.append("," + testdata.numInstances());
149        output.append("," + traindata.numInstances());
150
151        Evaluation eval = null;
152        Iterator<Classifier> classifierIter = classifiers.iterator();
153        Iterator<ExperimentResult> resultIter = experimentResults.iterator();
154        while (classifierIter.hasNext()) {
155            Classifier classifier = classifierIter.next();
156            eval = createEvaluator(testdata, classifier);
157
158            double pf =
159                eval.numFalsePositives(1) / (eval.numFalsePositives(1) + eval.numTrueNegatives(1));
160            double gmeasure = 2 * eval.recall(1) * (1.0 - pf) / (eval.recall(1) + (1.0 - pf));
161            double aucec = calculateReviewEffort(testdata, classifier, efforts);
162            double succHe = eval.recall(1) >= 0.7 && eval.precision(1) >= 0.5 ? 1.0 : 0.0;
163            double succZi = eval.recall(1) >= 0.75 && eval.precision(1) >= 0.75 && eval.errorRate()<=0.25 ? 1.0 : 0.0;
164            double succG75 = gmeasure > 0.75 ? 1.0 : 0.0;
165            double succG60 = gmeasure > 0.6 ? 1.0 : 0.0;
166
167            output.append("," + succHe);
168            output.append("," + succZi);
169            output.append("," + succG75);
170            output.append("," + succG60);
171            output.append("," + eval.errorRate());
172            output.append("," + eval.recall(1));
173            output.append("," + eval.precision(1));
174            output.append("," + eval.fMeasure(1));
175            output.append("," + gmeasure);
176            output.append("," + eval.matthewsCorrelationCoefficient(1));
177            output.append("," + eval.areaUnderROC(1));
178            output.append("," + aucec);
179            output.append("," + eval.truePositiveRate(1));
180            output.append("," + eval.trueNegativeRate(1));
181            output.append("," + eval.falsePositiveRate(1));
182            output.append("," + eval.falseNegativeRate(1));
183            output.append("," + eval.numTruePositives(1));
184            output.append("," + eval.numFalseNegatives(1));
185            output.append("," + eval.numTrueNegatives(1));
186            output.append("," + eval.numFalsePositives(1));
187
188            ExperimentResult result = resultIter.next();
189            result.setSizeTestData(testdata.numInstances());
190            result.setSizeTrainingData(traindata.numInstances());
191            result.setError(eval.errorRate());
192            result.setRecall(eval.recall(1));
193            result.setPrecision(eval.precision(1));
194            result.setFscore(eval.fMeasure(1));
195            result.setGscore(gmeasure);
196            result.setMcc(eval.matthewsCorrelationCoefficient(1));
197            result.setAuc(eval.areaUnderROC(1));
198            result.setAucec(aucec);
199            result.setTpr(eval.truePositiveRate(1));
200            result.setTnr(eval.trueNegativeRate(1));
201            result.setFpr(eval.falsePositiveRate(1));
202            result.setFnr(eval.falseNegativeRate(1));
203            result.setTp(eval.numTruePositives(1));
204            result.setFn(eval.numFalseNegatives(1));
205            result.setTn(eval.numTrueNegatives(1));
206            result.setFp(eval.numFalsePositives(1));
207            for (IResultStorage storage : storages) {
208                storage.addResult(result);
209            }
210        }
211
212        output.append(StringTools.ENDLINE);
213        output.flush();
214    }
215
216    /**
217     * <p>
218     * Calculates the effort. TODO: IMPLEMENTATION BUGGY! MUST BE FIXED!
219     * </p>
220     *
221     * @param testdata
222     *            the test data
223     * @param classifier
224     *            the classifier
225     * @param efforts
226     *            the effort information for each instance in the test data
227     * @return
228     */
229    private double calculateReviewEffort(Instances testdata,
230                                         Classifier classifier,
231                                         List<Double> efforts)
232    {
233        if (efforts == null) {
234            return 0;
235        }
236
237        final List<Integer> bugPredicted = new ArrayList<>();
238        final List<Integer> nobugPredicted = new ArrayList<>();
239        double totalLoc = 0.0d;
240        int totalBugs = 0;
241        for (int i = 0; i < testdata.numInstances(); i++) {
242            try {
243                if (Double.compare(classifier.classifyInstance(testdata.instance(i)), 0.0d) == 0) {
244                    nobugPredicted.add(i);
245                }
246                else {
247                    bugPredicted.add(i);
248                }
249            }
250            catch (Exception e) {
251                throw new RuntimeException("unexpected error during the evaluation of the review effort",
252                                           e);
253            }
254            if (Double.compare(testdata.instance(i).classValue(), 1.0d) == 0) {
255                totalBugs++;
256            }
257            totalLoc += efforts.get(i);
258        }
259
260        final List<Double> reviewLoc = new ArrayList<>(testdata.numInstances());
261        final List<Double> bugsFound = new ArrayList<>(testdata.numInstances());
262
263        double currentBugsFound = 0;
264
265        while (!bugPredicted.isEmpty()) {
266            double minLoc = Double.MAX_VALUE;
267            int minIndex = -1;
268            for (int i = 0; i < bugPredicted.size(); i++) {
269                double currentLoc = efforts.get(bugPredicted.get(i));
270                if (currentLoc < minLoc) {
271                    minIndex = i;
272                    minLoc = currentLoc;
273                }
274            }
275            if (minIndex != -1) {
276                reviewLoc.add(minLoc / totalLoc);
277
278                currentBugsFound += testdata.instance(bugPredicted.get(minIndex)).classValue();
279                bugsFound.add(currentBugsFound);
280
281                bugPredicted.remove(minIndex);
282            }
283            else {
284                throw new RuntimeException("Shouldn't happen!");
285            }
286        }
287
288        while (!nobugPredicted.isEmpty()) {
289            double minLoc = Double.MAX_VALUE;
290            int minIndex = -1;
291            for (int i = 0; i < nobugPredicted.size(); i++) {
292                double currentLoc = efforts.get(nobugPredicted.get(i));
293                if (currentLoc < minLoc) {
294                    minIndex = i;
295                    minLoc = currentLoc;
296                }
297            }
298            if (minIndex != -1) {
299                reviewLoc.add(minLoc / totalLoc);
300
301                currentBugsFound += testdata.instance(nobugPredicted.get(minIndex)).classValue();
302                bugsFound.add(currentBugsFound);
303                nobugPredicted.remove(minIndex);
304            }
305            else {
306                throw new RuntimeException("Shouldn't happen!");
307            }
308        }
309
310        double auc = 0.0;
311        for (int i = 0; i < bugsFound.size(); i++) {
312            auc += reviewLoc.get(i) * bugsFound.get(i) / totalBugs;
313        }
314
315        return auc;
316    }
317
318    /**
319     * <p>
320     * Calculates effort. Deprecated. Do not use!
321     * </p>
322     *
323     * @param testdata
324     *            the test data
325     * @param classifier
326     *            the classifier
327     * @return
328     */
329    @SuppressWarnings("unused")
330    @Deprecated
331    private double calculateReviewEffort(Instances testdata, Classifier classifier) {
332
333        // attribute in the JURECZKO data and default
334        Attribute loc = testdata.attribute("loc");
335        if (loc == null) {
336            // attribute in the NASA/SOFTMINE/MDP data
337            loc = testdata.attribute("LOC_EXECUTABLE");
338        }
339        if (loc == null) {
340            // attribute in the AEEEM data
341            loc = testdata.attribute("numberOfLinesOfCode");
342        }
343        if (loc == null) {
344            // attribute in the RELINK data
345            loc = testdata.attribute("CountLineCodeExe");
346        }
347        if (loc == null) {
348            return 0.0;
349        }
350
351        final List<Integer> bugPredicted = new ArrayList<>();
352        final List<Integer> nobugPredicted = new ArrayList<>();
353        double totalLoc = 0.0d;
354        int totalBugs = 0;
355        for (int i = 0; i < testdata.numInstances(); i++) {
356            try {
357                if (Double.compare(classifier.classifyInstance(testdata.instance(i)), 0.0d) == 0) {
358                    nobugPredicted.add(i);
359                }
360                else {
361                    bugPredicted.add(i);
362                }
363            }
364            catch (Exception e) {
365                throw new RuntimeException("unexpected error during the evaluation of the review effort",
366                                           e);
367            }
368            if (Double.compare(testdata.instance(i).classValue(), 1.0d) == 0) {
369                totalBugs++;
370            }
371            totalLoc += testdata.instance(i).value(loc);
372        }
373
374        final List<Double> reviewLoc = new ArrayList<>(testdata.numInstances());
375        final List<Double> bugsFound = new ArrayList<>(testdata.numInstances());
376
377        double currentBugsFound = 0;
378
379        while (!bugPredicted.isEmpty()) {
380            double minLoc = Double.MAX_VALUE;
381            int minIndex = -1;
382            for (int i = 0; i < bugPredicted.size(); i++) {
383                double currentLoc = testdata.instance(bugPredicted.get(i)).value(loc);
384                if (currentLoc < minLoc) {
385                    minIndex = i;
386                    minLoc = currentLoc;
387                }
388            }
389            if (minIndex != -1) {
390                reviewLoc.add(minLoc / totalLoc);
391
392                currentBugsFound += testdata.instance(bugPredicted.get(minIndex)).classValue();
393                bugsFound.add(currentBugsFound);
394
395                bugPredicted.remove(minIndex);
396            }
397            else {
398                throw new RuntimeException("Shouldn't happen!");
399            }
400        }
401
402        while (!nobugPredicted.isEmpty()) {
403            double minLoc = Double.MAX_VALUE;
404            int minIndex = -1;
405            for (int i = 0; i < nobugPredicted.size(); i++) {
406                double currentLoc = testdata.instance(nobugPredicted.get(i)).value(loc);
407                if (currentLoc < minLoc) {
408                    minIndex = i;
409                    minLoc = currentLoc;
410                }
411            }
412            if (minIndex != -1) {
413                reviewLoc.add(minLoc / totalLoc);
414
415                currentBugsFound += testdata.instance(nobugPredicted.get(minIndex)).classValue();
416                bugsFound.add(currentBugsFound);
417                nobugPredicted.remove(minIndex);
418            }
419            else {
420                throw new RuntimeException("Shouldn't happen!");
421            }
422        }
423
424        double auc = 0.0;
425        for (int i = 0; i < bugsFound.size(); i++) {
426            auc += reviewLoc.get(i) * bugsFound.get(i) / totalBugs;
427        }
428
429        return auc;
430    }
431
432    /*
433     * (non-Javadoc)
434     *
435     * @see de.ugoe.cs.cpdp.Parameterizable#setParameter(java.lang.String)
436     */
437    @Override
438    public void setParameter(String parameters) {
439        if (output != null && !outputIsSystemOut) {
440            output.close();
441        }
442        if ("system.out".equals(parameters) || "".equals(parameters)) {
443            output = new PrintWriter(System.out);
444            outputIsSystemOut = true;
445        }
446        else {
447            try {
448                output = new PrintWriter(new FileOutputStream(parameters));
449                outputIsSystemOut = false;
450                int filenameStart = parameters.lastIndexOf('/') + 1;
451                int filenameEnd = parameters.lastIndexOf('.');
452                configurationName = parameters.substring(filenameStart, filenameEnd);
453            }
454            catch (FileNotFoundException e) {
455                throw new RuntimeException(e);
456            }
457        }
458    }
459}
Note: See TracBrowser for help on using the repository browser.