Changeset 129 for trunk/CrossPare/src/de

Timestamp:

06/22/16 11:25:46 (8 years ago)

Author:

sherbold

Message:

• added the same workaround for the problem with Discretize to the TopMetricFilter?. We slightly refactored the implementation within the AbstractCODEP by putting the rescaling of sets to the WekaUtils? to facilitate better re-use.

Location:

trunk/CrossPare/src/de/ugoe/cs/cpdp

Files:

• dataprocessing/TopMetricFilter.java (modified) (5 diffs)
• util/WekaUtils.java (modified) (2 diffs)
• wekaclassifier/AbstractCODEP.java (modified) (5 diffs)

trunk/CrossPare/src/de/ugoe/cs/cpdp/dataprocessing/TopMetricFilter.java

@@ -21 +21 @@
 import java.util.TreeSet;
 import java.util.logging.Level;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 import java.util.stream.IntStream;
 
@@ -29 +31 @@
 
 import de.ugoe.cs.cpdp.util.SortUtils;
+import de.ugoe.cs.cpdp.util.WekaUtils;
 import de.ugoe.cs.util.console.Console;
 import weka.attributeSelection.AttributeSelection;
@@ -96 +99 @@
         List<Set<Integer>> cfsSets = new LinkedList<>();
         for( Instances traindata : traindataSet ) {
-            AttributeSelection attsel = new AttributeSelection();
-            CfsSubsetEval eval = new CfsSubsetEval();
-            GreedyStepwise search = new GreedyStepwise();
-            search.setSearchBackwards(true);
-            attsel.setEvaluator(eval);
-            attsel.setSearch(search);
-            attsel.SelectAttributes(traindata);
-            Set<Integer> cfsSet = new HashSet<>();
-            for( int attr : attsel.selectedAttributes() ) {
-                cfsSet.add(attr);
-            }
-            cfsSets.add(cfsSet);
+            boolean selectionSuccessful = false;
+            boolean secondAttempt = false;
+            Instances traindataCopy = null;
+            do {
+                try {
+                    if (secondAttempt) {
+                        AttributeSelection attsel = new AttributeSelection();
+                        CfsSubsetEval eval = new CfsSubsetEval();
+                        GreedyStepwise search = new GreedyStepwise();
+                        search.setSearchBackwards(true);
+                        attsel.setEvaluator(eval);
+                        attsel.setSearch(search);
+                        attsel.SelectAttributes(traindataCopy);
+                        Set<Integer> cfsSet = new HashSet<>();
+                        for( int attr : attsel.selectedAttributes() ) {
+                            cfsSet.add(attr);
+                        }
+                        cfsSets.add(cfsSet);
+                        selectionSuccessful = true;
+                    }
+                    else {
+                        AttributeSelection attsel = new AttributeSelection();
+                        CfsSubsetEval eval = new CfsSubsetEval();
+                        GreedyStepwise search = new GreedyStepwise();
+                        search.setSearchBackwards(true);
+                        attsel.setEvaluator(eval);
+                        attsel.setSearch(search);
+                        attsel.SelectAttributes(traindata);
+                        Set<Integer> cfsSet = new HashSet<>();
+                        for( int attr : attsel.selectedAttributes() ) {
+                            cfsSet.add(attr);
+                        }
+                        cfsSets.add(cfsSet);
+                        selectionSuccessful = true;
+                    }
+                }
+                catch (IllegalArgumentException e) {
+                    String regex = "A nominal attribute \\((.*)\\) cannot have duplicate labels.*";
+                    Pattern p = Pattern.compile(regex);
+                    Matcher m = p.matcher(e.getMessage());
+                    if (!m.find()) {
+                        // cannot treat problem, rethrow exception
+                        throw e;
+                    }
+                    String attributeName = m.group(1);
+                    int attrIndex = traindata.attribute(attributeName).index();
+                    if (secondAttempt) {
+                        traindataCopy = WekaUtils.upscaleAttribute(traindataCopy, attrIndex);
+                    }
+                    else {
+                        traindataCopy = WekaUtils.upscaleAttribute(traindata, attrIndex);
+                    }
+                    Console
+                        .traceln(Level.FINE,
+                                 "upscaled attribute " + attributeName + "; restarting training");
+                    secondAttempt = true;
+                    continue;
+                }
+            }
+            while (!selectionSuccessful); // dummy loop for internal continue
         }
         
@@ -143 +194 @@
         }
         Set<Integer> topkSetIndexSet = new TreeSet<>();
-        for( int j=0; j<bestCoverageIndex; j++ ) {
+        // j<30 ensures that the computational time does not explode since the powerset is 2^n in complexity
+        for( int j=0; j<bestCoverageIndex && j<30 ; j++ ) {
             topkSetIndexSet.add(j);
         }
@@ -157 +209 @@
                 }
                 for( Set<Integer> cfsSet : cfsSets ) {
-                    currentCoverage += (coverage(combination, cfsSet)/traindataSet.size());
+                    currentCoverage += (coverage(topkCombination, cfsSet)/traindataSet.size());
                 }
                 if( currentCoverage > bestOptCoverage ) {

trunk/CrossPare/src/de/ugoe/cs/cpdp/util/WekaUtils.java

@@ -37 +37 @@
         }
     }
+    
+    /**
+     * Scaling value that moves the decimal point by 5 digets.
+     */
+    public final static double SCALER = 10000.0d;
     
     /**
@@ -151 +156 @@
         return new DistChar(mean, std, min, max, data.numInstances());
     }
+    
+    /**
+     * <p>
+     * Upscales the value of a single attribute. This is a workaround to get BayesNet running for
+     * all data. Works on a copy of the training data, i.e., leaves the original data untouched.
+     * </p>
+     *
+     * @param traindata
+     *            data from which the attribute is upscaled.
+     * @param attributeIndex
+     *            index of the attribute
+     * @return data with upscaled attribute
+     */
+    public static Instances upscaleAttribute(Instances traindata, int attributeIndex) {
+        Instances traindataCopy = new Instances(traindata);
+        for (int i = 0; i < traindata.size(); i++) {
+            traindataCopy.get(i).setValue(attributeIndex,
+                                          traindata.get(i).value(attributeIndex) * SCALER);
+        }
+        return traindataCopy;
+    }
 }

trunk/CrossPare/src/de/ugoe/cs/cpdp/wekaclassifier/AbstractCODEP.java

@@ -23 +23 @@
 import java.util.regex.Pattern;
 
+import de.ugoe.cs.cpdp.util.WekaUtils;
 import de.ugoe.cs.util.console.Console;
 import weka.classifiers.AbstractClassifier;
@@ -71 +72 @@
     private Map<Integer, Integer> upscaleIndex = null;
 
-    /**
-     * Scaling value that moves the decimal point by 5 digets.
-     */
-    private final double SCALER = 10000.0d;
-
     /*
      * (non-Javadoc)
@@ -136 +132 @@
                     }
                     else {
-                        traindataCopy = upscaleAttribute(traindata, attrIndex);
+                        traindataCopy = WekaUtils.upscaleAttribute(traindata, attrIndex);
                     }
 
@@ -182 +178 @@
                 // instance value must be upscaled
                 int attrIndex = upscaleIndex.get(j);
-                double upscaledVal = instance.value(attrIndex) * SCALER;
+                double upscaledVal = instance.value(attrIndex) * WekaUtils.SCALER;
                 traindataCopy = new Instances(instance.dataset());
                 instance = new DenseInstance(instance.weight(), instance.toDoubleArray());
@@ -231 +227 @@
     /**
      * <p>
-     * Upscales the value of a single attribute. This is a workaround to get BayesNet running for
-     * all data. Works on a copy of the training data, i.e., leaves the original data untouched.
-     * </p>
-     *
-     * @param traindata
-     *            data from which the attribute is upscaled.
-     * @param attributeIndex
-     *            index of the attribute
-     * @return data with upscaled attribute
-     */
-    private Instances upscaleAttribute(Instances traindata, int attributeIndex) {
-        Instances traindataCopy = new Instances(traindata);
-        for (int i = 0; i < traindata.size(); i++) {
-            traindataCopy.get(i).setValue(attributeIndex,
-                                          traindata.get(i).value(attributeIndex) * SCALER);
-        }
-        return traindataCopy;
-    }
-
-    /**
-     * <p>
      * Abstract method through which implementing classes define which classifier is used for the
      * CODEP.