Index: trunk/CrossPare/decent/epsilon/libraries/arff/common.eol
===================================================================
--- trunk/CrossPare/decent/epsilon/libraries/arff/common.eol (revision 32)
+++ trunk/CrossPare/decent/epsilon/libraries/arff/common.eol (revision 32)
@@ -0,0 +1,43 @@
+operation ARFFx!Instance addValue(content : String, attribute : String) {
+ var value = new ARFFx!Value();
+ value.ofAttribute = attribute.getARFFAttribute(self.eContainer());
+ value.content = content;
+ self.values.add(value);
+}
+
+
+operation ARFFx!Instance updateValue(content : String, attribute : String) {
+ var value = self.values.select(v|v.ofAttribute.name = attribute).first();
+ if (value.isUndefined()) {
+ value = new ARFFx!Value();
+ value.ofAttribute = attribute.getARFFAttribute(self.eContainer());
+ self.values.add(value);
+ }
+ value.content = content;
+}
+
+
+@cached
+operation String getARFFAttribute(arffx : ARFFx!Model) : ARFFx!Attribute {
+ var attribute = arffx.attributes.select(a|a.name = self).first();
+ if (attribute.isUndefined()) {
+ attribute = new ARFFx!Attribute();
+ attribute.name = self;
+ arffx.attributes.add(attribute);
+ }
+ return attribute;
+}
+
+operation String getSelectedAttributes() : Sequence {
+ var file = new Native("java.io.File")(self);
+ var content = new Native("org.apache.commons.io.FileUtils").readLines(file);
+
+
+ content = content.select(x|not x.startsWith("#")).closure(x|x.trim());
+ return content;
+}
+
+@cached
+operation ARFFx!Instance getValue(attribute : ARFFx!Attribute) : String {
+ return self.values.select(v|v.ofAttribute = attribute).first().content;
+}
Index: trunk/CrossPare/decent/epsilon/libraries/decent/common.eol
===================================================================
--- trunk/CrossPare/decent/epsilon/libraries/decent/common.eol (revision 32)
+++ trunk/CrossPare/decent/epsilon/libraries/decent/common.eol (revision 32)
@@ -0,0 +1,384 @@
+operation Any addValue(attribute : DECENT!Attribute, content : Any, value : DECENT!Value) : DECENT!Value {
+ value.name = attribute.name;
+ value.content = content;
+ self.values.add(value);
+ //if the is to be kept this should be moved too
+ value.ofAttribute = attribute;
+ //TODO: outsource these
+// self.artifact.attributes.add(attribute);
+// attribute.artifactTypes.add(self.artifact.type);
+ return value;
+}
+
+operation getProfiler(activated : Boolean) : Any {
+ if (activated) {
+ var profiler : new Native("org.eclipse.epsilon.eol.tools.ProfilerTool");
+ return profiler;
+ } else {
+ return null;
+ }
+}
+
+operation Any startProfiling(target : String) {
+ if (self.isDefined()) {
+ self.start(target);
+ }
+}
+
+operation Any stopProfiling() {
+ if (self.isDefined()) {
+ self.stop();
+ }
+}
+
+operation Any startProfiling(target : String, data : Any) {
+ if (self.isDefined()) {
+ self.start(target,data);
+ }
+}
+
+operation Any getValue(attribute : DECENT!Attribute) : DECENT!Value {
+ return self.values.select(v|v.ofAttribute = attribute).first();
+}
+
+operation Any getValue(attribute : String) : DECENT!Value {
+ return self.values.select(v|v.ofAttribute.name = attribute).first();
+}
+
+
+@cached
+operation String getAttribute() : DECENT!Attribute {
+ return DECENT!Attribute.allInstances().select(a|a.name = self).first();
+}
+
+@cached
+operation String getArtifactType() : DECENT!ArtifactType {
+ return DECENT!ArtifactType.allInstances().select(a|a.name = self).first();
+}
+
+
+operation Any addValue(attribute : DECENT!Attribute, content : String) : DECENT!Value {
+ var value = self.getValue(attribute);
+ if (value.isUndefined()) {
+ value = new DECENT!StringValue;
+ }
+ return self.addValue(attribute, content, value);
+}
+
+operation DECENT!DECENT::Model addStep(name : String, duration : Real) {
+ var step : new DECENT!Step();
+ step.name = name;
+ step.duration = duration.toString();
+ self.steps.add(step);
+}
+
+
+//NOTE: order matters, otherwise integers are mistakenly treated as reals
+operation Any addValue(attribute : DECENT!Attribute, content : Integer) : DECENT!Value {
+ var value = self.getValue(attribute);
+ if (value.isUndefined()) {
+ value = new DECENT!IntegerValue;
+ }
+ return self.addValue(attribute, content, value);
+}
+
+operation Any addValue(attribute : DECENT!Attribute, content : Real) : DECENT!Value {
+ var value = self.getValue(attribute);
+ if (value.isUndefined()) {
+ value = new DECENT!DoubleValue;
+ }
+ return self.addValue(attribute, content, value);
+}
+
+operation Any addValue(attribute : DECENT!Attribute, content : Collection) : DECENT!Value {
+ var value = self.getValue(attribute);
+ if (value.isUndefined()) {
+ value = new DECENT!ListValue;
+ }
+ return self.addValue(attribute, content, value);
+}
+
+operation Any addStringCollectionValue(attribute : DECENT!Attribute, content : Collection(String)) : DECENT!Value {
+ var value = self.getValue(attribute);
+ if (value.isUndefined()) {
+ value = new DECENT!StringListValue;
+ }
+ return self.addValue(attribute, content, value);
+}
+
+operation Any addIntegerCollectionValue(attribute : DECENT!Attribute, content : Collection(Integer)) : DECENT!Value {
+ var value = self.getValue(attribute);
+ if (value.isUndefined()) {
+ value = new DECENT!IntegerListValue;
+ }
+ return self.addValue(attribute, content, value);
+}
+
+operation Any addDoubleCollectionValue(attribute : DECENT!Attribute, content : Collection(Real)) : DECENT!Value {
+ var value = self.getValue(attribute);
+ if (value.isUndefined()) {
+ value = new DECENT!DoubleListValue;
+ }
+ return self.addValue(attribute, content, value);
+}
+
+
+operation DECENT!Activity addDelta(attribute : DECENT!Attribute) : DECENT!Delta {
+ return self.addDelta(attribute, self.targetState.values.select(v|v.ofAttribute = attribute).first());
+}
+
+operation DECENT!Activity getDelta(attribute : DECENT!Attribute) : DECENT!Delta {
+ return self.deltas.select(d|d.onAttribute = attribute).first();
+}
+
+operation DECENT!Activity addDelta(attribute : DECENT!Attribute, value : DECENT!Value) : DECENT!Delta {
+ var delta = self.getDelta(attribute);
+ if (delta.isUndefined()) {
+ delta = new DECENT!Delta;
+ delta.onAttribute = attribute;
+ }
+
+ //TODO: see if there is any impact of removing these
+ // probably only on loading and saving times
+ // ***slighly negative performance impact for whatever reason***
+ //delta.targetValue = value;
+ var p = self.state;
+ while (p.isDefined() and not p.isHit()) {
+ //TODO: what if more than one?
+ p = p.previous.first();
+ }
+ if (p.isDefined()){
+ //delta.sourceValue = self.state.getValue(attribute);
+ //delta.change = delta.targetValue.content - delta.sourceValue.content;
+ //(self.targetState.artifact.name+":"+p.ID + "->"+self.targetState.ID + ":" + attribute.name+"="+p.getValue(attribute)).println();
+ delta.change = (value.content - p.getValue(attribute).content).asDouble();
+ } else {
+ //delta.change = delta.targetValue.content;
+ delta.change = value.content.asDouble();
+ }
+ self.deltas.add(delta);
+ return delta;
+}
+
+operation Any toDays() : Real {
+ return self.asDouble() / (3600*1000*24);
+}
+
+operation Any toMinutes() : Real {
+ return self.asDouble() / (60*1000);
+}
+
+operation Real round(places : Integer) : Real {
+ var factor = 10.pow(places).asDouble();
+ return (self.asDouble() * factor).round() / factor;
+}
+
+
+//TODO: reuse in famix2decent, dude2decent and elsewhere
+@cached
+operation Integer getMGStatesForID() : Collection {
+ //this makes a huge difference!
+ //return DECENT!State.allInstances().select(x|x.artifact.isMGType() and x.ID = self);
+ return DECENT!Artifact.allInstances().select(x|x.isMGType()).collect(y|y.states).flatten().select(z|z.ID = self);
+}
+
+@cached
+operation Integer getCodeStatesForID() : Collection {
+ //this makes a huge difference!
+ //return DECENT!State.allInstances().select(x|x.artifact.isCodeType() and x.ID = self);
+ return DECENT!Artifact.allInstances().select(x|x.isCodeType()).collect(y|y.states).flatten().select(z|z.ID = self);
+}
+
+@cached
+operation Integer getFAMIXStatesForID() : Collection {
+ //this makes a huge difference!
+ //return DECENT!State.allInstances().select(x|x.artifact.isCodeType() and x.ID = self);
+ return DECENT!Artifact.allInstances().select(x|x.isFAMIXType()).collect(y|y.states).flatten().select(z|z.ID = self);
+}
+
+@cached
+operation Integer getHitFAMIXStatesForID() : Collection {
+ //this makes a huge difference!
+ //return DECENT!State.allInstances().select(x|x.artifact.isCodeType() and x.ID = self);
+ return self.getFAMIXStatesForID().select(x|x.isHit());
+}
+
+@cached
+operation DECENT!State isHit() : Boolean {
+ if (self.artifact.isMGType()) {
+ return true;
+ }
+ var pre = self.getValue("LinesPostHits".getAttribute());
+ var post = self.getValue("LinesPreHits".getAttribute());
+ return ((pre.isDefined() and pre.content.isDefined() and pre.content.size() > 0)
+ or (post.isDefined() and post.content.isDefined() and post.content.size() > 0));
+}
+
+@cached
+operation Collection getMatchingStateForPath(filePath : String) : Collection {
+ return self.select(x|x.getValue(FilePathAttribute).content = filePath);
+}
+
+
+operation DECENT!Activity setActivityType(type : String) {
+ var t = DECENT!ActivityType.allInstances().select(a|a.name = type).first();
+ if (t.isUndefined()) {
+ t = new DECENT!ActivityType();
+ t.name = type;
+ DECENT!Model.allInstances().first().activityTypes.add(t);
+ }
+ self.type.add(t);
+}
+
+
+operation addAttribute(name : String) : DECENT!Attribute {
+ return addAttribute(name, "");
+}
+
+operation String printlno(writer) {
+ writer.write(self+"\n");
+ writer.flush();
+}
+
+operation addArtifactType(name : String) : DECENT!ArtifactType {
+ var type = DECENT!ArtifactType.allInstances().select(x|x.name = name).first();
+ if (type.isUndefined()) {
+ type = new DECENT!ArtifactType;
+ type.name = name;
+ }
+ return type;
+}
+
+
+operation addAttribute(name : String, description : String) : DECENT!Attribute {
+ var attribute = Model.attributePool.attributes.select(att|att.name = name).first();
+ if (attribute.isUndefined()) {
+ attribute = new DECENT!Attribute;
+ attribute.name = name;
+ attribute.description = description;
+ Model.attributePool.attributes.add(attribute);
+ }
+ return attribute;
+}
+
+operation addAttribute(name : String, description : String, dimension : String) : DECENT!Attribute {
+ var attribute = addAttribute(name, description);
+ var Dimension = Model.dimensions.select(x|x.name = dimension).first();
+ if (Dimension.isUndefined()) {
+ Dimension = new DECENT!Dimension;
+ Dimension.name = dimension;
+ Model.dimensions.add(Dimension);
+ }
+ attribute.dimension = Dimension;
+ return attribute;
+}
+
+operation DECENT!DECENT::Model addAttributes(attributes : Map) {
+ for (a in attributes.keySet()) {
+ if (self.attributePool.attributes.select(att|att.name = a).size() = 0) {
+ var Attribute = new DECENT!Attribute;
+ Attribute.name = a;
+ Attribute.description = attributes.get(a);
+ self.attributePool.attributes.add(Attribute);
+ }
+ }
+}
+
+operation Real normalize(min : Real, max : Real) : Real {
+ return ((self - min) / (max - min))*(1-0) + 0;
+}
+
+operation Real normalize(min : Real, max : Real,new_min : Real, new_max : Real) : Real {
+ return ((self - min) / (max - min))*(new_max - new_min) + new_min;
+}
+
+
+@cached
+operation DECENT!Artifact isMGType() : Boolean {
+ return self.type.name = "code" or
+ self.type.name = "documentation" or
+ self.type.name = "image" or
+ self.type.name = "devel-doc" or
+ self.type.name = "unknown" or
+ self.type.name = "build" or
+ self.type.name = "ui";
+}
+
+@cached
+operation DECENT!Artifact isFAMIXType() : Boolean {
+ return self.type.name = "Class" or
+ self.type.name = "ParameterizableClass" or
+ self.type.name = "Method" or
+ self.type.name = "Function" or
+ self.type.name = "Module";
+}
+
+
+@cached
+operation DECENT!Artifact isCodeType() : Boolean {
+ return self.type.name = "code";
+}
+
+operation Collection getMean() : Real {
+ if (self.size <= 1) {
+ //mean for a set of 1 elememts shall be 0
+ return 0.asReal();
+ }
+ var n = 0.asReal();
+ var sum1 = 0.asReal();
+ for (x in self){
+ n = n + 1;
+ sum1 = sum1 + x;
+ }
+ var mean = sum1/n;
+ return mean;
+}
+
+@cached
+operation Real getSqrt() : Real {
+ return Native("java.lang.Math").sqrt(self);
+}
+
+operation Collection getMin() : Real {
+ return Native("java.util.Collections").min(self);
+}
+
+operation Collection getMax() : Real {
+ return Native("java.util.Collections").max(self);
+}
+
+operation Collection normalizeMinMax(new_min : Real, new_max : Real) {
+ var min = self.getMin();
+ var max = self.getMax();
+ return self.collect(i|i.normalize(min, max, new_min, new_max));
+}
+
+
+operation Collection getStandardDeviation() : Real {
+ return self.getVariance().getSqrt();
+}
+
+operation Collection getVariance() : Real {
+ if (self.size <= 1) {
+ //variance for a set of 1 elememts shall be 0
+ return 0.asReal();
+ }
+
+ var n = 0.asReal();
+ var sum1 = 0.asReal();
+ for (x in self){
+ n = n + 1;
+ sum1 = sum1 + x;
+ }
+ var mean = sum1/n;
+
+ var sum2 = 0.asReal();
+ var sum3 = 0.asReal();
+ for (x in self){
+ sum2 = sum2 + ((x - mean) * (x - mean));
+ sum3 = sum3 + (x - mean);
+
+ }
+ var variance = (sum2 - (sum3*sum3)/n)/(n - 1);
+ return variance;
+}
Index: trunk/CrossPare/decent/epsilon/libraries/decent/logging.eol
===================================================================
--- trunk/CrossPare/decent/epsilon/libraries/decent/logging.eol (revision 32)
+++ trunk/CrossPare/decent/epsilon/libraries/decent/logging.eol (revision 32)
@@ -0,0 +1,25 @@
+
+operation String log(level : Integer) {
+ var levelLimit = 1;
+ //TODO: check if this has performance implications -> minor ~3-5% depending on density of logging
+ var logSetting = Native("java.lang.System").getProperty("epsilon.logLevel");
+ if (logSetting.isDefined()) {
+ levelLimit = logSetting.asInteger();
+ }
+ if (level <= levelLimit) {
+ var system = Native("java.lang.System");
+ var t = system.currentTimeMillis();
+ (t+ " : " + level + " : " + self).println();
+ var logToFile = Native("java.lang.System").getProperty("epsilon.logToFile");
+ var logFileAvailable = Native("java.lang.System").getProperty("epsilon.logFileAvailable");
+ if (logToFile.isDefined() and logToFile = 'true'
+ and logFileAvailable.isDefined()
+ and logFileAvailable = 'true') {
+ var i = new LOG!LogItem();
+ i.time = t.asInteger();
+ i.level = level;
+ i.message = self;
+ //LOG!LogModel.allInstances().first().items.add(i);
+ }
+ }
+}
Index: trunk/CrossPare/decent/epsilon/query/addLabels.eol
===================================================================
--- trunk/CrossPare/decent/epsilon/query/addLabels.eol (revision 32)
+++ trunk/CrossPare/decent/epsilon/query/addLabels.eol (revision 32)
@@ -0,0 +1,170 @@
+import "../libraries/arff/common.eol";
+import "../libraries/decent/common.eol";
+import "../libraries/decent/logging.eol";
+
+"Running addlabels".log(1);
+var start = Native("java.lang.System").currentTimeMillis();
+
+
+var s = ",";
+var targetAttributes = new Map;
+//TODO: export options
+targetAttributes.put("Artifact.Target.BugFix.AverageWeight",0.1);
+
+/*
+targetAttributes.put("Artifact.Target.Refactoring.AverageWeight",0.1);
+targetAttributes.put("Artifact.Target.Fix.AverageWeight",0.1);
+targetAttributes.put("Artifact.Target.IssueCount.AverageWeight",0.1);
+targetAttributes.put("Artifact.Target.IssueReference.AverageWeight",0.1);
+targetAttributes.put("Artifact.Target.UsersPerIssue.AverageWeight",0.1);
+targetAttributes.put("Artifact.Target.CommentsPerIssue.AverageWeight",0.1);
+
+targetAttributes.put("Artifact.Target.BugFix.Shared.AverageWeight",0.1);
+targetAttributes.put("Artifact.Target.Refactoring.Shared.AverageWeight",0.1);
+targetAttributes.put("Artifact.Target.Fix.Shared.AverageWeight",0.1);
+targetAttributes.put("Artifact.Target.IssueReference.Shared.AverageWeight",0.1);
+
+targetAttributes.put("Artifact.Target.BugFix.Churn.AverageWeight",0.1);
+targetAttributes.put("Artifact.Target.Refactoring.Churn.AverageWeight",0.1);
+targetAttributes.put("Artifact.Target.Fix.Churn.AverageWeight",0.1);
+targetAttributes.put("Artifact.Target.IssueReference.Churn.AverageWeight",0.1);
+
+targetAttributes.put("Artifact.Target.BugFix.Size.AverageWeight",0.1);
+targetAttributes.put("Artifact.Target.Refactoring.Size.AverageWeight",0.1);
+targetAttributes.put("Artifact.Target.Fix.Size.AverageWeight",0.1);
+targetAttributes.put("Artifact.Target.IssueReference.Size.AverageWeight",0.1);
+*/
+
+//does not work with binary resources
+//var modelFile = new Native("java.io.File") (ARFFx.getModelFile());
+var modelFile = new Native("java.io.File") (ARFFx.getModelImpl().getURI().toString().replaceAll("^file:",""));
+
+//var CONFIDENCE = "CONFIDENCE".getARFFAttribute();
+//var LABEL = "LABEL".getARFFAttribute();
+
+//TODO: move to common
+//TODO: remove once established at earlier steps
+var nestedAnonymousClassFilter = "\"[\\w]+\\$[\\d]+.*\"";
+
+var threshold = 100;
+
+//ARTIFACTS
+for (arffx in ARFFx!Model.allInstances().select(x|x.data.size() > 0)) {
+ arffx.checkTargetAttributes(targetAttributes);
+ arffx.checkForCompleteness();
+
+ //these will be recalculated for the bags..
+ arffx.setConfidenceThresholds(targetAttributes);
+ arffx.assignClassAndConfidence(targetAttributes);
+
+}
+
+
+var end = Native("java.lang.System").currentTimeMillis();
+var duration = end - start;
+("Duration: "+duration.toMinutes().round(5)).log(1);
+
+
+operation ARFFx!Model checkTargetAttributes(targetAttributes : Map) {
+ var notFound = targetAttributes.keySet().select(x|not self.attributes.exists(a|a.name = x));
+ for (a in notFound) {
+ targetAttributes.remove(a);
+ }
+}
+
+//slow?
+operation ARFFx!Model assignClassAndConfidence(targetAttributes : Map) {
+ for (baseAttribute in targetAttributes.keySet()) {
+ for (i in self.data) {
+ i.assignClassAndConfidence(baseAttribute, targetAttributes.get(baseAttribute));
+ }
+ }
+}
+
+operation ARFFx!Model setConfidenceThresholds(targetAttributes : Map) {
+ //self.name.println();
+ //TODO: store as meta data
+ //TODO: store mean divisor as meta-data
+ for (a in targetAttributes.keySet) {
+ //(" "+a +" -> "+ targetAttributes.get(a)).println();
+ var v = self.data.collect(x|x.getValue(a.getARFFAttribute(self)).asDouble());
+ //v = v.normalizeMinMax(0.asDouble(), 1.asDouble());
+ //(" "+v.getMin()+" : "+v.getMax()+" : "+v.getMean()+" : "+v.getVariance()+" : "+v.getStandardDeviation()).println();
+ var t = (v.getMean()/2).round(4);
+ //(" "+a +" -> "+ t).println();
+ targetAttributes.put(a,t);
+ //(" Non-zero:\t\t"+v.select(x|x <> 0).size()).println();
+ //(" Above threshold:\t"+v.select(x|x > t).size()).println();
+ }
+ //" updated".println();
+ //(" "+targetAttributes).println();
+}
+
+operation Collection setConfidenceThresholds(targetAttributes : Map) {
+ //TODO: store as meta data
+ //TODO: store mean divisor as meta-data
+ for (a in targetAttributes.keySet) {
+ var v = new Sequence();
+ for (arffx in self) {
+ v.addAll(arffx.data.collect(x|x.getValue(a.getARFFAttribute(arffx)).asDouble()));
+ }
+ //v = v.normalizeMinMax(0.asDouble(), 1.asDouble());
+ //(" "+v.getMin()+" : "+v.getMax()+" : "+v.getMean()+" : "+v.getVariance()+" : "+v.getStandardDeviation()).println();
+ var t = (v.getMean()/2).round(4);
+ targetAttributes.put(a,t);
+ //(" Non-zero:\t\t"+v.select(x|x <> 0).size()).println();
+ //(" Above threshold:\t"+v.select(x|x > t).size()).println();
+ }
+}
+
+
+operation ARFFx!Model checkForCompleteness() {
+ var line = 1;
+ var NameAttribute = "Artifact.Name".getARFFAttribute(self);
+ for (i in self.data.select(x|not x.getValue(NameAttribute).matches(nestedAnonymousClassFilter))) {
+ //for (i in self.data) {
+ line = line+1;
+ if (i.values.size() <> self.attributes.size()) {
+ (self.name+" : Line "+line+" : Value and attribute counts do not match : " + i.values.size() +" vs "+ self.attributes.size()).log(1);
+ (i.getValues(s).substring(1)).log(1);
+ i.printMissingAttributes();
+ }
+ }
+}
+
+operation ARFFx!Instance printMissingAttributes() {
+ for (a in self.eContainer.attributes) {
+ if (not self.values.exists(v|v.ofAttribute = a)) {
+ (" Missing attribute: "+a.name).log(1);
+ }
+ }
+}
+
+
+operation ARFFx!Instance assignClassAndConfidence(baseAttribute : String, threshold : Real) : OrderedSet {
+ var confidenceAttribute = "CONFIDENCE."+baseAttribute;
+ var labelAttribute = "LABEL."+baseAttribute;
+
+ //TODO: add attributes to filter
+ var base = self.getValue(baseAttribute.getARFFAttribute(self.eContainer()));
+ var label = "false";
+ var confidence = "high";
+ if (base.asDouble() > threshold) {
+ label = "true";
+ }
+ //TODO: also export as parameters
+ if (base.asDouble() < 1.01*threshold and base.asDouble() > 0.09*threshold) {
+ confidence = "low";
+ }
+ self.updateValue(confidence, confidenceAttribute);
+ self.updateValue(label, labelAttribute);
+}
+
+operation ARFFx!Instance getValues(s : String) : String {
+ var line = "";
+ for (v in self.values) {
+ line = line + s + v.content;
+ //line = line + s + v.ofAttribute.name+"="+v.content;
+ }
+ return line.replace("NaN","0.0").substring(0); //Substring? why?
+}
Index: trunk/CrossPare/decent/epsilon/query/preprocess.eol
===================================================================
--- trunk/CrossPare/decent/epsilon/query/preprocess.eol (revision 32)
+++ trunk/CrossPare/decent/epsilon/query/preprocess.eol (revision 32)
@@ -0,0 +1,178 @@
+import "../libraries/decent/common.eol";
+import "../libraries/decent/logging.eol";
+import "../libraries/arff/common.eol";
+
+var selectedType = Native("java.lang.System").getProperty("epsilon.transformation.decent2arffx.type");
+var skipSource = Native("java.lang.System").getProperty("epsilon.transformation.decent2arffx.skipSource").asBoolean();
+
+var Type = DECENT!ArtifactType.allInstances().select(a|a.name = selectedType).first();
+if (Type.isUndefined()) {
+ //handle wrong arguments here ?
+}
+
+var types = new OrderedSet();
+types.add("code");
+//types.add("Class");
+//types.add("Method");
+//types.add("Function");
+//types.add("Module");
+
+//TODO: move to common, use for tagging states?
+var nestedAnonymousClassFilter = "\"[\\w]+\\$[\\d]+.*\"";
+
+var Model = DECENT!Model.allInstances().first();
+
+var TouchCountAttribute = addAttribute("TouchCount", "Number of touches to an artifact");
+var ArtifactCountAttribute = addAttribute("ArtifactCount", "Number of known artifacts of the same type");
+var AuthorCountAttribute = addAttribute("AuthorCount", "Number of known authors for an artifact up to the current state");
+var GlobalAuthorCountAttribute = addAttribute("GlobalAuthorCount", "Number of known authors for all artifacts up to the current state");
+var FileCountAttribute = addAttribute("FileCount", "Number of existing artifacts up to the current state");
+
+"Running preprocess".log(1);
+var start = Native("java.lang.System").currentTimeMillis();
+
+for (type in types) {
+ ("Processing type "+type).log(1);
+ preprocessArtifacts(type);
+}
+updateDeltas();
+
+
+var end = Native("java.lang.System").currentTimeMillis();
+var duration = end - start;
+("Duration: "+duration.toMinutes().round(5)).log(1);
+
+
+operation updateDeltas() {
+ for (a in DECENT!Artifact.allInstances()) {
+ (""+a.name).log(3);
+ for (s in a.states.select(x|x.isHit())) {
+ for (v in s.values.select(v|v.isTypeOf(DoubleValue) or v.isTypeOf(IntegerValue))) {
+ ("["+a.type.name+"]"+a.name+"@"+s.ID+"::"+v.ofAttribute.name).log(4);
+ s.fromActivity.get(0).addDelta(v.ofAttribute);
+ }
+ }
+ }
+}
+
+operation preprocessArtifacts(typeName : String) {
+ var type = DECENT!ArtifactType.allInstances().select(t|t.name = typeName).first();
+ if (type.isUndefined()) {
+ //TODO: check that type is set
+ //handle wrong arguments here ?
+ return;
+ }
+
+ var artifacts = Artifact.allInstances().select(x|x.type = type and not x.name.matches(nestedAnonymousClassFilter)).sortBy(x|x.name);
+
+ var artifactDevelopers = new Set();
+
+
+ for (a in artifacts) {
+ var sCount = 0;
+ for (s in a.states.select(x|x.isHit()).sortBy(x|x.ID)) {
+ ("Processing ["+a.type.name+"]"+a.name+"@"+s.ID).log(4);
+ sCount = sCount+1;
+ s.addValue(TouchCountAttribute, sCount);
+
+
+
+
+
+ // Count artifacts and put existing ones in filecount value
+ var existingArtifacts = artifacts.select(x|x.states.select(y|y.fromActivity.first().date.time <= s.fromActivity.first().date.time).size() > 0);
+ var fileCount = existingArtifacts.size();
+ s.addValue(FileCountAttribute, fileCount);
+
+ //this refers more to the state of the project at a given point in time
+ //thus the information will be identical in all states of artifacts with the same state.ID
+ //perhaps it should be calculated during the export rather than stored in the model
+ //(note however that it is still artifact type related)
+ var aCount = artifacts.select(x|x.states.select(y|y.fromActivity.first().date <= s.fromActivity.first().date).size() > 0).size();
+ //(a.name+"@"+s.ID+ " -> Number of known artifacts of type "+a.type.name+": "+aCount).println();
+ s.addValue(ArtifactCountAttribute, aCount);
+
+ //author count, could also be calculated from all the activities for an artifact up to a point
+ artifactDevelopers.add(s.fromActivity.first().agent);
+ var dCount = artifactDevelopers.size();
+ s.addValue(AuthorCountAttribute, dCount);
+
+ //number of all known developers up to the point
+ //NOT restricted to the type of interest!!!
+ DECENT!Agent.allInstances().select(x|x.activities.select(y|y.date <= s.fromActivity.first().date).size() > 0).size();
+ var gdCount = artifactDevelopers.size();
+ s.addValue(GlobalAuthorCountAttribute, gdCount);
+ }
+ artifactDevelopers.clear();
+ }
+
+ ("Processing to arffx ...").log(2);
+
+ var arffx = new ARFFx!Model();
+ arffx.name = Model.name+"-artifacts-"+type.name;
+ arffx.meta.put("DECENT.ArtifactType", type.name);
+ arffx.meta.put("ARFFx.Type", "artifact");
+
+ for (a in artifacts) {
+ (" Artifact " +a.name).log(3);
+
+ //we exclude first and last states!
+ //careful with the filtering here...
+ // for (s in a.states.excluding(a.states.first()).excluding(a.states.last()).select(x|x.isHit() and x.next.first().isHit()).sortBy(x|x.ID)) {
+ for (s in a.states.excluding(a.states.first()).select(x|x.isHit()).sortBy(x|x.ID)) {
+ s.processTargetState(arffx);
+ }
+ }
+
+ //on transformation to Weka instances - either transform to ARFFx and then use Java EMF API on ARFFx
+ //or use Java EMF API on DECENT
+}
+
+
+operation DECENT!State processTargetState (arffx : ARFFx!Model) {
+ var p = self.previous.first();
+ while (p.isDefined() and not p.isHit()) {
+ //TODO: what if more than one?
+ p = p.previous.first();
+ }
+ if (p.isDefined() or skipSource){
+
+ var i = new ARFFx!Instance();
+ arffx.data.add(i);
+
+ i.addValue('\"'+self.artifact.name.asString()+'\"', "Artifact.Name");
+ i.addValue(self.ID.asString(), "Artifact.Target.StateID");
+ i.addValue('\"'+self.fromActivity.first().agent.name+'\"', "Agent.Name");
+
+ for (v in self.values.select(x|x.isTypeOf(DECENT!DoubleValue) or x.isTypeOf(DECENT!IntegerValue))) {
+ i.addValue(v.content.asString(), "Artifact.Target."+v.name);
+ }
+
+ var ax = self.fromActivity.first();
+
+ //TODO: add activity meta-data
+ for (v in ax.values.select(x|x.isTypeOf(DECENT!DoubleValue) or x.isTypeOf(DECENT!IntegerValue))) {
+ i.addValue(v.content.asString(), "Activity."+v.name);
+ }
+
+ for (v in ax.deltas) {
+ i.addValue(v.change.asString(), "Artifact.Delta."+v.onAttribute.name);
+ }
+
+ //TODO: these are notably missing in several places!
+ if (ax.agentState.isDefined()) {
+ for (v in ax.agentState.values.select(x|x.isTypeOf(DECENT!DoubleValue) or x.isTypeOf(DECENT!IntegerValue))) {
+ //i.addValue(v.content.asString(), "Agent."+v.name);
+ }
+ }
+
+ if (not skipSource) {
+ i.addValue(p.ID.asString(), "Artifact.Source.StateID");
+
+ for (v in p.values.select(x|x.isTypeOf(DECENT!DoubleValue) or x.isTypeOf(DECENT!IntegerValue))) {
+ i.addValue(v.content.asString(), "Artifact.Source."+v.name);
+ }
+ }
+ } else {
+ }
+}
Index: trunk/CrossPare/decent/models/ARFFx.ecore
===================================================================
--- trunk/CrossPare/decent/models/ARFFx.ecore (revision 32)
+++ trunk/CrossPare/decent/models/ARFFx.ecore (revision 32)
@@ -0,0 +1,43 @@
+
+
+
+
- * - load the data from the provided data path
- * - filter the data sets according to the provided version filters
- * - execute the following steps for each data sets as test data that is not ignored through the test version filter:
- *
- * - filter the data sets to setup the candidate training data:
- *
- * - remove all data sets from the same project
- * - filter all data sets according to the training data filter
- *
- * - apply the setwise preprocessors
- * - apply the setwise data selection algorithms
- * - apply the setwise postprocessors
- * - train the setwise training classifiers
- * - unify all remaining training data into one data set
- * - apply the preprocessors
- * - apply the pointwise data selection algorithms
- * - apply the postprocessors
- * - train the normal classifiers
- * - evaluate the results for all trained classifiers on the training data
- *
- *
- *
- * Note that this class implements {@link Runnable}, i.e., each experiment can be started in its own thread.
- * @author Steffen Herbold
- */
-public class Experiment implements Runnable {
-
- /**
- * configuration of the experiment
- */
- private final ExperimentConfiguration config;
-
- /**
- * Constructor. Creates a new experiment based on a configuration.
- * @param config configuration of the experiment
- */
- public Experiment(ExperimentConfiguration config) {
- this.config = config;
- }
-
- /**
- * Executes the experiment with the steps as described in the class comment.
- * @see Runnable#run()
- */
- @Override
- public void run() {
- final List versions = new LinkedList<>();
-
- for(IVersionLoader loader : config.getLoaders()) {
- versions.addAll(loader.load());
- }
-
- for( IVersionFilter filter : config.getVersionFilters() ) {
- filter.apply(versions);
- }
- boolean writeHeader = true;
- int versionCount = 1;
- int testVersionCount = 0;
-
- for( SoftwareVersion testVersion : versions ) {
- if( isVersion(testVersion, config.getTestVersionFilters()) ) {
- testVersionCount++;
- }
- }
-
- // sort versions
- Collections.sort(versions);
-
- for( SoftwareVersion testVersion : versions ) {
- if( isVersion(testVersion, config.getTestVersionFilters()) ) {
- Console.traceln(Level.INFO, String.format("[%s] [%02d/%02d] %s: starting", config.getExperimentName(), versionCount, testVersionCount, testVersion.getVersion()));
-
- // Setup testdata and training data
- Instances testdata = testVersion.getInstances();
- String testProject = testVersion.getProject();
- SetUniqueList traindataSet = SetUniqueList.setUniqueList(new LinkedList());
- for( SoftwareVersion trainingVersion : versions ) {
- if( isVersion(trainingVersion, config.getTrainingVersionFilters()) ) {
- if( trainingVersion!=testVersion ) {
- if( !trainingVersion.getProject().equals(testProject) ) {
- traindataSet.add(trainingVersion.getInstances());
- }
- }
- }
- }
-
- for( ISetWiseProcessingStrategy processor : config.getSetWisePreprocessors() ) {
- Console.traceln(Level.FINE, String.format("[%s] [%02d/%02d] %s: applying setwise preprocessor %s", config.getExperimentName(), versionCount, testVersionCount, testVersion.getVersion(), processor.getClass().getName()));
- processor.apply(testdata, traindataSet);
- }
- for( ISetWiseDataselectionStrategy dataselector : config.getSetWiseSelectors() ) {
- Console.traceln(Level.FINE, String.format("[%s] [%02d/%02d] %s: applying setwise selection %s", config.getExperimentName(), versionCount, testVersionCount, testVersion.getVersion(), dataselector.getClass().getName()));
- dataselector.apply(testdata, traindataSet);
- }
- for( ISetWiseProcessingStrategy processor : config.getSetWisePostprocessors() ) {
- Console.traceln(Level.FINE, String.format("[%s] [%02d/%02d] %s: applying setwise postprocessor %s", config.getExperimentName(), versionCount, testVersionCount, testVersion.getVersion(), processor.getClass().getName()));
- processor.apply(testdata, traindataSet);
- }
- for( ISetWiseTrainingStrategy setwiseTrainer : config.getSetWiseTrainers() ) {
- Console.traceln(Level.FINE, String.format("[%s] [%02d/%02d] %s: applying setwise trainer %s", config.getExperimentName(), versionCount, testVersionCount, testVersion.getVersion(), setwiseTrainer.getName()));
- setwiseTrainer.apply(traindataSet);
- }
- Instances traindata = makeSingleTrainingSet(traindataSet);
- for( IProcessesingStrategy processor : config.getPreProcessors() ) {
- Console.traceln(Level.FINE, String.format("[%s] [%02d/%02d] %s: applying preprocessor %s", config.getExperimentName(), versionCount, testVersionCount, testVersion.getVersion(), processor.getClass().getName()));
- processor.apply(testdata, traindata);
- }
- for( IPointWiseDataselectionStrategy dataselector : config.getPointWiseSelectors() ) {
- Console.traceln(Level.FINE, String.format("[%s] [%02d/%02d] %s: applying pointwise selection %s", config.getExperimentName(), versionCount, testVersionCount, testVersion.getVersion(), dataselector.getClass().getName()));
- traindata = dataselector.apply(testdata, traindata);
- }
- for( IProcessesingStrategy processor : config.getPostProcessors() ) {
- Console.traceln(Level.FINE, String.format("[%s] [%02d/%02d] %s: applying setwise postprocessor %s", config.getExperimentName(), versionCount, testVersionCount, testVersion.getVersion(), processor.getClass().getName()));
- processor.apply(testdata, traindata);
- }
- for( ITrainingStrategy trainer : config.getTrainers() ) {
- Console.traceln(Level.FINE, String.format("[%s] [%02d/%02d] %s: applying trainer %s", config.getExperimentName(), versionCount, testVersionCount, testVersion.getVersion(), trainer.getName()));
- trainer.apply(traindata);
- }
- File resultsDir = new File(config.getResultsPath());
- if (!resultsDir.exists()) {
- resultsDir.mkdir();
- }
- for( IEvaluationStrategy evaluator : config.getEvaluators() ) {
- Console.traceln(Level.FINE, String.format("[%s] [%02d/%02d] %s: applying evaluator %s", config.getExperimentName(), versionCount, testVersionCount, testVersion.getVersion(), evaluator.getClass().getName()));
- List allTrainers = new LinkedList<>();
- for( ISetWiseTrainingStrategy setwiseTrainer : config.getSetWiseTrainers() ) {
- allTrainers.add(setwiseTrainer);
- }
- for( ITrainingStrategy trainer : config.getTrainers() ) {
- allTrainers.add(trainer);
- }
- if( writeHeader ) {
- evaluator.setParameter(config.getResultsPath() + "/" + config.getExperimentName() + ".csv");
- }
- evaluator.apply(testdata, traindata, allTrainers, writeHeader);
- writeHeader = false;
- }
- Console.traceln(Level.INFO, String.format("[%s] [%02d/%02d] %s: finished", config.getExperimentName(), versionCount, testVersionCount, testVersion.getVersion()));
- versionCount++;
- }
- }
- }
-
- /**
- * Helper method that checks if a version passes all filters.
- * @param version version that is checked
- * @param filters list of the filters
- * @return true, if the version passes all filters, false otherwise
- */
- private boolean isVersion(SoftwareVersion version, List filters) {
- boolean result = true;
- for( IVersionFilter filter : filters) {
- result &= !filter.apply(version);
- }
- return result;
- }
-
- /**
- * Helper method that combines a set of Weka {@link Instances} sets into a single {@link Instances} set.
- * @param traindataSet set of {@link Instances} to be combines
- * @return single {@link Instances} set
- */
- public static Instances makeSingleTrainingSet(SetUniqueList traindataSet) {
- Instances traindataFull = null;
- for( Instances traindata : traindataSet) {
- if( traindataFull==null ) {
- traindataFull = new Instances(traindata);
- } else {
- for( int i=0 ; i evaluators;
+
+ /**
+ * indicates, if the classifier should be saved
+ */
+ private Boolean saveClassifier = null;
+
+ /**
+ * indicates, which execution strategy to choose
+ * (e.g. CrossProjectExperiment, ClassifierCreationExecution).
+ * Default is CrossProjectExperiment.
+ */
+ private String executionStrategy = "CrossProjectExperiment";
/**
@@ -315,4 +327,20 @@
}
+ /**
+ * returns boolean, if classifier should be saved
+ * @return boolean
+ */
+ public boolean getSaveClassifier() {
+ return saveClassifier;
+ }
+
+ /**
+ * returns the execution strategy
+ * @return String execution strategy
+ */
+ public String getExecutionStrategy() {
+ return executionStrategy;
+ }
+
/* (non-Javadoc)
* @see org.xml.sax.helpers.DefaultHandler#startElement(java.lang.String, java.lang.String, java.lang.String, org.xml.sax.Attributes)
@@ -393,4 +421,10 @@
final IEvaluationStrategy evaluator = (IEvaluationStrategy) Class.forName("de.ugoe.cs.cpdp.eval." + attributes.getValue("name")).newInstance();
evaluators.add(evaluator);
+ }
+ else if( qName.equals("saveClassifier")) {
+ saveClassifier = true;
+ }
+ else if( qName.equals("executionStrategy")) {
+ executionStrategy = attributes.getValue("name");
}
else if( qName.equals("partialconfig") ) {
@@ -423,6 +457,7 @@
* If the current data path is the empty string (""), it is override by the datapath of the other configuration. Otherwise, the current data path is kept.
* @param other experiment whose information is added
- */
- private void addConfigurations(ExperimentConfiguration other) {
+ * @throws ExperimentConfigurationException
+ */
+ private void addConfigurations(ExperimentConfiguration other) throws ExperimentConfigurationException {
if( "results".equals(resultsPath) ) {
resultsPath = other.resultsPath;
@@ -441,4 +476,16 @@
trainers.addAll(other.trainers);
evaluators.addAll(other.evaluators);
+
+ if(!executionStrategy.equals(other.executionStrategy)) {
+ throw new ExperimentConfigurationException("Executionstrategies must be the same, if config files should be added.");
+ }
+
+ /* Only if saveClassifier is not set in the main config and
+ * the other configs saveClassifier is true, it must be set.
+ */
+ if(saveClassifier == null && other.saveClassifier == true) {
+ saveClassifier = other.saveClassifier;
+ }
+
}
@@ -464,5 +511,7 @@
builder.append("Pointwise trainers: " + trainers.toString() + StringTools.ENDLINE);
builder.append("Evaluators: " + evaluators.toString() + StringTools.ENDLINE);
-
+ builder.append("Save Classifier?: " + saveClassifier + StringTools.ENDLINE);
+ builder.append("Execution Strategy: " + executionStrategy + StringTools.ENDLINE);
+
return builder.toString();
}
Index: trunk/CrossPare/src/de/ugoe/cs/cpdp/Runner.java
===================================================================
--- trunk/CrossPare/src/de/ugoe/cs/cpdp/Runner.java (revision 31)
+++ trunk/CrossPare/src/de/ugoe/cs/cpdp/Runner.java (revision 32)
@@ -2,4 +2,6 @@
import java.io.File;
+import java.lang.reflect.Constructor;
+import java.lang.reflect.InvocationTargetException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
@@ -7,4 +9,5 @@
import java.util.logging.Level;
+import de.ugoe.cs.cpdp.execution.IExecutionStrategy;
import de.ugoe.cs.util.console.Console;
import de.ugoe.cs.util.console.TextConsole;
@@ -23,5 +26,4 @@
public static void main(String[] args) {
new TextConsole(Level.FINE);
-
final int concurrentThreads = Runtime.getRuntime().availableProcessors();
final ExecutorService threadPool = Executors.newFixedThreadPool(concurrentThreads);
@@ -47,4 +49,9 @@
}
+ /**
+ * Creates the config and starts the corresponding experiment
+ * @param threadPool
+ * @param configFile location of the config file
+ */
public static void createConfig(ExecutorService threadPool, String configFile) {
ExperimentConfiguration config = null;
@@ -55,9 +62,42 @@
e.printStackTrace();
}
+
if( config!=null ) {
Console.trace(Level.FINE, config.toString());
- Experiment experiment = new Experiment(config);
- threadPool.execute(experiment);
+ // Instantiate the class like it was given as parameter in the config file and cast it to the interface
+ try {
+ // Because we need to pass a parameter, a normal new Instance call is not possible
+ Class executionStrategyClass = Class.forName("de.ugoe.cs.cpdp.execution."+config.getExecutionStrategy());
+ Constructor executionStrategyConstructor =
+ executionStrategyClass.getConstructor(ExperimentConfiguration.class);
+
+ IExecutionStrategy experiment = (IExecutionStrategy) executionStrategyConstructor.newInstance(config);
+ threadPool.execute(experiment);
+ } catch (NoSuchMethodException e) {
+ Console.printerrln("Class \"" + config.getExecutionStrategy()+ "\" does not have the right Constructor");
+ e.printStackTrace();
+ } catch (SecurityException e) {
+ Console.printerrln("Security manager prevents reflection");
+ e.printStackTrace();
+ } catch (IllegalArgumentException e) {
+ Console.printerrln("Class \"" + config.getExecutionStrategy()+ "\" does not have a Constructor, which"
+ + "matches the given arguments");
+ e.printStackTrace();
+ } catch (InvocationTargetException e) {
+ Console.printerrln("Constructor in Class \"" + config.getExecutionStrategy()+ "\" is not public");
+ e.printStackTrace();
+ } catch (InstantiationException e) {
+ Console.printerrln("Cannot instantiate Class \"" + config.getExecutionStrategy()+"\"");
+ e.printStackTrace();
+ } catch (IllegalAccessException e) {
+ Console.printerrln("Cannot access Class \"" + config.getExecutionStrategy()+"\"");
+ e.printStackTrace();
+ } catch (ClassNotFoundException e) {
+ Console.printerrln("Class \"" + config.getExecutionStrategy()+ "\" was not found");
+ e.printStackTrace();
+ }
+
}
+
}
}
Index: trunk/CrossPare/src/de/ugoe/cs/cpdp/dataprocessing/AttributeNonRemoval.java
===================================================================
--- trunk/CrossPare/src/de/ugoe/cs/cpdp/dataprocessing/AttributeNonRemoval.java (revision 32)
+++ trunk/CrossPare/src/de/ugoe/cs/cpdp/dataprocessing/AttributeNonRemoval.java (revision 32)
@@ -0,0 +1,65 @@
+package de.ugoe.cs.cpdp.dataprocessing;
+
+import java.util.ArrayList;
+
+import org.apache.commons.collections4.list.SetUniqueList;
+
+import weka.core.Instances;
+
+/**
+ * Removes attributes from all data sets, except the one defined, using their name.
+ * @author Fabian Trautsch
+ */
+public class AttributeNonRemoval implements ISetWiseProcessingStrategy, IProcessesingStrategy {
+
+ /**
+ * names of the attributes to be kept (determined by {@link #setParameter(String)})
+ */
+ private ArrayList attributeNames = new ArrayList();
+
+ /**
+ * Sets that attributes that will be kept. The string contains the blank-separated names of the attributes to be kept.
+ *
+ * Note, that keeping of attributes with blanks is currently not supported!
+ * @param parameters string with the blank-separated attribute names
+ */
+ @Override
+ public void setParameter(String parameters) {
+ if( parameters!=null ) {
+ String[] attributeNamesArray = parameters.split(" ");
+ for(String attributeName : attributeNamesArray) {
+ attributeNames.add(attributeName);
+ }
+ }
+ }
+
+ /**
+ * @see de.ugoe.cs.cpdp.dataprocessing.SetWiseProcessingStrategy#apply(weka.core.Instances, org.apache.commons.collections4.list.SetUniqueList)
+ */
+ @Override
+ public void apply(Instances testdata, SetUniqueList traindataSet) {
+ for( String attributeName : attributeNames ) {
+ for( int i=0 ; i=0; i--) {
+ if(!attributeNames.contains(testdata.attribute(i).name())) {
+ testdata.deleteAttributeAt(i);
+ traindata.deleteAttributeAt(i);
+ }
+ }
+ }
+}
Index: trunk/CrossPare/src/de/ugoe/cs/cpdp/dataprocessing/NominalAttributeFilter.java
===================================================================
--- trunk/CrossPare/src/de/ugoe/cs/cpdp/dataprocessing/NominalAttributeFilter.java (revision 32)
+++ trunk/CrossPare/src/de/ugoe/cs/cpdp/dataprocessing/NominalAttributeFilter.java (revision 32)
@@ -0,0 +1,88 @@
+package de.ugoe.cs.cpdp.dataprocessing;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+
+import weka.core.Attribute;
+import weka.core.Instance;
+import weka.core.Instances;
+
+/**
+ * Filters the given dataset for an nominal attribute.
+ * Every instance that has a value of the defined values of the given nominal attribute is removed.
+ *
+ *
+ * (e.g. param="CONFIDECNE low middle"; all instances where the "CONFIDENCE" attribute
+ * value is "low" or "middle" are removed from the dataset)
+ */
+
+public class NominalAttributeFilter implements IProcessesingStrategy{
+
+ private String nominalAttributeName = "";
+ private String[] nominalAttributeValues = new String[]{};
+
+ /**
+ * Sets the nominal attribute name (first parameter) and the nominal attribute values (other
+ * parameters), which should be removed from the dataset.
+ *
+ * @param parameters string with the blank-separated parameters (first parameter
+ * is the name of the nominal attribute, everything else are the values)
+ */
+ @Override
+ public void setParameter(String parameters) {
+ if( parameters!=null ) {
+ String[] parameter = parameters.split(" ");
+ nominalAttributeName = parameter[0];
+ nominalAttributeValues = Arrays.copyOfRange(parameter, 1, parameter.length);
+ }
+ }
+
+ /* (non-Javadoc)
+ * @see de.ugoe.cs.cpdp.dataprocessing.IProcessesingStrategy#apply(weka.core.Instances, weka.core.Instances)
+ */
+ @Override
+ public void apply(Instances testdata, Instances traindata) {
+ int indexOfConfidenceAttribute = -1;
+
+ // Find index of the named confidence attribute to filter for
+ for(int i=0; i nominalValuesOfConfidenceAttribute = Collections.list(confidenceAttribute.enumerateValues());
+ ArrayList indexOfnominalAttributeValues = new ArrayList();
+
+
+ for(int k=0; k=0; j--) {
+ Instance wekaInstance = traindata.get(j);
+
+ // delete all instances where nominal attribute has the value of one of the parameter
+ if(indexOfnominalAttributeValues.contains(wekaInstance.value(indexOfConfidenceAttribute))) {
+ traindata.delete(j);
+ }
+ }
+ }
+
+}
Index: trunk/CrossPare/src/de/ugoe/cs/cpdp/dataprocessing/SimulationFilter.java
===================================================================
--- trunk/CrossPare/src/de/ugoe/cs/cpdp/dataprocessing/SimulationFilter.java (revision 32)
+++ trunk/CrossPare/src/de/ugoe/cs/cpdp/dataprocessing/SimulationFilter.java (revision 32)
@@ -0,0 +1,121 @@
+package de.ugoe.cs.cpdp.dataprocessing;
+
+import java.util.ArrayList;
+
+import java.util.HashMap;
+
+import weka.core.Attribute;
+import weka.core.Instance;
+import weka.core.Instances;
+
+/**
+ * Filter for the Repast Simulation of Software Projects.
+ *
+ * Filters the training dataset in the following way: If 0 is no bug
+ * and 1 means there is a bug in this artifact, then this filter
+ * filters the dataset in this way:
+ *
+ * 10010111000101110101111011101
+ * x--x-x-----x-x---x-x----x---x
+ *
+ * The instances, which are marked with x in this graphic are included
+ * in the newly created dataset and form the trainingsdataset.
+ *
+ * @author Fabian Trautsch
+ *
+ */
+
+public class SimulationFilter implements IProcessesingStrategy{
+
+ /**
+ * Does not have parameters. String is ignored.
+ * @param parameters ignored
+ */
+ @Override
+ public void setParameter(String parameters) {
+ // dummy
+
+ }
+
+
+ /*
+ * (non-Javadoc)
+ * @see de.ugoe.cs.cpdp.dataprocessing.IProcessesingStrategy#apply(weka.core.Instances, weka.core.Instances)
+ */
+ @Override
+ public void apply(Instances testdata, Instances traindata) {
+ Instances newDataSet = new Instances(traindata);
+ traindata.delete();
+
+ HashMap artifactNames = new HashMap();
+
+ // This is to add all data, where the first occurence of the file has a bug
+ ArrayList firstOccurenceArtifactNames = new ArrayList();
+
+ // Sort dataset (StateID is connected to the date of commit: Lower StateID
+ // means earlier commit than a higher stateID)
+ Attribute wekaAttribute = newDataSet.attribute("Artifact.Target.StateID");
+ newDataSet.sort(wekaAttribute);
+
+
+ /*
+ * Logical summary:
+ * If there is an instance that dont have a bug, put it into the hashmap (only unique values in there)
+ *
+ * If there is an instance, that hava a bug look up if it is in the hashmap already (this means:
+ * it does not had a bug before!): If this is true add it to a new dataset and remove it from
+ * the hashmap, so that new changes from "nonBug" -> "bug" for this file can be found.
+ *
+ * If the instance has a bug and is not in the hashmap (this means: The file has a bug with its
+ * first occurence or this file only has bugs and not an instance with no bug), then (if it is
+ * not in the arrayList above) add it to the new dataset. This way it is possible to get
+ * the first occurence of a file, which has a bug
+ *
+ */
+ for(int i=0; i artifactNamesCopy = new HashMap(artifactNames);
+
+
+ for(Double artifactName : artifactNames.keySet()) {
+
+ for(int i=0; i metaModelCache = new HashMap<>();
+ private boolean useDECENTBinary = false;
+ private boolean useARFFxBinary = false;
+
+ public static String metaPath = "./decent/models/";
+
+ /**
+ * Returns the decent model as IModel instance
+ *
+ * @param decentModelLocation location of the decent model file
+ * @param read indicates if the model should be read from
+ * @param write indicates if data should be written in the model
+ * @return EmFModel (IModel) instance from the decent model, which was loaded
+ * @throws Exception
+ */
+ public IModel getDECENTModel(String decentModelLocation, boolean read, boolean write) throws Exception {
+
+ EmfModel model;
+
+ if (isUseDECENTBinary()) {
+ unregisterMetaModels("");
+ if (!read) {
+ new File(decentModelLocation).delete();
+ new File(decentModelLocation+"bin").delete();
+ }
+ DECENTResourceTool tool = new DECENTResourceTool();
+ if (new File(decentModelLocation).exists() && !new File(decentModelLocation+"bin").exists()) {
+ Resource resource = tool.loadResourceFromXMI(decentModelLocation,"decent", DECENTPackage.eINSTANCE);
+ tool.storeBinaryResourceContents(resource.getContents(), decentModelLocation+"bin", "decentbin");
+ }
+
+ Resource resourceBin = tool.loadResourceFromBinary(decentModelLocation+"bin","decentbin", DECENTPackage.eINSTANCE);
+ //alternative pattern
+// model = createInMemoryEmfModel("DECENT", resourceLocation, "../DECENT.Meta/model/DECENTv3.ecore", read, write, resourceBin, DECENTPackage.eINSTANCE);
+// restoreMetaModels();
+
+ //NOTE: Adding the package is essential as otherwise epsilon breaks
+ model = new InMemoryEmfModel("DECENT", resourceBin, DECENTPackage.eINSTANCE);
+ model.setStoredOnDisposal(write);
+ model.setReadOnLoad(read);
+ model.setCachingEnabled(true);
+ restoreMetaModels();
+ } else {
+ model = createEmfModel("DECENT", decentModelLocation, metaPath+"DECENTv3.ecore", read, write);
+ }
+
+ return model;
+ }
+
+ /**
+ * Converts the decent model to a binary form
+ *
+ * @param location of the decent model file
+ */
+ public void convertDECENTModelToBinary(String location) {
+ unregisterMetaModels("");
+ DECENTResourceTool tool = new DECENTResourceTool();
+ Resource resource = tool.loadResourceFromXMI(location+"/model.decent","decent", DECENTPackage.eINSTANCE);
+ tool.storeBinaryResourceContents(resource.getContents(), location+"/model.decent"+"bin", "decentbin");
+ restoreMetaModels();
+ }
+
+ /**
+ * Converts the decent model to a xmi form
+ *
+ * @param location of the decent model file
+ */
+
+ public void convertDECENTModelToXMI(String location) {
+ unregisterMetaModels("");
+ DECENTResourceTool tool = new DECENTResourceTool();
+ Resource resource = tool.loadResourceFromBinary(location+"/model.decentbin","decentbin", DECENTPackage.eINSTANCE);
+ restoreMetaModels();
+ tool.storeResourceContents(resource.getContents(), location+"/model.decent", "decent");
+ }
+
+ /**
+ * Returns the arffx model as IModel instance
+ *
+ * @param arffxModelLocation location of the arffx model file
+ * @param read indicates if the model should be read from
+ * @param write indicates if data should be written in the model
+ * @return EmFModel (IModel) instance from the arffx model, which was loaded
+ * @throws Exception
+ */
+
+ public IModel getARFFxModel(String arffxModelLocation, boolean read, boolean write) throws Exception {
+
+ EmfModel model;
+
+ if (isUseARFFxBinary()) {
+ unregisterMetaModels("");
+ if (!read) {
+ new File(arffxModelLocation).delete();
+ new File(arffxModelLocation+"bin").delete();
+ }
+ ARFFxResourceTool tool = new ARFFxResourceTool();
+ if (new File(arffxModelLocation).exists() && !new File(arffxModelLocation+"bin").exists()) {
+ Resource resource = tool.loadResourceFromXMI(arffxModelLocation,"arffx", ARFFxPackage.eINSTANCE);
+ tool.storeBinaryResourceContents(resource.getContents(), arffxModelLocation+"bin", "arffxbin");
+ }
+
+ Resource resourceBin = tool.loadResourceFromBinary(arffxModelLocation+"bin","arffxbin", ARFFxPackage.eINSTANCE);
+ //alternative pattern
+// model = createInMemoryEmfModel("DECENT", resourceLocation, "../DECENT.Meta/model/DECENTv3.ecore", read, write, resourceBin, DECENTPackage.eINSTANCE);
+// restoreMetaModels();
+
+ //NOTE: Adding the package is essential as otherwise epsilon breaks
+ model = new InMemoryEmfModel("ARFFx", resourceBin, ARFFxPackage.eINSTANCE);
+// model.getModelImpl().getURI().toFileString()
+ model.setStoredOnDisposal(write);
+ model.setReadOnLoad(read);
+ model.setCachingEnabled(true);
+ restoreMetaModels();
+ } else {
+ model = createEmfModel("ARFFx", arffxModelLocation, metaPath+"ARFFx.ecore", read, write);
+ }
+
+ return model;
+ }
+
+
+ /**
+ * Converts an arffx model to a binary version
+ *
+ * @param location of the arffx model
+ */
+ public void convertARFFxModelToBinary(String location) {
+ unregisterMetaModels("");
+ ARFFxResourceTool tool = new ARFFxResourceTool();
+ Resource resource = tool.loadResourceFromXMI(location+"/model.arffx","arffx", ARFFxPackage.eINSTANCE);
+ tool.storeBinaryResourceContents(resource.getContents(), location+"/model.arffx"+"bin", "arffxbin");
+ restoreMetaModels();
+ }
+
+ /**
+ * Converts an arffx model to xmi
+ *
+ * @param location of the arffx model
+ */
+
+ public void convertARFFxModelToXMI(String location) {
+ unregisterMetaModels("");
+ ARFFxResourceTool tool = new ARFFxResourceTool();
+ Resource resource = tool.loadResourceFromBinary(location+"/model.arffxbin","arffxbin", DECENTPackage.eINSTANCE);
+ restoreMetaModels();
+ tool.storeResourceContents(resource.getContents(), location+"/model.arffx", "arffx");
+ }
+
+
+ /**
+ * Returns the log model as IModel instance
+ *
+ * @param logModelLocation location of the log model file
+ * @param read indicates if the model should be read from
+ * @param write indicates if data should be written in the model
+ * @return EmFModel (IModel) instance from the log model, which was loaded
+ * @throws Exception
+ */
+
+ public IModel getLOGModel(String logModelLocation, boolean read, boolean write) throws Exception {
+ if (!new File(logModelLocation).exists()) {
+ read = false;
+ }
+ IModel model = createEmfModel("LOG", logModelLocation, metaPath +"LOG.ecore", read, write);
+ System.setProperty("epsilon.logFileAvailable", "true");
+ return model;
+ }
+
+ /**
+ * Creates an EMF Model
+ *
+ * @param name of the emf model
+ * @param model name of the model
+ * @param metamodel name of the metamodel
+ * @param readOnLoad indicates if the model should be read on load
+ * @param storeOnDisposal indicates if the model should be stored on disposal
+ * @return
+ * @throws EolModelLoadingException
+ * @throws URISyntaxException
+ */
+
+ @SuppressWarnings("deprecation")
+ protected EmfModel createEmfModel(String name, String model,
+ String metamodel, boolean readOnLoad, boolean storeOnDisposal)
+ throws EolModelLoadingException, URISyntaxException {
+ EmfModel emfModel = new EmfModel();
+ StringProperties properties = new StringProperties();
+ properties.put(EmfModel.PROPERTY_NAME, name);
+ properties.put(EmfModel.PROPERTY_ALIASES, name);
+ properties.put(EmfModel.PROPERTY_FILE_BASED_METAMODEL_URI,
+ "file:/" + getFile(metamodel).getAbsolutePath());
+ properties.put(EmfModel.PROPERTY_MODEL_URI,
+ "file:/" + getFile(model).getAbsolutePath());
+ properties.put(EmfModel.PROPERTY_IS_METAMODEL_FILE_BASED, "true");
+ properties.put(EmfModel.PROPERTY_READONLOAD, readOnLoad + "");
+ properties.put(EmfModel.PROPERTY_CACHED, "true");
+ properties.put(EmfModel.PROPERTY_STOREONDISPOSAL,
+ storeOnDisposal + "");
+ emfModel.load(properties, "");
+ //System.out.println(emfModel.allContents());
+ return emfModel;
+ }
+
+ /**
+ * Returns a new File instance from the given filename
+ *
+ * @param fileName of the file
+ * @return
+ * @throws URISyntaxException
+ */
+ public File getFile(String fileName) throws URISyntaxException {;
+ return new File(fileName);
+ }
+
+ /**
+ * Restores the metamodels, so that they are registered in the
+ * EPackage registry
+ */
+ private void restoreMetaModels() {
+ for (String key : metaModelCache .keySet()) {
+ EPackage.Registry.INSTANCE.put(key, metaModelCache.get(key));
+ };
+ }
+
+ /**
+ * Unregister the metamodels from the EPackage registry
+ *
+ * @param filter for filtering out certain instances
+ */
+ private void unregisterMetaModels(String filter) {
+ for (String key : EPackage.Registry.INSTANCE.keySet()) {
+ if (key.contains(filter)) {
+ metaModelCache.put(key, EPackage.Registry.INSTANCE.get(key));
+ }
+ };
+ for (String key : metaModelCache .keySet()) {
+ EPackage.Registry.INSTANCE.remove(key);
+ };
+ }
+
+ /**
+ * Returns true if decent binary model is used
+ * @return
+ */
+
+ public boolean isUseDECENTBinary() {
+ return useDECENTBinary;
+ }
+
+ /**
+ * Sets the boolean which indicates, if the decent binary
+ * model is used
+ * @param useDECENTBinary
+ */
+ public void setUseDECENTBinary(boolean useDECENTBinary) {
+ this.useDECENTBinary = useDECENTBinary;
+ }
+
+ /**
+ * Returns true if arffx binary model is used
+ * @return
+ */
+ public boolean isUseARFFxBinary() {
+ return useARFFxBinary;
+ }
+
+ /**
+ * Sets the boolean which indicates, if the arffx binary
+ * model is used
+ * @param useARFFxBinary
+ */
+
+ public void setUseARFFxBinary(boolean useARFFxBinary) {
+ this.useARFFxBinary = useARFFxBinary;
+ }
+
+
+}
Index: trunk/CrossPare/src/de/ugoe/cs/cpdp/decentApp/DECENTResourceTool.java
===================================================================
--- trunk/CrossPare/src/de/ugoe/cs/cpdp/decentApp/DECENTResourceTool.java (revision 32)
+++ trunk/CrossPare/src/de/ugoe/cs/cpdp/decentApp/DECENTResourceTool.java (revision 32)
@@ -0,0 +1,41 @@
+package de.ugoe.cs.cpdp.decentApp;
+
+import org.eclipse.emf.ecore.EValidator;
+import org.eclipse.emf.ecore.util.EObjectValidator;
+
+import DECENT.DECENTPackage;
+import DECENT.impl.DECENTPackageImpl;
+import DECENT.util.DECENTResourceFactoryImpl;
+
+/**
+ * Class for handling decent model files
+ *
+ * @author Philip Makedonski, Fabian Trautsch
+ *
+ */
+public class DECENTResourceTool extends ResourceTool {
+
+ /**
+ * Initializes the Tool Factory, from which the models can be loaded and
+ * inizializes the validator.
+ */
+ public DECENTResourceTool(){
+ super(DECENTResourceTool.class.getName());
+ DECENTPackageImpl.init();
+ this.resourceFactory = new DECENTResourceFactoryImpl();
+ initializeValidator();
+ }
+
+ /**
+ * Inizializes the model validator
+ */
+ @Override
+ protected void initializeValidator(){
+ super.initializeValidator();
+ EObjectValidator validator = new EObjectValidator();
+ EValidator.Registry.INSTANCE.put(DECENTPackage.eINSTANCE, validator);
+ }
+
+
+
+}
Index: trunk/CrossPare/src/de/ugoe/cs/cpdp/decentApp/FileWatcher.java
===================================================================
--- trunk/CrossPare/src/de/ugoe/cs/cpdp/decentApp/FileWatcher.java (revision 32)
+++ trunk/CrossPare/src/de/ugoe/cs/cpdp/decentApp/FileWatcher.java (revision 32)
@@ -0,0 +1,42 @@
+package de.ugoe.cs.cpdp.decentApp;
+
+import java.util.*;
+import java.io.*;
+
+/**
+ * Helper class for watching if a file was changed
+ *
+ * @author Philip Makedonski
+ *
+ */
+public abstract class FileWatcher extends TimerTask {
+ // Last timestamp
+ private long timeStamp;
+
+ // File to watch
+ private File file;
+
+ /**
+ * Constructor
+ * @param file
+ */
+ public FileWatcher(File file) {
+ this.file = file;
+ this.timeStamp = file.lastModified();
+ }
+
+ /**
+ * Watches a file and executes the onChange Method
+ * if a file is changed
+ */
+ public final void run() {
+ long timeStamp = file.lastModified();
+
+ if (this.timeStamp != timeStamp) {
+ this.timeStamp = timeStamp;
+ onChange(file);
+ }
+ }
+
+ protected abstract void onChange(File file);
+}
Index: trunk/CrossPare/src/de/ugoe/cs/cpdp/decentApp/ResourceTool.java
===================================================================
--- trunk/CrossPare/src/de/ugoe/cs/cpdp/decentApp/ResourceTool.java (revision 32)
+++ trunk/CrossPare/src/de/ugoe/cs/cpdp/decentApp/ResourceTool.java (revision 32)
@@ -0,0 +1,320 @@
+package de.ugoe.cs.cpdp.decentApp;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.eclipse.emf.common.util.BasicDiagnostic;
+import org.eclipse.emf.common.util.Diagnostic;
+import org.eclipse.emf.common.util.EList;
+import org.eclipse.emf.common.util.URI;
+import org.eclipse.emf.ecore.EObject;
+import org.eclipse.emf.ecore.EOperation;
+import org.eclipse.emf.ecore.EPackage;
+import org.eclipse.emf.ecore.EStructuralFeature;
+import org.eclipse.emf.ecore.EValidator;
+import org.eclipse.emf.ecore.resource.Resource;
+import org.eclipse.emf.ecore.resource.ResourceSet;
+import org.eclipse.emf.ecore.resource.impl.BinaryResourceImpl;
+import org.eclipse.emf.ecore.resource.impl.ResourceFactoryImpl;
+import org.eclipse.emf.ecore.resource.impl.ResourceSetImpl;
+import org.eclipse.emf.ecore.util.Diagnostician;
+import org.eclipse.emf.ecore.xmi.impl.XMIResourceFactoryImpl;
+import org.eclipse.emf.ecore.xmi.impl.XMIResourceImpl;
+import org.eclipse.ocl.common.OCLConstants;
+import org.eclipse.ocl.ecore.delegate.OCLInvocationDelegateFactory;
+import org.eclipse.ocl.ecore.delegate.OCLSettingDelegateFactory;
+import org.eclipse.ocl.ecore.delegate.OCLValidationDelegateFactory;
+
+/**
+ * Class for handling different EMF Ressources
+ *
+ * @author Philip Makedonski
+ *
+ */
+public class ResourceTool {
+
+ protected ResourceFactoryImpl resourceFactory = new XMIResourceFactoryImpl();
+
+ /**
+ * Constructor
+ * @param loggedClass
+ */
+ public ResourceTool(String loggedClass) {
+ System.setProperty("org.slf4j.simpleLogger.logFile","validation.log");
+ System.setProperty("org.slf4j.simpleLogger.logFile","System.out");
+ }
+
+ /**
+ * Initializes the validator
+ */
+ protected void initializeValidator() {
+ // OCL.initialize(null);
+ String oclDelegateURI = OCLConstants.OCL_DELEGATE_URI+"/Pivot";
+
+ EOperation.Internal.InvocationDelegate.Factory.Registry.INSTANCE.put(oclDelegateURI,
+ new OCLInvocationDelegateFactory(oclDelegateURI));
+ EStructuralFeature.Internal.SettingDelegate.Factory.Registry.INSTANCE.put(oclDelegateURI,
+ new OCLSettingDelegateFactory(oclDelegateURI));
+ EValidator.ValidationDelegate.Registry.INSTANCE.put(oclDelegateURI,
+ new OCLValidationDelegateFactory(oclDelegateURI));
+
+ // EStructuralFeature.Internal.SettingDelegate.Factory.Registry.INSTANCE.put(oclDelegateURI,
+ // new OCLSettingDelegateFactory.Global());
+ // QueryDelegate.Factory.Registry.INSTANCE.put(oclDelegateURI, new OCLQueryDelegateFactory.Global());
+
+ }
+
+ /**
+ * Validates the ressource
+ * @param resource to validate
+ */
+ public void validateResource(Resource resource) {
+ BasicDiagnostic diagnostics = new BasicDiagnostic();
+ boolean valid = true;
+ for (EObject eo : resource.getContents())
+ {
+ Map context = new HashMap();
+ boolean validationResult = Diagnostician.INSTANCE.validate(eo, diagnostics, context);
+ showDiagnostics(diagnostics, "");
+ valid &= validationResult;
+ }
+
+ if (!valid){
+ System.out.println("Problem with validation!");
+ }
+ }
+
+ /**
+ * Output method for showing diagnostics for different ressources
+ * @param diagnostics
+ * @param indent
+ */
+ protected void showDiagnostics(Diagnostic diagnostics, String indent) {
+ indent+=" ";
+ for (Diagnostic d : diagnostics.getChildren()){
+ System.out.println(indent+d.getSource());
+ System.out.println(indent+" "+d.getMessage());
+ showDiagnostics(d,indent);
+ }
+ }
+
+
+ /**
+ * Loads a ressource from XMI
+ * @param inputPath path to the xmi
+ * @param extension of the ressource to load
+ * @param p the given EPackage
+ * @return
+ */
+ //TODO: workarounds copied from respective methods without EPackage parameter
+ @SuppressWarnings({ "rawtypes", "unchecked" })
+ public Resource loadResourceFromXMI(String inputPath, String extension, EPackage p) {
+ Resource.Factory.Registry reg = Resource.Factory.Registry.INSTANCE;
+ Map m = reg.getExtensionToFactoryMap();
+ m.put(extension, resourceFactory);
+ ResourceSet resSetIn = new ResourceSetImpl();
+ //critical part
+ resSetIn.getPackageRegistry().put(p.getNsURI(), p);
+
+ Resource inputResource = resSetIn.createResource(URI.createURI(inputPath));
+ try {
+ Map options = new HashMap<>();
+ options.put(XMIResourceImpl.OPTION_DEFER_IDREF_RESOLUTION, Boolean.TRUE);
+// options.put(XMIResourceImpl.OPTION_PROCESS_DANGLING_HREF, XMIResourceImpl.OPTION_PROCESS_DANGLING_HREF_DISCARD);
+ inputResource.load(options);
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ return inputResource;
+ }
+
+ /**
+ * Loads a ressource from XMI
+ * @param inputPath path to the xmi
+ * @param extension of the ressource to load
+ * @return
+ */
+
+ @SuppressWarnings({ "rawtypes", "unchecked" })
+ public Resource loadResourceFromXMI(String inputPath, String extension) {
+ Resource.Factory.Registry reg = Resource.Factory.Registry.INSTANCE;
+ Map m = reg.getExtensionToFactoryMap();
+ m.put(extension, resourceFactory);
+ ResourceSet resSetIn = new ResourceSetImpl();
+ Resource inputResource = resSetIn.createResource(URI.createURI(inputPath));
+ try {
+ Map options = new HashMap<>();
+ options.put(XMIResourceImpl.OPTION_DEFER_IDREF_RESOLUTION, Boolean.TRUE);
+// options.put(XMIResourceImpl.OPTION_PROCESS_DANGLING_HREF, XMIResourceImpl.OPTION_PROCESS_DANGLING_HREF_DISCARD);
+ inputResource.load(options);
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ return inputResource;
+ }
+
+ /**
+ * Gets a ressource from a binary form
+ * @param inputPath path to the binary
+ * @param extension of the model to load
+ * @param p EPackage to put the loaded ressource in
+ * @return
+ */
+ public Resource getResourceFromBinary(String inputPath, String extension, EPackage p) {
+ Resource.Factory.Registry reg = Resource.Factory.Registry.INSTANCE;
+ Map m = reg.getExtensionToFactoryMap();
+ m.put(extension, new Resource.Factory() {
+
+ @Override
+ public Resource createResource(URI uri) {
+ return new BinaryResourceImpl(uri);
+ }
+
+ });
+
+ ResourceSet resSetIn = new ResourceSetImpl();
+ //critical part
+ resSetIn.getPackageRegistry().put(p.getNsURI(), p);
+
+ Resource inputResource = resSetIn.createResource(URI.createURI(inputPath));
+ return inputResource;
+ }
+
+
+ /**
+ * Loads a ressource from a binary form
+ * @param inputPath path to the binary
+ * @param extension of the model to load
+ * @param p EPackage to put the loaded ressource in
+ * @return
+ */
+ //TODO: workarounds copied from respective methods without EPackage parameter
+ @SuppressWarnings({ "rawtypes" })
+ public Resource loadResourceFromBinary(String inputPath, String extension, EPackage p) {
+ Resource.Factory.Registry reg = Resource.Factory.Registry.INSTANCE;
+ Map m = reg.getExtensionToFactoryMap();
+ m.put(extension, new Resource.Factory() {
+
+ @Override
+ public Resource createResource(URI uri) {
+ return new BinaryResourceImpl(uri);
+ }
+
+ });
+
+ ResourceSet resSetIn = new ResourceSetImpl();
+ //critical part
+ resSetIn.getPackageRegistry().put(p.getNsURI(), p);
+
+ Resource inputResource = resSetIn.createResource(URI.createURI(inputPath));
+ if (new File(inputPath).exists()) {
+
+ try {
+ Map options = new HashMap<>();
+// options.put(BinaryResourceImpl.OPTION_DEFER_IDREF_RESOLUTION, Boolean.TRUE);
+// options.put(BinaryResourceImpl.OPTION_DEFER_IDREF_RESOLUTION, Boolean.TRUE);
+ // options.put(XMIResourceImpl.OPTION_PROCESS_DANGLING_HREF, XMIResourceImpl.OPTION_PROCESS_DANGLING_HREF_DISCARD);
+ inputResource.load(options);
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ }
+ return inputResource;
+ }
+
+ /**
+ * Loads a ressource from a binary form
+ *
+ * @param inputPath path to the binary
+ * @param extension of the model to load
+ * @return
+ */
+ @SuppressWarnings({ "rawtypes" })
+ public Resource loadResourceFromBinary(String inputPath, String extension) {
+ Resource.Factory.Registry reg = Resource.Factory.Registry.INSTANCE;
+ Map m = reg.getExtensionToFactoryMap();
+ m.put(extension, new Resource.Factory() {
+
+ @Override
+ public Resource createResource(URI uri) {
+ return new BinaryResourceImpl(uri);
+ }
+
+ });
+
+ ResourceSet resSetIn = new ResourceSetImpl();
+ Resource inputResource = resSetIn.createResource(URI.createURI(inputPath));
+ try {
+ Map options = new HashMap<>();
+// options.put(XMIResourceImpl.OPTION_DEFER_IDREF_RESOLUTION, Boolean.TRUE);
+// options.put(XMIResourceImpl.OPTION_PROCESS_DANGLING_HREF, XMIResourceImpl.OPTION_PROCESS_DANGLING_HREF_DISCARD);
+ inputResource.load(options);
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ return inputResource;
+ }
+
+ /**
+ * Stores the binary resource contents to a given path
+ *
+ * @param contents EList of different EObjects to store
+ * @param outputPath path to store to
+ * @param extension of the model to store
+ */
+ @SuppressWarnings({ "rawtypes" })
+ public void storeBinaryResourceContents(EList contents, String outputPath, String extension) {
+ Resource.Factory.Registry reg = Resource.Factory.Registry.INSTANCE;
+ Map m = reg.getExtensionToFactoryMap();
+ m.put(extension, new Resource.Factory() {
+
+ @Override
+ public Resource createResource(URI uri) {
+ return new BinaryResourceImpl(uri);
+ }
+
+ });
+
+ ResourceSet resSet = new ResourceSetImpl();
+ Resource outputResource = resSet.createResource(URI.createURI(outputPath));
+ outputResource.getContents().addAll(contents);
+ try {
+ Map options = new HashMap<>();
+// options.put(XMIResourceImpl.OPTION_PROCESS_DANGLING_HREF, XMIResourceImpl.OPTION_PROCESS_DANGLING_HREF_DISCARD);
+ outputResource.save(options);
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ }
+
+ /**
+ * Stores the resource contents to a given path
+ *
+ * @param contents EList of different EObjects to store
+ * @param outputPath path to store to
+ * @param extension of the model to store
+ */
+ @SuppressWarnings({ "unchecked", "rawtypes" })
+ public void storeResourceContents(EList contents, String outputPath, String extension) {
+ //TODO: duplicated from loadResourceFromXMI => move to a more appropriate location
+ Resource.Factory.Registry reg = Resource.Factory.Registry.INSTANCE;
+ Map m = reg.getExtensionToFactoryMap();
+ m.put(extension, resourceFactory);
+
+ ResourceSet resSet = new ResourceSetImpl();
+ Resource outputResource = resSet.createResource(URI.createURI(outputPath));
+ outputResource.getContents().addAll(contents);
+ try {
+ Map options = new HashMap<>();
+ options.put(XMIResourceImpl.OPTION_ENCODING, "UTF-8");
+// options.put(XMIResourceImpl.OPTION_PROCESS_DANGLING_HREF, XMIResourceImpl.OPTION_PROCESS_DANGLING_HREF_DISCARD);
+ outputResource.save(options);
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ }
+
+
+}
Index: trunk/CrossPare/src/de/ugoe/cs/cpdp/execution/ClassifierCreationExperiment.java
===================================================================
--- trunk/CrossPare/src/de/ugoe/cs/cpdp/execution/ClassifierCreationExperiment.java (revision 32)
+++ trunk/CrossPare/src/de/ugoe/cs/cpdp/execution/ClassifierCreationExperiment.java (revision 32)
@@ -0,0 +1,189 @@
+package de.ugoe.cs.cpdp.execution;
+
+import java.io.File;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.logging.Level;
+
+import weka.core.Instances;
+import de.ugoe.cs.cpdp.ExperimentConfiguration;
+import de.ugoe.cs.cpdp.dataprocessing.IProcessesingStrategy;
+import de.ugoe.cs.cpdp.dataselection.IPointWiseDataselectionStrategy;
+import de.ugoe.cs.cpdp.eval.IEvaluationStrategy;
+import de.ugoe.cs.cpdp.loader.IVersionLoader;
+import de.ugoe.cs.cpdp.training.ITrainer;
+import de.ugoe.cs.cpdp.training.ITrainingStrategy;
+import de.ugoe.cs.cpdp.training.IWekaCompatibleTrainer;
+import de.ugoe.cs.cpdp.versions.SoftwareVersion;
+import de.ugoe.cs.util.console.Console;
+
+/**
+ * Class responsible for executing an experiment according to an {@link ExperimentConfiguration}. The steps
+ * of this ClassifierCreationExperiment are as follows:
+ *
+ * - load the data from the provided data path
+ * - check if given resultsdir exists, if not create one
+ * - execute the following steps for each data set:
+ *
+ * - load the dataset
+ * - set testdata == traindata
+ * - preprocess the data
+ * - postprocess the data
+ * - for each configured trainer do the following:
+ *
+ * - if the classifier should be saved, train it with the dataset
+ * - save it in the results dir
+ * - For each configured evaluator: Do the evaluation and save results
+ *
+ *
+ *
+ *
+ * Note that this class implements {@link IExectuionStrategy}, i.e., each experiment can be started
+ * in its own thread.
+ *
+ * @author Fabian Trautsch
+ */
+public class ClassifierCreationExperiment implements IExecutionStrategy {
+
+ /**
+ * configuration of the experiment
+ */
+ private final ExperimentConfiguration config;
+
+ /**
+ * Constructor. Creates a new experiment based on a configuration.
+ * @param config configuration of the experiment
+ */
+ public ClassifierCreationExperiment(ExperimentConfiguration config) {
+ this.config = config;
+ }
+
+ /**
+ * Executes the experiment with the steps as described in the class comment.
+ * @see Runnable#run()
+ */
+ @Override
+ public void run() {
+ final List versions = new LinkedList<>();
+
+ boolean writeHeader = true;
+
+ for(IVersionLoader loader : config.getLoaders()) {
+ versions.addAll(loader.load());
+ }
+
+
+ File resultsDir = new File(config.getResultsPath());
+ if (!resultsDir.exists()) {
+ resultsDir.mkdir();
+ }
+
+
+ int versionCount = 1;
+ for( SoftwareVersion testVersion : versions ) {
+
+ // At first: traindata == testdata
+ Instances testdata = testVersion.getInstances();
+ Instances traindata = new Instances(testdata);
+
+ // Give the dataset a new name
+ testdata.setRelationName(testVersion.getProject());
+
+ for( IProcessesingStrategy processor : config.getPreProcessors() ) {
+ Console.traceln(Level.FINE, String.format("[%s] [%02d/%02d] %s: applying preprocessor %s", config.getExperimentName(), versionCount, versions.size(), testVersion.getProject(), processor.getClass().getName()));
+ processor.apply(testdata, traindata);
+ }
+
+ for( IPointWiseDataselectionStrategy dataselector : config.getPointWiseSelectors() ) {
+ Console.traceln(Level.FINE, String.format("[%s] [%02d/%02d] %s: applying pointwise selection %s", config.getExperimentName(), versionCount, versions.size(), testVersion.getProject(), dataselector.getClass().getName()));
+ traindata = dataselector.apply(testdata, traindata);
+ }
+
+ for( IProcessesingStrategy processor : config.getPostProcessors() ) {
+ Console.traceln(Level.FINE, String.format("[%s] [%02d/%02d] %s: applying setwise postprocessor %s", config.getExperimentName(), versionCount, versions.size(), testVersion.getProject(), processor.getClass().getName()));
+ processor.apply(testdata, traindata);
+ }
+
+
+
+
+ // Get bug/non bug (TODO)
+ int traindataBug = 0;
+ int traindataNonBug = 0;
+
+ for(int i=0; i allTrainers = new LinkedList<>();
+
+ for( ITrainingStrategy trainer : config.getTrainers() ) {
+
+ // Add trainer to list for evaluation
+ allTrainers.add(trainer);
+
+ // Train classifier
+ trainer.apply(traindata);
+
+ if(config.getSaveClassifier()) {
+ // If classifier should be saved, train him and save him
+ // be careful with typecasting here!
+ IWekaCompatibleTrainer trainerToSave = (IWekaCompatibleTrainer) trainer;
+ //Console.println(trainerToSave.getClassifier().toString());
+ try {
+ weka.core.SerializationHelper.write(resultsDir.getAbsolutePath()+"/"+trainer.getName()+"-"+testVersion.getProject(), trainerToSave.getClassifier());
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+
+ }
+ }
+
+
+
+ for( IEvaluationStrategy evaluator : config.getEvaluators() ) {
+ Console.traceln(Level.FINE, String.format("[%s] [%02d/%02d] %s: applying evaluator %s", config.getExperimentName(), versionCount, versions.size(), testVersion.getProject(), evaluator.getClass().getName()));
+
+ if( writeHeader ) {
+ evaluator.setParameter(config.getResultsPath() + "/" + config.getExperimentName() + ".csv");
+ }
+ evaluator.apply(testdata, traindata, allTrainers, writeHeader);
+ writeHeader = false;
+ }
+
+ versionCount++;
+
+ Console.traceln(Level.INFO, String.format("[%s] [%02d/%02d] %s: finished", config.getExperimentName(), versionCount, versions.size(), testVersion.getProject()));
+
+ }
+
+ }
+
+}
Index: trunk/CrossPare/src/de/ugoe/cs/cpdp/execution/CrossProjectExperiment.java
===================================================================
--- trunk/CrossPare/src/de/ugoe/cs/cpdp/execution/CrossProjectExperiment.java (revision 32)
+++ trunk/CrossPare/src/de/ugoe/cs/cpdp/execution/CrossProjectExperiment.java (revision 32)
@@ -0,0 +1,205 @@
+package de.ugoe.cs.cpdp.execution;
+
+import java.io.File;
+import java.util.Collections;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.logging.Level;
+
+import org.apache.commons.collections4.list.SetUniqueList;
+
+import weka.core.Instances;
+import de.ugoe.cs.cpdp.ExperimentConfiguration;
+import de.ugoe.cs.cpdp.dataprocessing.IProcessesingStrategy;
+import de.ugoe.cs.cpdp.dataprocessing.ISetWiseProcessingStrategy;
+import de.ugoe.cs.cpdp.dataselection.IPointWiseDataselectionStrategy;
+import de.ugoe.cs.cpdp.dataselection.ISetWiseDataselectionStrategy;
+import de.ugoe.cs.cpdp.eval.IEvaluationStrategy;
+import de.ugoe.cs.cpdp.loader.IVersionLoader;
+import de.ugoe.cs.cpdp.training.ISetWiseTrainingStrategy;
+import de.ugoe.cs.cpdp.training.ITrainer;
+import de.ugoe.cs.cpdp.training.ITrainingStrategy;
+import de.ugoe.cs.cpdp.versions.IVersionFilter;
+import de.ugoe.cs.cpdp.versions.SoftwareVersion;
+import de.ugoe.cs.util.console.Console;
+
+/**
+ * Class responsible for executing an experiment according to an {@link ExperimentConfiguration}. The steps of an experiment are as follows:
+ *
+ * - load the data from the provided data path
+ * - filter the data sets according to the provided version filters
+ * - execute the following steps for each data sets as test data that is not ignored through the test version filter:
+ *
+ * - filter the data sets to setup the candidate training data:
+ *
+ * - remove all data sets from the same project
+ * - filter all data sets according to the training data filter
+ *
+ * - apply the setwise preprocessors
+ * - apply the setwise data selection algorithms
+ * - apply the setwise postprocessors
+ * - train the setwise training classifiers
+ * - unify all remaining training data into one data set
+ * - apply the preprocessors
+ * - apply the pointwise data selection algorithms
+ * - apply the postprocessors
+ * - train the normal classifiers
+ * - evaluate the results for all trained classifiers on the training data
+ *
+ *
+ *
+ * Note that this class implements {@link Runnable}, i.e., each experiment can be started in its own thread.
+ * @author Steffen Herbold
+ */
+public class CrossProjectExperiment implements IExecutionStrategy {
+
+ /**
+ * configuration of the experiment
+ */
+ private final ExperimentConfiguration config;
+
+ /**
+ * Constructor. Creates a new experiment based on a configuration.
+ * @param config configuration of the experiment
+ */
+ public CrossProjectExperiment(ExperimentConfiguration config) {
+ this.config = config;
+ }
+
+ /**
+ * Executes the experiment with the steps as described in the class comment.
+ * @see Runnable#run()
+ */
+ @Override
+ public void run() {
+ final List versions = new LinkedList<>();
+
+ for(IVersionLoader loader : config.getLoaders()) {
+ versions.addAll(loader.load());
+ }
+
+ for( IVersionFilter filter : config.getVersionFilters() ) {
+ filter.apply(versions);
+ }
+ boolean writeHeader = true;
+ int versionCount = 1;
+ int testVersionCount = 0;
+
+ for( SoftwareVersion testVersion : versions ) {
+ if( isVersion(testVersion, config.getTestVersionFilters()) ) {
+ testVersionCount++;
+ }
+ }
+
+ // sort versions
+ Collections.sort(versions);
+
+ for( SoftwareVersion testVersion : versions ) {
+ if( isVersion(testVersion, config.getTestVersionFilters()) ) {
+ Console.traceln(Level.INFO, String.format("[%s] [%02d/%02d] %s: starting", config.getExperimentName(), versionCount, testVersionCount, testVersion.getVersion()));
+
+ // Setup testdata and training data
+ Instances testdata = testVersion.getInstances();
+ String testProject = testVersion.getProject();
+ SetUniqueList traindataSet = SetUniqueList.setUniqueList(new LinkedList());
+ for( SoftwareVersion trainingVersion : versions ) {
+ if( isVersion(trainingVersion, config.getTrainingVersionFilters()) ) {
+ if( trainingVersion!=testVersion ) {
+ if( !trainingVersion.getProject().equals(testProject) ) {
+ traindataSet.add(trainingVersion.getInstances());
+ }
+ }
+ }
+ }
+
+ for( ISetWiseProcessingStrategy processor : config.getSetWisePreprocessors() ) {
+ Console.traceln(Level.FINE, String.format("[%s] [%02d/%02d] %s: applying setwise preprocessor %s", config.getExperimentName(), versionCount, testVersionCount, testVersion.getVersion(), processor.getClass().getName()));
+ processor.apply(testdata, traindataSet);
+ }
+ for( ISetWiseDataselectionStrategy dataselector : config.getSetWiseSelectors() ) {
+ Console.traceln(Level.FINE, String.format("[%s] [%02d/%02d] %s: applying setwise selection %s", config.getExperimentName(), versionCount, testVersionCount, testVersion.getVersion(), dataselector.getClass().getName()));
+ dataselector.apply(testdata, traindataSet);
+ }
+ for( ISetWiseProcessingStrategy processor : config.getSetWisePostprocessors() ) {
+ Console.traceln(Level.FINE, String.format("[%s] [%02d/%02d] %s: applying setwise postprocessor %s", config.getExperimentName(), versionCount, testVersionCount, testVersion.getVersion(), processor.getClass().getName()));
+ processor.apply(testdata, traindataSet);
+ }
+ for( ISetWiseTrainingStrategy setwiseTrainer : config.getSetWiseTrainers() ) {
+ Console.traceln(Level.FINE, String.format("[%s] [%02d/%02d] %s: applying setwise trainer %s", config.getExperimentName(), versionCount, testVersionCount, testVersion.getVersion(), setwiseTrainer.getName()));
+ setwiseTrainer.apply(traindataSet);
+ }
+ Instances traindata = makeSingleTrainingSet(traindataSet);
+ for( IProcessesingStrategy processor : config.getPreProcessors() ) {
+ Console.traceln(Level.FINE, String.format("[%s] [%02d/%02d] %s: applying preprocessor %s", config.getExperimentName(), versionCount, testVersionCount, testVersion.getVersion(), processor.getClass().getName()));
+ processor.apply(testdata, traindata);
+ }
+ for( IPointWiseDataselectionStrategy dataselector : config.getPointWiseSelectors() ) {
+ Console.traceln(Level.FINE, String.format("[%s] [%02d/%02d] %s: applying pointwise selection %s", config.getExperimentName(), versionCount, testVersionCount, testVersion.getVersion(), dataselector.getClass().getName()));
+ traindata = dataselector.apply(testdata, traindata);
+ }
+ for( IProcessesingStrategy processor : config.getPostProcessors() ) {
+ Console.traceln(Level.FINE, String.format("[%s] [%02d/%02d] %s: applying setwise postprocessor %s", config.getExperimentName(), versionCount, testVersionCount, testVersion.getVersion(), processor.getClass().getName()));
+ processor.apply(testdata, traindata);
+ }
+ for( ITrainingStrategy trainer : config.getTrainers() ) {
+ Console.traceln(Level.FINE, String.format("[%s] [%02d/%02d] %s: applying trainer %s", config.getExperimentName(), versionCount, testVersionCount, testVersion.getVersion(), trainer.getName()));
+ trainer.apply(traindata);
+ }
+ File resultsDir = new File(config.getResultsPath());
+ if (!resultsDir.exists()) {
+ resultsDir.mkdir();
+ }
+ for( IEvaluationStrategy evaluator : config.getEvaluators() ) {
+ Console.traceln(Level.FINE, String.format("[%s] [%02d/%02d] %s: applying evaluator %s", config.getExperimentName(), versionCount, testVersionCount, testVersion.getVersion(), evaluator.getClass().getName()));
+ List allTrainers = new LinkedList<>();
+ for( ISetWiseTrainingStrategy setwiseTrainer : config.getSetWiseTrainers() ) {
+ allTrainers.add(setwiseTrainer);
+ }
+ for( ITrainingStrategy trainer : config.getTrainers() ) {
+ allTrainers.add(trainer);
+ }
+ if( writeHeader ) {
+ evaluator.setParameter(config.getResultsPath() + "/" + config.getExperimentName() + ".csv");
+ }
+ evaluator.apply(testdata, traindata, allTrainers, writeHeader);
+ writeHeader = false;
+ }
+ Console.traceln(Level.INFO, String.format("[%s] [%02d/%02d] %s: finished", config.getExperimentName(), versionCount, testVersionCount, testVersion.getVersion()));
+ versionCount++;
+ }
+ }
+ }
+
+ /**
+ * Helper method that checks if a version passes all filters.
+ * @param version version that is checked
+ * @param filters list of the filters
+ * @return true, if the version passes all filters, false otherwise
+ */
+ private boolean isVersion(SoftwareVersion version, List filters) {
+ boolean result = true;
+ for( IVersionFilter filter : filters) {
+ result &= !filter.apply(version);
+ }
+ return result;
+ }
+
+ /**
+ * Helper method that combines a set of Weka {@link Instances} sets into a single {@link Instances} set.
+ * @param traindataSet set of {@link Instances} to be combines
+ * @return single {@link Instances} set
+ */
+ public static Instances makeSingleTrainingSet(SetUniqueList traindataSet) {
+ Instances traindataFull = null;
+ for( Instances traindata : traindataSet) {
+ if( traindataFull==null ) {
+ traindataFull = new Instances(traindata);
+ } else {
+ for( int i=0 ; i ARFFX -> ARFF
+ *
+ * @author Fabian Trautsch
+ *
+ */
+public class DecentDataLoader implements SingleVersionLoader{
+
+ // Model Handler for Decent Models
+ private DECENTEpsilonModelHandler modelHandler = new DECENTEpsilonModelHandler();
+
+ // Set log level
+ String logLevel = "1";
+ String logToFile = "false";
+
+ // This list contains attributes, that should be removed before building the arff file
+ private static List attributeFilter = new LinkedList();
+
+ // This list contains all names of the different artifacts
+ private static Set artifactNames = new LinkedHashSet();
+
+ // Name of the class attribute.
+ private static final String classAttributeName = "LABEL.Artifact.Target.BugFix.AverageWeight";
+
+
+ private int getIndexOfArtifactName(String artifactName) {
+ int index = -1;
+ if(artifactNames.contains(artifactName)) {
+ int i=0;
+ for(String nameInSet: artifactNames) {
+ if(nameInSet.equals(artifactName)) {
+ index = i;
+ } else {
+ i++;
+ }
+ }
+ }
+
+ return index;
+ }
+
+ /**
+ * Defines attributes, that should be removed before building the
+ * ARFF File from.
+ */
+ private void setAttributeFilter() {
+ attributeFilter.add("Agent.Name");
+
+ }
+
+ /**
+ * Saves the dataset as arff after transformation (decent->arffx) and
+ * filtering
+ *
+ * @param dataSet the WEKA dataset to save
+ * @param arffLocation location where it should be saved to
+ */
+ public void save(Instances dataSet, String arffLocation) {
+
+
+ ArffSaver saver = new ArffSaver();
+ saver.setInstances(dataSet);
+ try {
+ saver.setFile(new File(arffLocation));
+ saver.writeBatch();
+ } catch (IOException e) {
+ Console.printerrln("Cannot save the file to path: "+arffLocation);
+ e.printStackTrace();
+ }
+ }
+
+
+ /**
+ * Loads the given decent file and tranform it from decent->arffx->arff
+ * @return Instances in WEKA format
+ */
+ @Override
+ public Instances load(File file) {
+
+ // Set attributeFilter
+ setAttributeFilter();
+
+ // Register MetaModels
+ try {
+ registerMetaModels();
+ } catch (Exception e1) {
+ Console.printerrln("Metamodels cannot be registered!");
+ e1.printStackTrace();
+ }
+
+ // Set location of decent and arffx Model
+ String decentModelLocation = file.getAbsolutePath();
+ String pathToDecentModelFolder = decentModelLocation.substring(0,decentModelLocation.lastIndexOf(File.separator));
+ String arffxModelLocation = pathToDecentModelFolder+"/model.arffx";
+ String logModelLocation = pathToDecentModelFolder+"/model.log";
+ String arffLocation = pathToDecentModelFolder+"/model.arff";
+
+ // If arff File exists, load from it!
+ if(new File(arffLocation).exists()) {
+ System.out.println("Loading arff File...");
+ BufferedReader reader;
+ Instances data = null;
+ try {
+ reader = new BufferedReader(new FileReader(arffLocation));
+ data = new Instances(reader);
+ reader.close();
+ } catch (FileNotFoundException e) {
+ Console.printerrln("File with path: "+arffLocation+" was not found.");
+ e.printStackTrace();
+ } catch (IOException e) {
+ Console.printerrln("File with path: "+arffLocation+" cannot be read.");
+ e.printStackTrace();
+ }
+
+ // Set class attribute if not set
+ if(data.classIndex() == -1) {
+ Attribute classAttribute = data.attribute(classAttributeName);
+ data.setClass(classAttribute);
+ }
+
+
+ return data;
+ }
+
+ // Location of EOL Scripts
+ String preprocess = "./decent/epsilon/query/preprocess.eol";
+ String arffxToArffSource = "./decent/epsilon/query/addLabels.eol";
+
+ // Set Log Properties
+ System.setProperty("epsilon.logLevel", logLevel);
+ System.setProperty("epsilon.logToFile", logToFile);
+ System.setProperty("epsilon.logFileAvailable", "false");
+
+ // Set decent2arffx Properties
+ System.setProperty("epsilon.transformation.decent2arffx.skipSource", "false");
+ System.setProperty("epsilon.transformation.decent2arffx.type", "code");
+
+
+
+ // Preprocess Data, transform from decent2arffx
+ try {
+ IEolExecutableModule preProcessModule = loadModule(preprocess);
+ IModel preProcessDecentModel = modelHandler.getDECENTModel(decentModelLocation, true, true);
+ IModel preProcessArffxarffxModel = modelHandler.getARFFxModel(arffxModelLocation, false, true);
+ preProcessModule.getContext().getModelRepository().addModel(preProcessDecentModel);
+ preProcessModule.getContext().getModelRepository().addModel(preProcessArffxarffxModel);
+ execute(preProcessModule, logModelLocation);
+ preProcessDecentModel.dispose();
+ preProcessArffxarffxModel.dispose();
+ preProcessModule.reset();
+ } catch (URISyntaxException e) {
+ Console.printerrln("URI Syntax for decent or arffx model is wrong.");
+ e.printStackTrace();
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+
+
+
+
+ // Transform to arff, for label and confidence attributes
+ try {
+ IEolExecutableModule arffxToArffModule = loadModule(arffxToArffSource);
+ IModel arffxToArffArffxModel = modelHandler.getARFFxModel(arffxModelLocation, true, true);
+ arffxToArffModule.getContext().getModelRepository().addModel(arffxToArffArffxModel);
+ execute(arffxToArffModule, logModelLocation);
+ arffxToArffArffxModel.dispose();
+ // can be stored and retained alternatively
+ arffxToArffModule.reset();
+ } catch (URISyntaxException e) {
+ Console.printerrln("URI Syntax for arffx model is wrong.");
+ e.printStackTrace();
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+
+ // Unregister MetaModels, otherwise cast will fail
+ HashMap metaModelCache = new HashMap<>();
+ for (String key : EPackage.Registry.INSTANCE.keySet()) {
+ metaModelCache.put(key, EPackage.Registry.INSTANCE.get(key));
+ };
+
+ for (String key : metaModelCache .keySet()) {
+ EPackage.Registry.INSTANCE.remove(key);
+ };
+
+
+ // Workaround to gernerate a usable URI. Absolute path is not
+ // possible, therefore we need to construct a relative path
+
+ URL location = DecentDataLoader.class.getProtectionDomain().getCodeSource().getLocation();
+ String basePath = location.getFile();
+
+ // Location is the bin folder, so we need to delete the last 4 characters
+ basePath = basePath.substring(0, basePath.length() - 4);
+ String relativePath = new File(basePath).toURI().relativize(new File(arffxModelLocation).toURI()).getPath();
+
+ // Loard arffx file and create WEKA Instances
+ ARFFxResourceTool tool = new ARFFxResourceTool();
+ Resource resource = tool.loadResourceFromXMI(relativePath, "arffx");
+
+ Instances dataSet = null;
+ for(EObject o: resource.getContents()) {
+ Model m = (Model) o;
+ dataSet = createWekaDataFormat(m);
+
+ for(Instance i : m.getData()) {
+ createWekaInstance(dataSet, i);
+ }
+ }
+
+ // Set class attribute
+ Attribute classAttribute = dataSet.attribute(classAttributeName);
+ dataSet.setClass(classAttribute);
+
+ // Save as ARFF
+ save(dataSet, arffLocation);
+
+ return dataSet;
+
+ }
+
+
+ /**
+ * Creates a WekaInstance from an ARFFX Model Instance
+ *
+ * @param dataSet WekaInstance dataset, where the arffx model instances should be
+ * added to
+ * @param i arffx model instance
+ */
+ private void createWekaInstance(Instances dataSet, Instance i) {
+ double[] values = new double[dataSet.numAttributes()];
+ int j=0;
+
+ for(Value value : i.getValues()) {
+ String dataValue = value.getContent();
+ String attributeName = value.getOfAttribute().getName();
+
+ if(attributeFilter.contains(attributeName)) {
+ continue;
+ }
+
+ // Is value a LABEL.* attribute?
+ if(isLabel(attributeName)) {
+ values[j] = dataSet.attribute(j).indexOfValue(dataValue);
+ } else if (isConfidenceLabel(attributeName)){
+ // Is value a CONFIDENCE.* attribute?
+ values[j] = dataSet.attribute(j).indexOfValue(dataValue);
+ } else if(attributeName.equals("Artifact.Name")){
+ // Is it the name of the artifact?
+ artifactNames.add(dataValue);
+ values[j] = getIndexOfArtifactName(dataValue);
+ } else {
+ // Is it a numeric value?
+ values[j] = Double.parseDouble(dataValue);
+ }
+
+ j++;
+ }
+
+ DenseInstance inst = new DenseInstance(1.0, values);
+ dataSet.add(inst);
+ }
+
+ /**
+ * Creates a Weka Instances set out of a arffx model
+ * @param m arffx model
+ * @return
+ */
+ private Instances createWekaDataFormat(Model m) {
+
+ // Bad solution, can be enhanced (continue in for loop)
+ ArrayList datasetAttributes = new ArrayList();
+ for(ARFFx.Attribute attribute :m.getAttributes()) {
+ String attributeName = attribute.getName();
+
+ if(attributeFilter.contains(attributeName)) {
+ continue;
+ }
+
+ Attribute wekaAttr;
+
+ // Is attribute a LABEL.* attribute?
+ if(isLabel(attributeName)) {
+ // Classattribute
+ final ArrayList classAttVals = new ArrayList();
+ classAttVals.add("false");
+ classAttVals.add("true");
+ wekaAttr = new Attribute(attributeName, classAttVals);
+ } else if(isConfidenceLabel(attributeName)){
+ // Is attribute a CONFIDENCE.* attribute?
+ ArrayList labels = new ArrayList();
+ labels.add("high");
+ labels.add("low");
+ wekaAttr = new Attribute(attributeName, labels);
+ } else {
+ // Is it a numeric attribute?
+ wekaAttr = new Attribute(attributeName);
+ }
+
+ datasetAttributes.add(wekaAttr);
+ }
+
+
+ return new Instances("test-dataset", datasetAttributes, 0);
+ }
+
+ /**
+ * Helper methods which indicates if the given value starts with "LABEL"
+ *
+ * @param value to test
+ * @return
+ */
+ private boolean isLabel(String value) {
+ if(value.length()>= 5 && value.substring(0, 5).equals("LABEL")) {
+ return true;
+ }
+
+ return false;
+ }
+
+ /**
+ * Helper method which indicates if the given value starts with "CONFIDENCE"
+ * @param value to test
+ * @return
+ */
+ private boolean isConfidenceLabel(String value) {
+ if(value.length()>= 10 && value.substring(0, 10).equals("CONFIDENCE")) {
+ return true;
+ }
+
+ return false;
+ }
+
+
+ /**
+ * Returns if a filename ends with ".decent"
+ * @return
+ */
+ @Override
+ public boolean filenameFilter(String filename) {
+ return filename.endsWith(".decent");
+ }
+
+ /**
+ * Helper method for executing a eol scripts and adding the log model beforehand
+ * @param module module to execute
+ * @param logModelLocation location of the log model
+ * @throws Exception
+ */
+ private void execute(IEolExecutableModule module, String logModelLocation)
+ throws Exception {
+ IModel logModel = modelHandler.getLOGModel(logModelLocation, true, true);
+ module.getContext().getModelRepository().addModel(logModel);
+ module.execute();
+ logModel.dispose();
+ }
+
+ /**
+ * Loads the module from a given source
+ *
+ * @param source where the module is (e.g. eol script)
+ * @return
+ * @throws Exception
+ * @throws URISyntaxException
+ */
+ private IEolExecutableModule loadModule(String source) throws Exception,
+ URISyntaxException {
+
+ IEolExecutableModule module = null;
+ if (source.endsWith("etl")) {
+ module = new EtlModule();
+ } else if (source.endsWith("eol")) {
+ module = new EolModule();
+ } else {
+
+ }
+
+ module.parse(modelHandler.getFile(source));
+
+ if (module.getParseProblems().size() > 0) {
+ Console.printerrln("Parse error occured...");
+ for (ParseProblem problem : module.getParseProblems()) {
+ System.err.println(problem.toString());
+ }
+ // System.exit(-1);
+ }
+
+ return module;
+ }
+
+ /**
+ * Helper method for registering the metamodels
+ * @throws Exception
+ */
+ private void registerMetaModels() throws Exception {
+ String metaModelsPath = DECENTEpsilonModelHandler.metaPath;
+ File metaModelsLocation = new File(metaModelsPath);
+ for (File file : metaModelsLocation.listFiles()) {
+ if (file.getName().endsWith(".ecore")) {
+ EmfUtil.register(URI.createFileURI(file.getAbsolutePath()), EPackage.Registry.INSTANCE);
+ }
+ }
+ }
+
+}
Index: trunk/CrossPare/src/de/ugoe/cs/cpdp/loader/DecentFolderLoader.java
===================================================================
--- trunk/CrossPare/src/de/ugoe/cs/cpdp/loader/DecentFolderLoader.java (revision 32)
+++ trunk/CrossPare/src/de/ugoe/cs/cpdp/loader/DecentFolderLoader.java (revision 32)
@@ -0,0 +1,89 @@
+package de.ugoe.cs.cpdp.loader;
+
+import java.io.File;
+import java.util.LinkedList;
+import java.util.List;
+
+import weka.core.Instances;
+
+import de.ugoe.cs.cpdp.versions.SoftwareVersion;
+
+/**
+ * Implements the {@link AbstractFolderLoader}
+ *
+ * @author Fabian Trautsch
+ */
+public class DecentFolderLoader extends AbstractFolderLoader {
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see de.ugoe.cs.cpdp.loader.AbstractFolderLoader#getSingleLoader()
+ */
+ @Override
+ protected SingleVersionLoader getSingleLoader() {
+ return new DecentDataLoader();
+ }
+
+ /**
+ * @see de.ugoe.cs.cpdp.loader.IVersionLoader#load()
+ */
+ @Override
+ public List load() {
+ final List versions = new LinkedList();
+
+ final File dataDir = new File(path);
+ final SingleVersionLoader instancesLoader = getSingleLoader();
+
+ String projectName = dataDir.getName();
+
+
+ /*
+ * The following lines make it possible, that we can have two different possibilities
+ * to load data:
+ * 1) From one project (e.g. /decent/input/project1)
+ * 2) From more than one project (e.g. /decent/input/)
+ *
+ * Requirement is, that we have a folder structure like this:
+ * "/decent/input/project1/model.decent, /decent/input/project2/model.decent, ..."
+ *
+ * In the first one the "else" is executed, therefore it will just search the folder "project1"
+ * for a "model.decent" file. In the second one, it will look into each folder and searches for
+ * "model.decent" files.
+ */
+ for (File projectDir : dataDir.listFiles()) {
+ if (projectDir.isDirectory()) {
+ projectName = projectDir.getName();
+ for (File versionFile : projectDir.listFiles()) {
+ loadDataFromFile(versionFile,instancesLoader, projectName, versions);
+ }
+ } else {
+ loadDataFromFile(projectDir, instancesLoader, projectName, versions);
+ }
+ }
+ return versions;
+ }
+
+ /**
+ * Loads data from a file and adds the instances from the load method to the
+ * versions List.
+ *
+ * @param versionFile file to load from
+ * @param instancesLoader loader that should be used
+ * @param projectName name of the project which was loaded
+ * @param versions list, where the weka instances are added to
+ */
+
+ private void loadDataFromFile(File versionFile,
+ SingleVersionLoader instancesLoader, String projectName, List versions) {
+ if (versionFile.isFile()
+ && instancesLoader.filenameFilter(versionFile
+ .getName())) {
+ String versionName = versionFile.getName();
+ Instances data = instancesLoader.load(versionFile);
+ versions.add(new SoftwareVersion(projectName,
+ versionName, data));
+ }
+ }
+
+}
Index: trunk/CrossPare/src/de/ugoe/cs/cpdp/loader/IDecentVersionLoader.java
===================================================================
--- trunk/CrossPare/src/de/ugoe/cs/cpdp/loader/IDecentVersionLoader.java (revision 32)
+++ trunk/CrossPare/src/de/ugoe/cs/cpdp/loader/IDecentVersionLoader.java (revision 32)
@@ -0,0 +1,11 @@
+package de.ugoe.cs.cpdp.loader;
+
+import java.util.List;
+
+import de.ugoe.cs.cpdp.versions.SoftwareVersion;
+
+public interface IDecentVersionLoader extends IVersionLoader{
+
+ public List load(List decentAttributes);
+
+}
Index: trunk/CrossPare/src/de/ugoe/cs/cpdp/loader/IVersionLoader.java
===================================================================
--- trunk/CrossPare/src/de/ugoe/cs/cpdp/loader/IVersionLoader.java (revision 31)
+++ trunk/CrossPare/src/de/ugoe/cs/cpdp/loader/IVersionLoader.java (revision 32)
@@ -26,3 +26,4 @@
*/
public List load();
+
}