Context Navigation

DecentDataLoader.java @ 39

Last change on this file since 39 was 36, checked in by ftrautsch, 11 years ago
integration of decent models in crosspare
Property svn:mime-type set to `text/plain`
File size: 13.4 KB

Line
1	package de.ugoe.cs.cpdp.loader;
2
3	import java.io.BufferedReader;
4	import java.io.File;
5	import java.io.FileNotFoundException;
6	import java.io.FileReader;
7	import java.io.IOException;
8	import java.net.URISyntaxException;
9	import java.net.URL;
10	import java.util.ArrayList;
11	import java.util.HashMap;
12	import java.util.LinkedHashSet;
13	import java.util.LinkedList;
14	import java.util.List;
15	import java.util.Set;
16
17	import org.eclipse.emf.common.util.URI;
18	import org.eclipse.emf.ecore.EObject;
19	import org.eclipse.emf.ecore.EPackage;
20	import org.eclipse.emf.ecore.resource.Resource;
21	import org.eclipse.epsilon.common.parse.problem.ParseProblem;
22	import org.eclipse.epsilon.emc.emf.EmfUtil;
23	import org.eclipse.epsilon.eol.EolModule;
24	import org.eclipse.epsilon.eol.IEolExecutableModule;
25	import org.eclipse.epsilon.eol.models.IModel;
26	import org.eclipse.epsilon.etl.EtlModule;
27
28	import de.ugoe.cs.cpdp.decentApp.models.arffx.Instance;
29	import de.ugoe.cs.cpdp.decentApp.models.arffx.Model;
30	import de.ugoe.cs.cpdp.decentApp.models.arffx.Value;
31	import de.ugoe.cs.cpdp.decentApp.ARFFxResourceTool;
32	import de.ugoe.cs.cpdp.decentApp.DECENTEpsilonModelHandler;
33	import de.ugoe.cs.util.console.Console;
34	import weka.core.Attribute;
35	import weka.core.DenseInstance;
36	import weka.core.Instances;
37	import weka.core.converters.ArffSaver;
38
39	/**
40	* Class for loading a decent model file.
41	* Loads a decent model file and (if no arff file is present) and does the
42	* following conversions:
43	* DECENT -> ARFFX -> ARFF
44	*
45	* @author Fabian Trautsch
46	*
47	*/
48	public class DecentDataLoader implements SingleVersionLoader{
49
50	// Model Handler for Decent Models
51	private DECENTEpsilonModelHandler modelHandler = new DECENTEpsilonModelHandler();
52
53	// Set log level
54	String logLevel = "1";
55	String logToFile = "false";
56
57	// This list contains attributes, that should be removed before building the arff file
58	private static List<String> attributeFilter = new LinkedList<String>();
59
60	// This list contains all names of the different artifacts
61	private static Set<String> artifactNames = new LinkedHashSet<String>();
62
63	// Name of the class attribute.
64	private static final String classAttributeName = "LABEL.Artifact.Target.BugFix.AverageWeight";
65
66
67	private int getIndexOfArtifactName(String artifactName) {
68	int index = -1;
69	if(artifactNames.contains(artifactName)) {
70	int i=0;
71	for(String nameInSet: artifactNames) {
72	if(nameInSet.equals(artifactName)) {
73	index = i;
74	} else {
75	i++;
76	}
77	}
78	}
79
80	return index;
81	}
82
83	/**
84	* Defines attributes, that should be removed before building the
85	* ARFF File from.
86	*/
87	private void setAttributeFilter() {
88	attributeFilter.add("Agent.Name");
89
90	}
91
92	/**
93	* Saves the dataset as arff after transformation (decent->arffx) and
94	* filtering
95	*
96	* @param dataSet the WEKA dataset to save
97	* @param arffLocation location where it should be saved to
98	*/
99	public void save(Instances dataSet, String arffLocation) {
100
101
102	ArffSaver saver = new ArffSaver();
103	saver.setInstances(dataSet);
104	try {
105	saver.setFile(new File(arffLocation));
106	saver.writeBatch();
107	} catch (IOException e) {
108	Console.printerrln("Cannot save the file to path: "+arffLocation);
109	e.printStackTrace();
110	}
111	}
112
113
114	/**
115	* Loads the given decent file and tranform it from decent->arffx->arff
116	* @return Instances in WEKA format
117	*/
118	@Override
119	public Instances load(File file) {
120
121	// Set attributeFilter
122	setAttributeFilter();
123
124	// Register MetaModels
125	try {
126	registerMetaModels();
127	} catch (Exception e1) {
128	Console.printerrln("Metamodels cannot be registered!");
129	e1.printStackTrace();
130	}
131
132	// Set location of decent and arffx Model
133	String decentModelLocation = file.getAbsolutePath();
134	String pathToDecentModelFolder = decentModelLocation.substring(0,decentModelLocation.lastIndexOf(File.separator));
135	String arffxModelLocation = pathToDecentModelFolder+"/model.arffx";
136	String logModelLocation = pathToDecentModelFolder+"/model.log";
137	String arffLocation = pathToDecentModelFolder+"/model.arff";
138
139	// If arff File exists, load from it!
140	if(new File(arffLocation).exists()) {
141	System.out.println("Loading arff File...");
142	BufferedReader reader;
143	Instances data = null;
144	try {
145	reader = new BufferedReader(new FileReader(arffLocation));
146	data = new Instances(reader);
147	reader.close();
148	} catch (FileNotFoundException e) {
149	Console.printerrln("File with path: "+arffLocation+" was not found.");
150	e.printStackTrace();
151	} catch (IOException e) {
152	Console.printerrln("File with path: "+arffLocation+" cannot be read.");
153	e.printStackTrace();
154	}
155
156	// Set class attribute if not set
157	if(data.classIndex() == -1) {
158	Attribute classAttribute = data.attribute(classAttributeName);
159	data.setClass(classAttribute);
160	}
161
162
163	return data;
164	}
165
166	// Location of EOL Scripts
167	String preprocess = "./decent/epsilon/query/preprocess.eol";
168	String arffxToArffSource = "./decent/epsilon/query/addLabels.eol";
169
170	// Set Log Properties
171	System.setProperty("epsilon.logLevel", logLevel);
172	System.setProperty("epsilon.logToFile", logToFile);
173	System.setProperty("epsilon.logFileAvailable", "false");
174
175	// Set decent2arffx Properties
176	System.setProperty("epsilon.transformation.decent2arffx.skipSource", "false");
177	System.setProperty("epsilon.transformation.decent2arffx.type", "code");
178
179
180
181	// Preprocess Data, transform from decent2arffx
182	try {
183	IEolExecutableModule preProcessModule = loadModule(preprocess);
184	IModel preProcessDecentModel = modelHandler.getDECENTModel(decentModelLocation, true, true);
185	IModel preProcessArffxarffxModel = modelHandler.getARFFxModel(arffxModelLocation, false, true);
186	preProcessModule.getContext().getModelRepository().addModel(preProcessDecentModel);
187	preProcessModule.getContext().getModelRepository().addModel(preProcessArffxarffxModel);
188	execute(preProcessModule, logModelLocation);
189	preProcessDecentModel.dispose();
190	preProcessArffxarffxModel.dispose();
191	preProcessModule.reset();
192	} catch (URISyntaxException e) {
193	Console.printerrln("URI Syntax for decent or arffx model is wrong.");
194	e.printStackTrace();
195	} catch (Exception e) {
196	e.printStackTrace();
197	}
198
199
200
201
202	// Transform to arff, for label and confidence attributes
203	try {
204	IEolExecutableModule arffxToArffModule = loadModule(arffxToArffSource);
205	IModel arffxToArffArffxModel = modelHandler.getARFFxModel(arffxModelLocation, true, true);
206	arffxToArffModule.getContext().getModelRepository().addModel(arffxToArffArffxModel);
207	execute(arffxToArffModule, logModelLocation);
208	arffxToArffArffxModel.dispose();
209	// can be stored and retained alternatively
210	arffxToArffModule.reset();
211	} catch (URISyntaxException e) {
212	Console.printerrln("URI Syntax for arffx model is wrong.");
213	e.printStackTrace();
214	} catch (Exception e) {
215	e.printStackTrace();
216	}
217
218	// Unregister MetaModels, otherwise cast will fail
219	HashMap<String, Object> metaModelCache = new HashMap<>();
220	for (String key : EPackage.Registry.INSTANCE.keySet()) {
221	metaModelCache.put(key, EPackage.Registry.INSTANCE.get(key));
222	};
223
224	for (String key : metaModelCache .keySet()) {
225	EPackage.Registry.INSTANCE.remove(key);
226	};
227
228
229	// Workaround to gernerate a usable URI. Absolute path is not
230	// possible, therefore we need to construct a relative path
231
232	URL location = DecentDataLoader.class.getProtectionDomain().getCodeSource().getLocation();
233	String basePath = location.getFile();
234
235	// Location is the bin folder, so we need to delete the last 4 characters
236	basePath = basePath.substring(0, basePath.length() - 4);
237	String relativePath = new File(basePath).toURI().relativize(new File(arffxModelLocation).toURI()).getPath();
238
239	// Loard arffx file and create WEKA Instances
240	ARFFxResourceTool tool = new ARFFxResourceTool();
241	Resource resource = tool.loadResourceFromXMI(relativePath, "arffx");
242
243	Instances dataSet = null;
244	for(EObject o: resource.getContents()) {
245	Model m = (Model) o;
246	dataSet = createWekaDataFormat(m);
247
248	for(Instance i : m.getData()) {
249	createWekaInstance(dataSet, i);
250	}
251	}
252
253	// Set class attribute
254	Attribute classAttribute = dataSet.attribute(classAttributeName);
255	dataSet.setClass(classAttribute);
256
257	// Save as ARFF
258	save(dataSet, arffLocation);
259
260	return dataSet;
261
262	}
263
264
265	/**
266	* Creates a WekaInstance from an ARFFX Model Instance
267	*
268	* @param dataSet WekaInstance dataset, where the arffx model instances should be
269	* added to
270	* @param i arffx model instance
271	*/
272	private void createWekaInstance(Instances dataSet, Instance i) {
273	double[] values = new double[dataSet.numAttributes()];
274	int j=0;
275
276	for(Value value : i.getValues()) {
277	String dataValue = value.getContent();
278	String attributeName = value.getOfAttribute().getName();
279
280	if(attributeFilter.contains(attributeName)) {
281	continue;
282	}
283
284	// Is value a LABEL.* attribute?
285	if(isLabel(attributeName)) {
286	values[j] = dataSet.attribute(j).indexOfValue(dataValue);
287	} else if (isConfidenceLabel(attributeName)){
288	// Is value a CONFIDENCE.* attribute?
289	values[j] = dataSet.attribute(j).indexOfValue(dataValue);
290	} else if(attributeName.equals("Artifact.Name")){
291	// Is it the name of the artifact?
292	artifactNames.add(dataValue);
293	values[j] = getIndexOfArtifactName(dataValue);
294	} else {
295	// Is it a numeric value?
296	values[j] = Double.parseDouble(dataValue);
297	}
298
299	j++;
300	}
301
302	DenseInstance inst = new DenseInstance(1.0, values);
303	dataSet.add(inst);
304	}
305
306	/**
307	* Creates a Weka Instances set out of a arffx model
308	* @param m arffx model
309	* @return
310	*/
311	private Instances createWekaDataFormat(Model m) {
312
313	// Bad solution, can be enhanced (continue in for loop)
314	ArrayList<Attribute> datasetAttributes = new ArrayList<Attribute>();
315	for(de.ugoe.cs.cpdp.decentApp.models.arffx.Attribute attribute :m.getAttributes()) {
316	String attributeName = attribute.getName();
317
318	if(attributeFilter.contains(attributeName)) {
319	continue;
320	}
321
322	Attribute wekaAttr;
323
324	// Is attribute a LABEL.* attribute?
325	if(isLabel(attributeName)) {
326	// Classattribute
327	final ArrayList<String> classAttVals = new ArrayList<String>();
328	classAttVals.add("false");
329	classAttVals.add("true");
330	wekaAttr = new Attribute(attributeName, classAttVals);
331	} else if(isConfidenceLabel(attributeName)){
332	// Is attribute a CONFIDENCE.* attribute?
333	ArrayList<String> labels = new ArrayList<String>();
334	labels.add("high");
335	labels.add("low");
336	wekaAttr = new Attribute(attributeName, labels);
337	} else {
338	// Is it a numeric attribute?
339	wekaAttr = new Attribute(attributeName);
340	}
341
342	datasetAttributes.add(wekaAttr);
343	}
344
345
346	return new Instances("test-dataset", datasetAttributes, 0);
347	}
348
349	/**
350	* Helper methods which indicates if the given value starts with "LABEL"
351	*
352	* @param value to test
353	* @return
354	*/
355	private boolean isLabel(String value) {
356	if(value.length()>= 5 && value.substring(0, 5).equals("LABEL")) {
357	return true;
358	}
359
360	return false;
361	}
362
363	/**
364	* Helper method which indicates if the given value starts with "CONFIDENCE"
365	* @param value to test
366	* @return
367	*/
368	private boolean isConfidenceLabel(String value) {
369	if(value.length()>= 10 && value.substring(0, 10).equals("CONFIDENCE")) {
370	return true;
371	}
372
373	return false;
374	}
375
376
377	/**
378	* Returns if a filename ends with ".decent"
379	* @return
380	*/
381	@Override
382	public boolean filenameFilter(String filename) {
383	return filename.endsWith(".decent");
384	}
385
386	/**
387	* Helper method for executing a eol scripts and adding the log model beforehand
388	* @param module module to execute
389	* @param logModelLocation location of the log model
390	* @throws Exception
391	*/
392	private void execute(IEolExecutableModule module, String logModelLocation)
393	throws Exception {
394	IModel logModel = modelHandler.getLOGModel(logModelLocation, true, true);
395	module.getContext().getModelRepository().addModel(logModel);
396	module.execute();
397	logModel.dispose();
398	}
399
400	/**
401	* Loads the module from a given source
402	*
403	* @param source where the module is (e.g. eol script)
404	* @return
405	* @throws Exception
406	* @throws URISyntaxException
407	*/
408	private IEolExecutableModule loadModule(String source) throws Exception,
409	URISyntaxException {
410
411	IEolExecutableModule module = null;
412	if (source.endsWith("etl")) {
413	module = new EtlModule();
414	} else if (source.endsWith("eol")) {
415	module = new EolModule();
416	} else {
417
418	}
419
420	module.parse(modelHandler.getFile(source));
421
422	if (module.getParseProblems().size() > 0) {
423	Console.printerrln("Parse error occured...");
424	for (ParseProblem problem : module.getParseProblems()) {
425	System.err.println(problem.toString());
426	}
427	// System.exit(-1);
428	}
429
430	return module;
431	}
432
433	/**
434	* Helper method for registering the metamodels
435	* @throws Exception
436	*/
437	private void registerMetaModels() throws Exception {
438	String metaModelsPath = DECENTEpsilonModelHandler.metaPath;
439	File metaModelsLocation = new File(metaModelsPath);
440	for (File file : metaModelsLocation.listFiles()) {
441	if (file.getName().endsWith(".ecore")) {
442	EmfUtil.register(URI.createFileURI(file.getAbsolutePath()), EPackage.Registry.INSTANCE);
443	}
444	}
445	}
446
447	}

Note: See TracBrowser for help on using the repository browser.

Download in other formats:

Original Format