diff options
Diffstat (limited to 'sandbox/ctrezzo/Map-Reduce-Java/src')
13 files changed, 619 insertions, 0 deletions
diff --git a/sandbox/ctrezzo/Map-Reduce-Java/src/main/java/services/IntermediateCollector.java b/sandbox/ctrezzo/Map-Reduce-Java/src/main/java/services/IntermediateCollector.java new file mode 100644 index 0000000000..df5a405ea0 --- /dev/null +++ b/sandbox/ctrezzo/Map-Reduce-Java/src/main/java/services/IntermediateCollector.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package services; + +import org.osoa.sca.annotations.Remotable; + +@Remotable +public interface IntermediateCollector { + + public void collect(String key, int value); + + public void reducePhase(); +} diff --git a/sandbox/ctrezzo/Map-Reduce-Java/src/main/java/services/IntermediateCollectorImpl.java b/sandbox/ctrezzo/Map-Reduce-Java/src/main/java/services/IntermediateCollectorImpl.java new file mode 100644 index 0000000000..5fd5661ea2 --- /dev/null +++ b/sandbox/ctrezzo/Map-Reduce-Java/src/main/java/services/IntermediateCollectorImpl.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package services; + +import java.util.Iterator; +import java.util.Vector; + +import org.osoa.sca.annotations.Reference; + +public class IntermediateCollectorImpl implements IntermediateCollector { + +private Vector<KeyValuePair> table = new Vector<KeyValuePair>(); + + @Reference + public Reducer myReducer; + + + public void collect(String key, int value) { + table.add(new KeyValuePair(key, value)); + } + + public void reducePhase() { + java.util.Collections.sort(table); + + Iterator<KeyValuePair> it = table.iterator(); + + //no data + if(!it.hasNext()) { + return; + } + + Vector<Integer> values = new Vector<Integer>(); + KeyValuePair last = it.next(); + values.add(last.value); + while(it.hasNext()) { + KeyValuePair curr = it.next(); + if(last.compareTo(curr) == 0) { + values.add(curr.value); + last = curr; + } + else { + myReducer.reduce(last.key, values.iterator()); + values.clear(); + values.add(curr.value); + last = curr; + + } + } + + myReducer.reduce(last.key, values.iterator()); + + } + +} diff --git a/sandbox/ctrezzo/Map-Reduce-Java/src/main/java/services/JobConf.java b/sandbox/ctrezzo/Map-Reduce-Java/src/main/java/services/JobConf.java new file mode 100644 index 0000000000..6d9eda5449 --- /dev/null +++ b/sandbox/ctrezzo/Map-Reduce-Java/src/main/java/services/JobConf.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package services; + +import org.osoa.sca.annotations.Remotable; + +@Remotable +public interface JobConf { + + public void run(); +} diff --git a/sandbox/ctrezzo/Map-Reduce-Java/src/main/java/services/JobConfImpl.java b/sandbox/ctrezzo/Map-Reduce-Java/src/main/java/services/JobConfImpl.java new file mode 100644 index 0000000000..98f0b6d946 --- /dev/null +++ b/sandbox/ctrezzo/Map-Reduce-Java/src/main/java/services/JobConfImpl.java @@ -0,0 +1,110 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package services; + +import java.io.BufferedReader; +import java.io.DataInputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.PrintWriter; +import java.util.Iterator; +import org.osoa.sca.annotations.Property; +import org.osoa.sca.annotations.Reference; + +public class JobConfImpl implements JobConf { + + @Property + public String input; + @Property + public String output; + @Reference + public Mapper myMapper; + @Reference + public IntermediateCollector iCollector; + @Reference + public OutputCollector oCollector; + + public void run() { + + FileInputStream fstream = null; + try { + fstream = new FileInputStream(input); + } + catch(FileNotFoundException e) { + System.out.println("ERROR: Input file does not exist."); + return; + } + + DataInputStream in = new DataInputStream(fstream); + BufferedReader br = new BufferedReader(new InputStreamReader(in)); + + //Read File Line By Line + String strLine = null; + int count = 1; + try { + strLine = br.readLine(); + } + catch(IOException e) { + System.out.println("ERROR: Input file can not be read."); + return; + } + while (strLine != null) { + myMapper.map(count, strLine); + count++; + + try { + strLine = br.readLine(); + } + catch(IOException e) { + System.out.println("ERROR: Input file can not be read."); + return; + } + } + + iCollector.reducePhase(); + + Iterator<KeyValuePair> it = oCollector.getCollection(); + + //Write to output file + + FileOutputStream oStream = null; + try { + oStream = new FileOutputStream(output); + } + catch(FileNotFoundException e) { + System.out.println("ERROR: Output file can not be written to."); + return; + } + + PrintWriter pw = new PrintWriter(oStream); + + while(it.hasNext()) { + KeyValuePair kv = it.next(); + pw.println(kv.key + " " + kv.value); + } + + pw.close(); + } + +} diff --git a/sandbox/ctrezzo/Map-Reduce-Java/src/main/java/services/JobSubmit.java b/sandbox/ctrezzo/Map-Reduce-Java/src/main/java/services/JobSubmit.java new file mode 100644 index 0000000000..1922d2a14b --- /dev/null +++ b/sandbox/ctrezzo/Map-Reduce-Java/src/main/java/services/JobSubmit.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package services; + +import java.io.File; + +import org.apache.tuscany.sca.host.embedded.SCADomain; + + +public class JobSubmit { + + /** + * @param args + */ + public static void main(String[] args) { + + System.out.println("Starting ..."); + SCADomain scaDomain = SCADomain.newInstance("mapred.composite"); + JobConf myJob = scaDomain.getService(JobConf.class, "JobConf"); + System.out.println("Executing Map-Reduce Job."); + myJob.run(); + System.out.println("Map-Reduce Job Complete."); + System.out.println("Closing Domain ..."); + scaDomain.close(); + System.out.println(); + + + //Test code to check java classes without SCA + /* + JobConfImpl jConf = new JobConfImpl(); + MapperImpl myMap = new MapperImpl(); + IntermediateCollectorImpl iColl = new IntermediateCollectorImpl(); + ReducerImpl myRed = new ReducerImpl(); + OutputCollectorImpl oColl = new OutputCollectorImpl(); + + jConf.input = "/test/input/file01.txt"; + jConf.output = "/test/output/out.txt"; + jConf.myMapper = myMap; + jConf.oCollector = oColl; + jConf.iCollector = iColl; + myMap.iCollector = iColl; + iColl.myReducer = myRed; + myRed.oCollector = oColl; + + jConf.run(); + */ + + } + +} diff --git a/sandbox/ctrezzo/Map-Reduce-Java/src/main/java/services/KeyValuePair.java b/sandbox/ctrezzo/Map-Reduce-Java/src/main/java/services/KeyValuePair.java new file mode 100644 index 0000000000..94d3e60d8a --- /dev/null +++ b/sandbox/ctrezzo/Map-Reduce-Java/src/main/java/services/KeyValuePair.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package services; + +public class KeyValuePair implements Comparable<KeyValuePair> { + + public String key; + public int value; + + public KeyValuePair() { + key = ""; + value = 0; + } + + public KeyValuePair(String k, int v) { + key = k; + value = v; + } + + public int compareTo(KeyValuePair o) { + + return key.compareTo(o.key); + } + +} diff --git a/sandbox/ctrezzo/Map-Reduce-Java/src/main/java/services/Mapper.java b/sandbox/ctrezzo/Map-Reduce-Java/src/main/java/services/Mapper.java new file mode 100644 index 0000000000..0f5f321eb9 --- /dev/null +++ b/sandbox/ctrezzo/Map-Reduce-Java/src/main/java/services/Mapper.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package services; + +import org.osoa.sca.annotations.Remotable; + +@Remotable +public interface Mapper { + + public void map(int key, String value); +} diff --git a/sandbox/ctrezzo/Map-Reduce-Java/src/main/java/services/MapperImpl.java b/sandbox/ctrezzo/Map-Reduce-Java/src/main/java/services/MapperImpl.java new file mode 100644 index 0000000000..be7e91ebd3 --- /dev/null +++ b/sandbox/ctrezzo/Map-Reduce-Java/src/main/java/services/MapperImpl.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package services; + +import java.util.StringTokenizer; +import org.osoa.sca.annotations.Reference; + +public class MapperImpl implements Mapper { + + @Reference + public IntermediateCollector iCollector2; + + public void map(int key, String value) { + + StringTokenizer itr = new StringTokenizer(value); + while (itr.hasMoreTokens()) { + iCollector2.collect(itr.nextToken(), 1); + } + } + +} diff --git a/sandbox/ctrezzo/Map-Reduce-Java/src/main/java/services/OutputCollector.java b/sandbox/ctrezzo/Map-Reduce-Java/src/main/java/services/OutputCollector.java new file mode 100644 index 0000000000..d3b7930ed3 --- /dev/null +++ b/sandbox/ctrezzo/Map-Reduce-Java/src/main/java/services/OutputCollector.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package services; + +import java.util.Iterator; +import org.osoa.sca.annotations.Remotable; + +@Remotable +public interface OutputCollector { + + public void collect(String key, int value); + + public Iterator<KeyValuePair> getCollection(); +} diff --git a/sandbox/ctrezzo/Map-Reduce-Java/src/main/java/services/OutputCollectorImpl.java b/sandbox/ctrezzo/Map-Reduce-Java/src/main/java/services/OutputCollectorImpl.java new file mode 100644 index 0000000000..02fdbf58e4 --- /dev/null +++ b/sandbox/ctrezzo/Map-Reduce-Java/src/main/java/services/OutputCollectorImpl.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package services; + +import java.util.Vector; +import java.util.Iterator; + +public class OutputCollectorImpl implements OutputCollector { + + private Vector<KeyValuePair> table = new Vector<KeyValuePair>(); + private boolean sorted = false; + + + public void collect(String key, int value) { + + table.add(new KeyValuePair(key, value)); + sorted = false; + } + + public Iterator<KeyValuePair> getCollection() { + + if(!sorted) { + java.util.Collections.sort(table); + sorted = true; + } + + return table.iterator(); + } + +} diff --git a/sandbox/ctrezzo/Map-Reduce-Java/src/main/java/services/Reducer.java b/sandbox/ctrezzo/Map-Reduce-Java/src/main/java/services/Reducer.java new file mode 100644 index 0000000000..375a6f146c --- /dev/null +++ b/sandbox/ctrezzo/Map-Reduce-Java/src/main/java/services/Reducer.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package services; + +import java.util.Iterator; +import org.osoa.sca.annotations.Remotable; + +@Remotable +public interface Reducer { + + public void reduce(String key, Iterator<Integer> values); +} diff --git a/sandbox/ctrezzo/Map-Reduce-Java/src/main/java/services/ReducerImpl.java b/sandbox/ctrezzo/Map-Reduce-Java/src/main/java/services/ReducerImpl.java new file mode 100644 index 0000000000..9f733573a7 --- /dev/null +++ b/sandbox/ctrezzo/Map-Reduce-Java/src/main/java/services/ReducerImpl.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package services; + +import java.util.Iterator; +import org.osoa.sca.annotations.Reference; + +public class ReducerImpl implements Reducer { + + @Reference + public OutputCollector oCollector2; + + public void reduce(String key, Iterator<Integer> values) { + int sum = 0; + while (values.hasNext()) { + sum += values.next().intValue(); + } + + oCollector2.collect(key, sum); + } + +} diff --git a/sandbox/ctrezzo/Map-Reduce-Java/src/main/resources/mapred.composite b/sandbox/ctrezzo/Map-Reduce-Java/src/main/resources/mapred.composite new file mode 100644 index 0000000000..e465ae4fb0 --- /dev/null +++ b/sandbox/ctrezzo/Map-Reduce-Java/src/main/resources/mapred.composite @@ -0,0 +1,59 @@ +<?xml version="1.0" encoding="UTF-8"?> + +<!-- + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. +--> + + +<composite xmlns="http://www.osoa.org/xmlns/sca/1.0" + xmlns:t="http://tuscany.apache.org/xmlns/sca/1.0" + targetNamespace="http://mapred" + name="mapred"> + + <component name="JobConf"> + <implementation.java class="services.JobConfImpl"/> + + <property name="input">/test/input/file01.txt</property> + <property name="output">/test/output/out.txt</property> + + <reference name="myMapper" target="MapperComponent"/> + <reference name="iCollector" target="IntermediateCollectorComponent"/> + <reference name="oCollector" target="OutputCollectorComponent"/> + </component> + + <component name="MapperComponent"> + <implementation.java class="services.MapperImpl"/> + <reference name="iCollector2" target="IntermediateCollectorComponent"/> + </component> + + <component name="IntermediateCollectorComponent"> + <implementation.java class="services.IntermediateCollectorImpl"/> + <reference name="myReducer" target="ReducerComponent"/> + </component> + + <component name="ReducerComponent"> + <implementation.java class="services.ReducerImpl"/> + <reference name="oCollector2" target="OutputCollectorComponent"/> + </component> + + <component name="OutputCollectorComponent"> + <implementation.java class="services.OutputCollectorImpl"/> + </component> + + +</composite>
\ No newline at end of file |