| 1 | package felix.io; |
| 2 | import edu.umd.cloud9.collection.XMLInputFormat; |
| 3 | |
| 4 | import java.io.IOException; |
| 5 | import java.util.ArrayList; |
| 6 | import java.util.Date; |
| 7 | import java.util.Iterator; |
| 8 | import java.util.StringTokenizer; |
| 9 | import org.apache.hadoop.streaming.StreamInputFormat; |
| 10 | import org.apache.hadoop.streaming.StreamXmlRecordReader; |
| 11 | |
| 12 | import org.apache.hadoop.conf.Configuration; |
| 13 | import org.apache.hadoop.conf.Configured; |
| 14 | import org.apache.hadoop.fs.FileSystem; |
| 15 | import org.apache.hadoop.fs.FileUtil; |
| 16 | import org.apache.hadoop.fs.Path; |
| 17 | import org.apache.hadoop.io.*; |
| 18 | import org.apache.hadoop.mapred.JobClient; |
| 19 | import org.apache.hadoop.mapred.OutputCollector; |
| 20 | import org.apache.hadoop.mapred.Reporter; |
| 21 | import org.apache.hadoop.mapred.lib.db.DBInputFormat; |
| 22 | |
| 23 | import org.apache.hadoop.mapreduce.Job; |
| 24 | import org.apache.hadoop.mapreduce.Mapper; |
| 25 | import org.apache.hadoop.mapreduce.Reducer; |
| 26 | import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; |
| 27 | import org.apache.hadoop.mapreduce.lib.input.TextInputFormat; |
| 28 | import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; |
| 29 | import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat; |
| 30 | |
| 31 | |
| 32 | import org.apache.hadoop.util.Tool; |
| 33 | import org.apache.hadoop.util.ToolRunner; |
| 34 | import org.python.core.PyArray; |
| 35 | import org.python.core.PyList; |
| 36 | import org.python.core.PyObject; |
| 37 | import org.python.core.PyString; |
| 38 | |
| 39 | import tuffy.util.ExceptionMan; |
| 40 | |
| 41 | import felix.thirdpart.XmlInputFormat; |
| 42 | import felix.util.FelixConfig; |
| 43 | |
| 44 | /** |
| 45 | * @deprecated |
| 46 | * @author czhang |
| 47 | * |
| 48 | */ |
| 49 | public class TestHadoop_0_20 extends Configured implements Tool{ |
| 50 | |
| 51 | |
| 52 | public static class Map extends Mapper<LongWritable, Text, Text, IntWritable>{ |
| 53 | |
| 54 | private Text word = new Text(); |
| 55 | public static PythonExecutor pyMap; |
| 56 | |
| 57 | public static String _inputvalue; |
| 58 | |
| 59 | @Override |
| 60 | public void setup(Mapper.Context contex) { |
| 61 | |
| 62 | pyMap = new PythonExecutor( |
| 63 | "_felix_donotusemyname_outkey=[]\n" + "_felix_donotusemyname_outvalue=[]\n"+ |
| 64 | contex.getConfiguration().get("pyMapScript")); |
| 65 | _inputvalue = contex.getConfiguration().get("mapinputvalue"); |
| 66 | |
| 67 | if(contex.getConfiguration().get("pyMapInitScript") != null){ |
| 68 | pyMap.execSingle(contex.getConfiguration().get("pyMapInitScript")); |
| 69 | } |
| 70 | |
| 71 | } |
| 72 | |
| 73 | public void map(LongWritable key, Text value, Mapper.Context context) throws IOException, InterruptedException { |
| 74 | |
| 75 | pyMap.set(_inputvalue, new PyString(value.toString())); |
| 76 | |
| 77 | pyMap.run(); |
| 78 | PyList outKey = (PyList) pyMap.get("_felix_donotusemyname_outkey"); |
| 79 | PyList outValue = (PyList) pyMap.get("_felix_donotusemyname_outvalue"); |
| 80 | |
| 81 | for(int i=0;i<outKey.__len__();i++){ |
| 82 | PyObject _key = outKey.__getitem__(i); |
| 83 | PyObject _value = outValue.__getitem__(i); |
| 84 | |
| 85 | context.write(new Text(""+_key), new Text(""+_value)); |
| 86 | // System.out.println(""+key + "\t" + value); |
| 87 | } |
| 88 | |
| 89 | } |
| 90 | |
| 91 | } |
| 92 | |
| 93 | public static class Reduce extends Reducer<Text, Text, Text, Text>{ |
| 94 | |
| 95 | public static PythonExecutor pyReducer; |
| 96 | |
| 97 | public static String _inputkey; |
| 98 | public static String _inputvalues; |
| 99 | |
| 100 | @Override |
| 101 | public void setup(Reducer.Context contex) { |
| 102 | pyReducer = new PythonExecutor( |
| 103 | "_felix_donotusemyname_outkey=[]\n" + "_felix_donotusemyname_outvalue=[]\n" + |
| 104 | contex.getConfiguration().get("pyReduceScript")); |
| 105 | _inputkey = contex.getConfiguration().get("reduceinputkey"); |
| 106 | _inputvalues = contex.getConfiguration().get("reduceinputvalues"); |
| 107 | |
| 108 | } |
| 109 | |
| 110 | protected void reduce(Text _key, Iterable _values, Context context) throws IOException, InterruptedException{ |
| 111 | |
| 112 | |
| 113 | ArrayList<PyString> args = new ArrayList<PyString>(); |
| 114 | Iterator it = _values.iterator(); |
| 115 | while(it.hasNext()){ |
| 116 | args.add(new PyString(it.next().toString())); |
| 117 | } |
| 118 | |
| 119 | PyList values = new PyList(args); |
| 120 | PyObject inputkey = new PyString(_key.toString()); |
| 121 | |
| 122 | pyReducer.set(_inputvalues, values); |
| 123 | pyReducer.set(_inputkey, inputkey); |
| 124 | |
| 125 | pyReducer.run(); |
| 126 | PyList outKey = (PyList) pyReducer.get("_felix_donotusemyname_outkey"); |
| 127 | PyList outValue = (PyList) pyReducer.get("_felix_donotusemyname_outvalue"); |
| 128 | |
| 129 | for(int i=0;i<outKey.__len__();i++){ |
| 130 | PyObject key = outKey.__getitem__(i); |
| 131 | PyObject value = outValue.__getitem__(i); |
| 132 | context.write(new Text(""+key), new Text(""+value)); |
| 133 | } |
| 134 | |
| 135 | } |
| 136 | |
| 137 | } |
| 138 | |
| 139 | public TestHadoop_0_20(String mapScript, String reduceScript){ |
| 140 | Map.pyMap = new PythonExecutor( |
| 141 | mapScript); |
| 142 | |
| 143 | Reduce.pyReducer = new PythonExecutor( |
| 144 | reduceScript); |
| 145 | } |
| 146 | |
| 147 | public static void executeHadoopProgram(String[] toPass) throws Exception{ |
| 148 | |
| 149 | String[] dirtoPass = toPass.clone(); |
| 150 | dirtoPass[2] += "_dir"; |
| 151 | int res = ToolRunner.run(new Configuration(), new TestHadoop_0_20(dirtoPass[3], dirtoPass[4]), |
| 152 | dirtoPass); |
| 153 | |
| 154 | } |
| 155 | |
| 156 | public static void post(String[] toPass) throws Exception{ |
| 157 | |
| 158 | Configuration fsconf = new Configuration(); |
| 159 | fsconf.set("fs.default.name", "hdfs://d-02.cs.wisc.edu:9000"); |
| 160 | FileSystem fileSystem = FileSystem.get(fsconf); |
| 161 | |
| 162 | String[] dirtoPass = toPass.clone(); |
| 163 | dirtoPass[2] += "_dir"; |
| 164 | |
| 165 | FileUtil.copyMerge(fileSystem, new Path(dirtoPass[2]), fileSystem, new Path(toPass[2]), true, |
| 166 | fsconf, ""); |
| 167 | |
| 168 | fileSystem.close(); |
| 169 | |
| 170 | } |
| 171 | |
| 172 | |
| 173 | |
| 174 | public static void main(String[] args) throws Exception{ |
| 175 | |
| 176 | String[] toPass = { |
| 177 | |
| 178 | "hdfs://d-02.cs.wisc.edu:9000/felixNE/Entity.db", |
| 179 | //"hdfs://d-02.cs.wisc.edu:9000/firstMapReduceOut42/part-00000", |
| 180 | |
| 181 | "hdfs://d-02.cs.wisc.edu:9000/firstMapReduceOut100", |
| 182 | |
| 183 | "for k in _input.split(' '):\n"+ |
| 184 | "\t_outkey.append(k)\n"+ |
| 185 | "\t_outvalue.append('1')\n", |
| 186 | |
| 187 | "_outkey.append(_inputkey)\n"+ |
| 188 | "_outvalue.append(len(_inputvalues))\n" |
| 189 | |
| 190 | }; |
| 191 | |
| 192 | TestHadoop_0_20.executeHadoopProgram(toPass); |
| 193 | |
| 194 | |
| 195 | } |
| 196 | |
| 197 | |
| 198 | @Override |
| 199 | public int run(String[] arg0) throws Exception { |
| 200 | |
| 201 | |
| 202 | Configuration conf = getConf(); |
| 203 | |
| 204 | if(!FelixConfig.hadoopLocal){ |
| 205 | conf.set("fs.defaultFS", "hdfs://d-02.cs.wisc.edu:9000"); |
| 206 | conf.set("mapred.job.tracker", "d-02.cs.wisc.edu:9001"); |
| 207 | } |
| 208 | conf.set("pyMapScript", arg0[3]); |
| 209 | conf.set("pyReduceScript", arg0[4]); |
| 210 | conf.set("mapinputvalue", arg0[5]); |
| 211 | conf.set("reduceinputkey", arg0[6]); |
| 212 | conf.set("reduceinputvalues", arg0[7]); |
| 213 | conf.set("pyMapInitScript", arg0[8]); |
| 214 | |
| 215 | conf.set("mapred.map.tasks", "30"); |
| 216 | conf.set("mapred.min.split.size", "1B"); |
| 217 | |
| 218 | if(arg0[0].equals("xml")){ |
| 219 | conf.set("xmlinput.start", arg0[9]); |
| 220 | conf.set("xmlinput.end", arg0[10]); |
| 221 | } |
| 222 | |
| 223 | Job job = new Job(conf, "Felix_Run_On" + (new Date()).toLocaleString()); |
| 224 | |
| 225 | job.setNumReduceTasks(30); |
| 226 | |
| 227 | job.setJarByClass(TestHadoop_0_20.class); |
| 228 | |
| 229 | job.setMapperClass(Map.class); |
| 230 | job.setReducerClass(Reduce.class); |
| 231 | |
| 232 | job.setOutputKeyClass(Text.class); |
| 233 | job.setOutputValueClass(Text.class); |
| 234 | |
| 235 | FileInputFormat.addInputPath(job, new Path(arg0[1])); |
| 236 | FileOutputFormat.setOutputPath(job, new Path(arg0[2])); |
| 237 | |
| 238 | |
| 239 | if(arg0[0].equals("xml")){ |
| 240 | //XML |
| 241 | |
| 242 | job.setInputFormatClass(XmlInputFormat.class); |
| 243 | job.setOutputFormatClass(TextOutputFormat.class); |
| 244 | |
| 245 | job.setOutputKeyClass(Text.class); |
| 246 | job.setOutputValueClass(Text.class); |
| 247 | |
| 248 | }else if(arg0[0].equals("standard")){ |
| 249 | |
| 250 | job.setInputFormatClass(TextInputFormat.class); |
| 251 | job.setOutputFormatClass(TextOutputFormat.class); |
| 252 | |
| 253 | job.setOutputKeyClass(Text.class); |
| 254 | job.setOutputValueClass(Text.class); |
| 255 | } |
| 256 | |
| 257 | job.waitForCompletion(true); |
| 258 | return 0; |
| 259 | } |
| 260 | |
| 261 | } |