在hadoop(旧api)中将变量传递给mapper和reducer

cczfrluj  于 2021-05-29  发布在  Hadoop
关注(0)|答案(0)|浏览(249)

我不是hadoop的Maven,我有以下问题。我有一个作业必须在hadoop版本为0.20.2的集群上运行。当我开始作业时,我指定了一些参数。其中的两个我想传递给mapper并减少类,因为我需要它。
我尝试了不同的解决方案,现在我的代码如下所示:

  1. package bigdata;
  2. import java.io.IOException;
  3. import java.util.ArrayList;
  4. import java.util.HashMap;
  5. import java.util.TreeMap;
  6. import org.apache.commons.math3.stat.regression.SimpleRegression;
  7. import org.apache.hadoop.conf.Configuration;
  8. import org.apache.hadoop.conf.Configured;
  9. import org.apache.hadoop.fs.Path;
  10. import org.apache.hadoop.io.LongWritable;
  11. import org.apache.hadoop.io.Text;
  12. import org.apache.hadoop.io.IntWritable;
  13. import org.apache.hadoop.mapred.JobConf;
  14. import org.apache.hadoop.mapred.JobConfigurable;
  15. import org.apache.hadoop.mapreduce.Job;
  16. import org.apache.hadoop.mapreduce.Mapper;
  17. import org.apache.hadoop.mapreduce.Reducer;
  18. import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
  19. import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
  20. import org.apache.hadoop.util.Tool;
  21. import org.apache.hadoop.util.ToolRunner;
  22. import com.vividsolutions.jts.geom.Geometry;
  23. import com.vividsolutions.jts.io.ParseException;
  24. import com.vividsolutions.jts.io.WKTReader;
  25. public class BoxCount extends Configured implements Tool{
  26. private static String mbr;
  27. private static double cs;
  28. public static class Map extends Mapper<LongWritable, Text, IntWritable, Text> implements JobConfigurable
  29. {
  30. public void configure(JobConf job) {
  31. mbr = job.get(mbr);
  32. cs = job.getDouble("cellSide", 0.1);
  33. }
  34. protected void setup(Context context)
  35. throws IOException, InterruptedException {
  36. // metodo in cui leggere l'MBR passato come parametro
  37. System.out.println("mbr: " + mbr + "\ncs: " + cs);
  38. // ...
  39. }
  40. public void map(LongWritable key, Text value, Context context)
  41. throws IOException, InterruptedException {
  42. // some code here
  43. }
  44. protected void cleanup(Context context) throws IOException, InterruptedException
  45. {
  46. // other code
  47. }
  48. }
  49. public static class Reduce extends Reducer<IntWritable,Text,IntWritable,IntWritable>implements JobConfigurable
  50. {
  51. private static String mbr;
  52. private static double cs;
  53. public void configure(JobConf job) {
  54. mbr = job.get(mbr);
  55. cs = job.getDouble("cellSide", 0.1);
  56. }
  57. protected void setup(Context context) throws IOException, InterruptedException
  58. {
  59. System.out.println("mbr: " + mbr + " cs: " + cs);
  60. }
  61. public void reduce(IntWritable key, Iterable<Text> values, Context context)
  62. throws IOException, InterruptedException {
  63. //the reduce code
  64. }
  65. @SuppressWarnings("unused")
  66. protected void cleanup(Context context)
  67. throws IOException, InterruptedException {
  68. // cleanup code
  69. }
  70. public BoxCount (String[] args) {
  71. if (args.length != 4) {
  72. // 0 1 2 3
  73. System.out.println("Usage: OneGrid <mbr (Rectangle: (xmin,ymin)-(xmax,ymax))> <cell_Side> <input_path> <output_path>");
  74. System.out.println("args.length = "+args.length);
  75. for(int i = 0; i< args.length;i++)
  76. System.out.println("args["+i+"]"+" = "+args[i]);
  77. System.exit(0);
  78. }
  79. this.numReducers = 1;
  80. //this.mbr = new String(args[0]);
  81. // this.mbr = "Rectangle: (0.01,0.01)-(99.99,99.99)";
  82. // per sierpinski_jts
  83. this.mbr = "Rectangle: (0.0,0.0)-(100.01,86.6125)";
  84. // per diagonale
  85. //this.mbr = "Rectangle: (1.5104351688932738,1.0787616413335854)-(99999.3453727045,99999.98043392139)";
  86. // per uniforme
  87. // this.mbr = "Rectangle: (0.3020720559407146,0.2163091760095974)-(99999.68881210628,99999.46079314972)";
  88. this.cellSide = Double.parseDouble(args[1]);
  89. this.inputPath = new Path(args[2]);
  90. this.outputDir = new Path(args[3]);
  91. // Ricalcola la cellSize in modo da ottenere
  92. // almeno minMunGriglie (10) griglie!
  93. Grid g = new Grid(mbr, cellSide);
  94. if ((this.cellSide*(Math.pow(2,minNumGriglie))) > g.width)
  95. this.cellSide = g.width/(Math.pow(2,minNumGriglie));
  96. }
  97. public static void main(String[] args) throws Exception {
  98. int res = ToolRunner.run(new Configuration(), new BoxCount(args), args);
  99. System.exit(res);
  100. }
  101. public int run(String[] args) throws Exception
  102. {
  103. // define new job instead of null using conf
  104. Configuration conf = getConf();
  105. @SuppressWarnings("deprecation")
  106. Job job = new Job(conf, "BoxCount");
  107. // conf.set("mapreduce.framework.name", "local");
  108. // conf.set("mapreduce.jobtracker.address", "local");
  109. // conf.set("fs.defaultFS","file:///");
  110. // passo il valore mbr per creare la griglia
  111. conf.set("mbr", mbr);
  112. // passo lato cella
  113. conf.setDouble("cellSide", cellSide);
  114. job.setJarByClass(BoxCount.class);
  115. // set job input format
  116. job.setInputFormatClass(TextInputFormat.class);
  117. // set map class and the map output key and value classes
  118. job.setMapOutputKeyClass(IntWritable.class);
  119. job.setMapOutputValueClass(Text.class);
  120. job.setMapperClass(Map.class);
  121. // set reduce class and the reduce output key and value classes
  122. job.setReducerClass(Reduce.class);
  123. // set job output format
  124. job.setOutputFormatClass(TextOutputFormat.class);
  125. // add the input file as job input (from HDFS) to the variable
  126. // inputFile
  127. TextInputFormat.setInputPaths(job, inputPath);
  128. // set the output path for the job results (to HDFS) to the variable
  129. // outputPath
  130. TextOutputFormat.setOutputPath(job, outputDir);
  131. // set the number of reducers using variable numberReducers
  132. job.setNumReduceTasks(numReducers);
  133. // set the jar class
  134. job.setJarByClass(BoxCount.class);
  135. return job.waitForCompletion(true) ? 0 : 1; // this will execute the job
  136. }
  137. }

但是作业没有运行。正确的解决方案是什么?

暂无答案!

目前还没有任何答案,快来回答吧!

相关问题