为什么输出文件中没有mapreduce矩阵和向量乘法？

xlpyo6sf  于 2021-07-13  发布在  Hadoop
关注(0)|答案(0)|浏览(334)

我想用mapreduce把一个1000x1000的矩阵和一个1000x1的向量相乘。显示为图片的输入文件由随机的单个数字变量组成，并用空格分隔。请在此处输入图像描述
我的代码如下：

  1. import java.io.IOException;
  2. import java.util.*;
  4. import org.apache.hadoop.fs.Path;
  5. import org.apache.hadoop.conf.*;
  6. import org.apache.hadoop.io.*;
  7. import org.apache.hadoop.mapreduce.*;
  8. import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
  9. import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
  10. import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
  11. import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
  12. import org.apache.hadoop.util.GenericOptionsParser;
  13. import org.apache.hadoop.mapreduce.lib.input.FileSplit;
  15. public class Matrix_Vector {
  17.     public static class MatrixMapper extends
  18.             Mapper<LongWritable, Text, Text, Text> {
  19.         private String flag = null;// name of the data set
  20.         private int rowNum = 1000;// row of matrix A
  21.         private int colNum = 1;//cols of matrix B (for vector is one)
  22.         private int rowIndexA = 1; // row index of matrix A
  23.         private int rowIndexB = 1; // row index of matrix B
  25.         @Override
  26.         protected void setup(Context context) throws IOException,
  27.                 InterruptedException {
  28.             flag = ((FileSplit) context.getInputSplit()).getPath().getName();// Get the file name
  29.         }
  31.         @Override
  32.         protected void map(LongWritable key, Text value, Context context)
  33.                 throws IOException, InterruptedException {
  34.             String[] tokens = value.toString().split(" ");
  35.             if ("matrix".equals(flag)) {
  36.                 for (int i = 1; i <= colNum; i++) {
  37.                     Text k = new Text(rowIndexA + "," + i);
  38.                     for (int j = 0; j < tokens.length; j++) {
  39.                         Text v = new Text("a," + (j + 1) + "," + tokens[j]);
  40.                         context.write(k, v);
  41.                     }
  42.                 }
  43.                 rowIndexA++;// move to next line
  44.             } else if ("vector".equals(flag)) {
  45.                 for (int i = 1; i <= rowNum; i++) {
  46.                     for (int j = 0; j < tokens.length; j++) {
  47.                         Text k = new Text(i + "," + (j + 1));
  48.                         Text v = new Text("b," + rowIndexB + "," + tokens[j]);
  49.                         context.write(k, v);
  50.                     }
  51.                 }
  52.                 rowIndexB++;// move to next line
  53.             }
  54.         }
  55.     }
  57.     public static class MatrixReducer extends
  58.             Reducer<Text, Text, Text, IntWritable> {
  59.         @Override
  60.         protected void reduce(Text key, Iterable<Text> values, Context context)
  61.                 throws IOException, InterruptedException {
  62.             Map<String, String> mapA = new HashMap<String, String>();
  63.             Map<String, String> mapB = new HashMap<String, String>();
  65.             for (Text value : values) {
  66.                 String[] val = value.toString().split(",");
  67.                 if ("a".equals(val[0])) {
  68.                     mapA.put(val[1], val[2]);
  69.                 } else if ("b".equals(val[0])) {
  70.                     mapB.put(val[1], val[2]);
  71.                 }
  72.             }
  74.             int result = 0;
  75.             Iterator<String> mKeys = mapA.keySet().iterator();
  76.             while (mKeys.hasNext()) {
  77.                 String mkey = mKeys.next();
  78.                 if (mapB.get(mkey) == null) {
  79.                     continue;
  80.                 }
  81.                 result += Integer.parseInt(mapA.get(mkey))
  82.                         * Integer.parseInt(mapB.get(mkey));
  83.             }
  84.             context.write(key, new IntWritable(result));
  85.         }
  86.     }
  88.     public static void main(String[] args) throws Exception {
  89.         Configuration conf = new Configuration();
  90.         //job start running
  91.         long main_start=System.currentTimeMillis();
  93.         String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();
  94.         if (otherArgs.length != 2) 
  95.         {
  96.           System.err.println("Usage: MatrixVector <in> <out>");
  97.           System.exit(2);
  98.         }
  100.         Job job = Job.getInstance(conf, "Matrix_Vector");
  101.         job.setJarByClass(Matrix_Vector.class);
  103.         job.setMapOutputKeyClass(Text.class);
  104.         job.setMapOutputValueClass(Text.class);
  106.         job.setMapperClass(MatrixMapper.class);
  107.         job.setReducerClass(MatrixReducer.class);
  109.         job.setOutputKeyClass(Text.class);
  110.         job.setOutputValueClass(Text.class);
  112.         FileInputFormat.addInputPath(job, new Path(otherArgs[0]));
  113.         FileOutputFormat.setOutputPath(job, new Path(otherArgs[1]));
  115.         job.setInputFormatClass(TextInputFormat.class);
  116.         job.setOutputFormatClass(TextOutputFormat.class);
  118.         if (job.waitForCompletion(true)) { 
  119.                long main_end=System.currentTimeMillis();      
  120.                System.out.println("The process is end");
  121.                System.out.println("Time of running is "+ " :  "+(main_end - main_start) +"  ms");
  122.                System.exit(0);
  123.            }else {
  124.                System.out.println("The process ended with false");
  125.            }
  127.     }
  128. }

这是我得到的结果，在我的输出路径中有一个成功文件和一个输出文件，但是在输出文件中没有任何内容，也可以从“bytes writed=0”中看到。我找不到我的代码逻辑哪里出错了，很抱歉结果显示为代码格式，我仍然在学习如何做更好的职位。

  1. 21/03/26 08:05:08 INFO mapreduce.Job: Job job_1616694898641_0007 completed successfully
  2. 21/03/26 08:05:10 INFO mapreduce.Job: Counters: 49
  3.         File System Counters
  4.                 FILE: Number of bytes read=6
  5.                 FILE: Number of bytes written=606314
  6.                 FILE: Number of read operations=0
  7.                 FILE: Number of large read operations=0
  8.                 FILE: Number of write operations=0
  9.                 HDFS: Number of bytes read=2002216
  10.                 HDFS: Number of bytes written=0
  11.                 HDFS: Number of read operations=9
  12.                 HDFS: Number of large read operations=0
  13.                 HDFS: Number of write operations=2
  14.         Job Counters
  15.                 Launched map tasks=2
  16.                 Launched reduce tasks=1
  17.                 Data-local map tasks=2
  18.                 Total time spent by all maps in occupied slots (ms)=102529
  19.                 Total time spent by all reduces in occupied slots (ms)=13848
  20.                 Total time spent by all map tasks (ms)=102529
  21.                 Total time spent by all reduce tasks (ms)=13848
  22.                 Total vcore-milliseconds taken by all map tasks=102529
  23.                 Total vcore-milliseconds taken by all reduce tasks=13848
  24.                 Total megabyte-milliseconds taken by all map tasks=104989696
  25.                 Total megabyte-milliseconds taken by all reduce tasks=14180352
  26.         Map-Reduce Framework
  27.                 Map input records=2000
  28.                 Map output records=0
  29.                 Map output bytes=0
  30.                 Map output materialized bytes=12
  31.                 Input split bytes=212
  32.                 Combine input records=0
  33.                 Combine output records=0
  34.                 Reduce input groups=0
  35.                 Reduce shuffle bytes=12
  36.                 Reduce input records=0
  37.                 Reduce output records=0
  38.                 Spilled Records=0
  39.                 Shuffled Maps =2
  40.                 Failed Shuffles=0
  41.                 Merged Map outputs=2
  42.                 GC time elapsed (ms)=658
  43.                 CPU time spent (ms)=4140
  44.                 Physical memory (bytes) snapshot=519413760
  45.                 Virtual memory (bytes) snapshot=6241820672
  46.                 Total committed heap usage (bytes)=263872512
  47.         Shuffle Errors
  48.                 BAD_ID=0
  49.                 CONNECTION=0
  50.                 IO_ERROR=0
  51.                 WRONG_LENGTH=0
  52.                 WRONG_MAP=0
  53.                 WRONG_REDUCE=0
  54.         File Input Format Counters
  55.                 Bytes Read=2002004
  56.         File Output Format Counters
  57.                 Bytes Written=0
  58. The process is end
  59. Time of running is  :  182011  ms

任何帮助都将不胜感激！
谢谢您，
文

hadoopmapreduceJavaScriptbigdata

暂无答案！

目前还没有任何答案，快来回答吧！

我来回答

相关问题

    查看更多

    热门标签

    更多
    JavaquerypythonNode开发语言requestUtil数据库Table后端算法LoggerMessageElementParser

    最新问答

    更多
    • 蜀ICP备13028337号-1 大数据知识库 https://www.saoniuhuo.com © All rights reserved
    • 本站内容来源互联网,如果侵犯您的权益请联系我们删除, 联系方式：448109455@qq.com