运行从eclipse导出的jar时发生invalidinputexception

qnzebej0  于 2021-06-01  发布在  Hadoop
关注(0)|答案(1)|浏览(382)

我已经在centos7中安装了Hadoop2.6,运行良好。但是,当我运行从eclipse导出的jar时,会出现以下错误:

  1. [root@myspark ~]# hadoop jar fengcount.jar intput output1
  2. 17/05/26 21:24:51 INFO client.RMProxy: Connecting to ResourceManager
  3. at myspark/192.168.44.100:8032 17/05/26 21:24:53 INFO mapreduce.JobSubmitter: Cleaning up the staging area /tmp/hadoop-yarn/staging/root/.staging/job_1495765615548_0004 Exception in thread "main" org.apache.hadoop.mapreduce.lib.input.InvalidInputException: Input path does not exist: hdfs://myspark:54310/user/root/intput
  4. at org.apache.hadoop.mapreduce.lib.input.FileInputFormat.singleThreadedListStatus(FileInputFormat.java:321)
  5. at org.apache.hadoop.mapreduce.lib.input.FileInputFormat.listStatus(FileInputFormat.java:264)
  6. at org.apache.hadoop.mapreduce.lib.input.FileInputFormat.getSplits(FileInputFormat.java:385)
  7. at org.apache.hadoop.mapreduce.JobSubmitter.writeNewSplits(JobSubmitter.java:302)
  8. at org.apache.hadoop.mapreduce.JobSubmitter.writeSplits(JobSubmitter.java:319)
  9. at org.apache.hadoop.mapreduce.JobSubmitter.submitJobInternal(JobSubmitter.java:197)
  10. at org.apache.hadoop.mapreduce.Job$10.run(Job.java:1297)
  11. at org.apache.hadoop.mapreduce.Job$10.run(Job.java:1294)
  12. at java.security.AccessController.doPrivileged(Native Method)
  13. at javax.security.auth.Subject.doAs(Subject.java:422)
  14. at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1692)
  15. at org.apache.hadoop.mapreduce.Job.submit(Job.java:1294)
  16. at org.apache.hadoop.mapreduce.Job.waitForCompletion(Job.java:1315)
  17. at hdfs.hadoop_hdfs.fengcount.main(fengcount.java:39)
  18. at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
  19. at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
  20. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
  21. at java.lang.reflect.Method.invoke(Method.java:498)
  22. at org.apache.hadoop.util.RunJar.run(RunJar.java:221)
  23. at org.apache.hadoop.util.RunJar.main(RunJar.java:136)

文件input/test1.txt实际上存在:

  1. [root@myspark ~]# hdfs dfs -ls -R
  2. drwxr-xr-x - root supergroup 0 2017-05-26 21:02 input
  3. -rw-r--r-- 1 root supergroup 16 2017-05-24 01:57 input/test1.txt

我的代码:

  1. package hdfs.hadoop_hdfs;
  2. import java.io.IOException;
  3. import java.util.StringTokenizer;
  4. import org.apache.hadoop.conf.Configuration;
  5. import org.apache.hadoop.fs.Path;
  6. import org.apache.hadoop.io.IntWritable;
  7. import org.apache.hadoop.io.Text;
  8. import org.apache.hadoop.mapreduce.Job;
  9. import org.apache.hadoop.mapreduce.Mapper;
  10. import org.apache.hadoop.mapreduce.Reducer;
  11. import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
  12. import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
  13. import org.apache.hadoop.util.GenericOptionsParser;
  14. public class fengcount {
  15. public static void main(String[] args) throws IOException, ClassNotFoundException, InterruptedException {
  16. // TODO Auto-generated method stub
  17. Configuration conf=new Configuration();
  18. String[] otherargs=new GenericOptionsParser(conf,args).getRemainingArgs();
  19. if (otherargs.length!=2) {
  20. System.err.println("Usage:fengcount<int><out>");
  21. System.exit(2);
  22. }
  23. @SuppressWarnings("deprecation")
  24. Job job=new Job(conf, "fengcount");
  25. job.setJarByClass(fengcount.class);
  26. job.setMapperClass(TokerizerMapper.class);
  27. job.setCombinerClass(IntSumReducer.class);
  28. job.setReducerClass(IntSumReducer.class);
  29. job.setOutputKeyClass(Text.class);
  30. job.setOutputValueClass(IntWritable.class);
  31. FileInputFormat.addInputPath(job, new Path(otherargs[0]));
  32. FileOutputFormat.setOutputPath(job, new Path(otherargs[1]));
  33. System.exit(job.waitForCompletion(true)?0:1);
  34. }
  35. // mapper class
  36. public static class TokerizerMapper extends Mapper<Object, Text, Text, IntWritable> {
  37. private final static IntWritable one = new IntWritable(1);
  38. private Text word = new Text();
  39. @Override
  40. public void map(Object key, Text value, Context context) throws IOException, InterruptedException {
  41. // TODO Auto-generated method stub
  42. System.out.println("key=" + key.toString());
  43. System.out.println("value=" + value.toString());
  44. StringTokenizer itr = new StringTokenizer(value.toString());
  45. while (itr.hasMoreTokens()) {
  46. word.set(itr.nextToken());
  47. context.write(word, one);
  48. }
  49. }
  50. }
  51. //reduce process
  52. public static class IntSumReducer extends Reducer<Text, IntWritable, Text, IntWritable> {
  53. private IntWritable result = new IntWritable();
  54. @Override
  55. public void reduce(Text key, Iterable<IntWritable> values,
  56. Reducer<Text, IntWritable, Text, IntWritable>.Context context)
  57. throws IOException, InterruptedException {
  58. // TODO Auto-generated method stub
  59. int sum = 0;
  60. for (IntWritable val : values) {
  61. sum += val.get();
  62. }
  63. result.set(sum);
  64. context.write(key, result);
  65. }
  66. }
  67. //mapreduce process
  68. }
yvfmudvl

yvfmudvl1#

从错误日志中,我可以看到 hdfs://myspark:54310/user/root/intput ,我怀疑它不正确。我想这条路的尽头是 input .
祝你好运!

相关问题