无法在hadoop上运行word count谢谢

oxcyiej7  于 2021-05-29  发布在  Hadoop
关注(0)|答案(1)|浏览(383)

我尝试在eclipse中运行hadoop字数统计。但它有问题;它甚至不能被调试。

package test;

import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;

import test.test.Map2.Combine;

public class test {

public static class Map2 extends Mapper<LongWritable, Text, Text, Text> {
  public void map(LongWritable key, Text value, Context context) 
            throws IOException, InterruptedException {
        String line = value.toString();
       String values=line.split(" ")[0]+"\\|"+line.split(" ")[1];
        context.write(new Text(" "),new Text(values));
        }
    //method reduce start
    public static class Combine extends Reducer<Text, Text, Text, IntWritable> {
    ArrayList<String> top5array= new ArrayList<String>();
    public void reduce(Text key, Iterable<Text> values, Context context) 
      throws IOException, InterruptedException {

        //arraylist    
        while(top5array.get(4)==null)
        {
            top5array.add(values.iterator().next().toString());
        }

     while(values.iterator().hasNext())
     {
       String  currentValues=values.iterator().next().toString();
       String currentkey=currentValues.split("\\|")[0];
       Integer currentnum=Integer.parseInt(currentValues.split("\\|")[1]);

      for(int i=0;i<5;i++)
         {
         Integer numofArray = Integer.parseInt(top5array.get(i).split("\\|")[1]);
             if(top5array.get(i) != null && currentnum < numofArray)
                {
                 break;
                 }
                 if(i == 4)
                         {
                         String currentKeyValuePair = currentkey + currentnum.toString();
                         top5array.add(5, currentKeyValuePair);
                         Collections.sort(top5array);
                         top5array.remove(0);
                         }
          }// for  end
      }// while end
    }//method reduce end
 } // Combine end
 }
 // map end

 public static class Reduce2 extends Reducer<Text, Text, Text, Text> {
 ArrayList<String> top5array= new ArrayList<String>();
    public void reduce(Text key, Iterable<Text> values, Context context) 
      throws IOException, InterruptedException {

        while(top5array.get(4)==null)
        {
            top5array.add(values.iterator().next().toString());
        }

     while(values.iterator().hasNext())
     {
       String  currentValues=values.iterator().next().toString();
       String currentkey=currentValues.split("\\|")[0];
       Integer currentnum=Integer.parseInt(currentValues.split("\\|")[1]);

      for(int i=0;i<5;i++)
         {
         Integer numofArray = Integer.parseInt(top5array.get(i).split("\\|")[1]);
             if(top5array.get(i) != null && currentnum < numofArray)
                {
                 break;
                 }
                 if(i == 4)
                         {
                         String currentKeyValuePair = currentkey + currentnum.toString();
                         top5array.add(5, currentKeyValuePair);
                         Collections.sort(top5array);
                         top5array.remove(0);
                         }
          }
      }
        String top5StringConca = ""; 
        for(int i=0; i < 5; i++){
        top5StringConca = top5StringConca + top5array.get(i);
        }
        context.write(new Text(" "), new Text(top5StringConca));
    }
 } 

 //the second of mapreduce  end

 public static void main(String[] args) throws Exception {
   Configuration conf = new Configuration();
   Job job = Job.getInstance(conf);
   job.setOutputKeyClass(Text.class);
   job.setOutputValueClass(Text.class);
   job.setMapperClass(Map2.class);
   job.setReducerClass(Reduce2.class);
   job.setCombinerClass(Combine.class);
   job.setInputFormatClass(TextInputFormat.class);
   job.setOutputFormatClass(TextOutputFormat.class);

   FileInputFormat.addInputPath(job, new Path(args[0]));
   FileOutputFormat.setOutputPath(job, new Path(args[1]));
   job.waitForCompletion(true);

 }

}

运行时出现的问题显示以下异常:

WARN  [main] util.NativeCodeLoader (NativeCodeLoader.java:<clinit>(62))
 - Unable to load native-hadoop library for your platform using builtin-java classes where applicable`

我怎样才能解决这个问题?

cnh2zyt3

cnh2zyt31#

在项目中添加hadoopjar。
如果已经配置了hadoop,那么可以将hdfs指向eclipse内部。为此,您必须包含依赖项。
在pom.xml中添加hadoop依赖项(如果您使用的是maven)。同时为eclipse添加第三方插件。这是导游。这些将在eclipse中启用map reduce透视图。我在项目中添加了以下依赖项:

<dependency>
    <groupId>org.apache.hadoop</groupId>
    <artifactId>hadoop-core</artifactId>
    <version>1.2.1</version>
    <scope>compile</scope>
</dependency>

<dependency>
    <groupId>org.apache.hadoop</groupId>
    <artifactId>hadoop-common</artifactId>
    <version>2.6.0</version>
</dependency>

<dependency>
    <groupId>org.apache.hadoop</groupId>
    <artifactId>hadoop-client</artifactId>
    <version>2.6.0</version>
</dependency>

您将看到依赖项本身包含hadoopjar。现在将取决于您是要使用jar提供的现有配置还是默认配置。
现在试着运行hadoop驱动程序类。您可以轻松地在eclipse中调试代码。现在您的hadoop透视图也被启用了。您可以在这里添加hdfs路径。
您也可以检查这个以进行远程调试。

相关问题