我尝试在eclipse中运行hadoop字数统计。但它有问题;它甚至不能被调试。
package test;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
import test.test.Map2.Combine;
public class test {
public static class Map2 extends Mapper<LongWritable, Text, Text, Text> {
public void map(LongWritable key, Text value, Context context)
throws IOException, InterruptedException {
String line = value.toString();
String values=line.split(" ")[0]+"\\|"+line.split(" ")[1];
context.write(new Text(" "),new Text(values));
}
//method reduce start
public static class Combine extends Reducer<Text, Text, Text, IntWritable> {
ArrayList<String> top5array= new ArrayList<String>();
public void reduce(Text key, Iterable<Text> values, Context context)
throws IOException, InterruptedException {
//arraylist
while(top5array.get(4)==null)
{
top5array.add(values.iterator().next().toString());
}
while(values.iterator().hasNext())
{
String currentValues=values.iterator().next().toString();
String currentkey=currentValues.split("\\|")[0];
Integer currentnum=Integer.parseInt(currentValues.split("\\|")[1]);
for(int i=0;i<5;i++)
{
Integer numofArray = Integer.parseInt(top5array.get(i).split("\\|")[1]);
if(top5array.get(i) != null && currentnum < numofArray)
{
break;
}
if(i == 4)
{
String currentKeyValuePair = currentkey + currentnum.toString();
top5array.add(5, currentKeyValuePair);
Collections.sort(top5array);
top5array.remove(0);
}
}// for end
}// while end
}//method reduce end
} // Combine end
}
// map end
public static class Reduce2 extends Reducer<Text, Text, Text, Text> {
ArrayList<String> top5array= new ArrayList<String>();
public void reduce(Text key, Iterable<Text> values, Context context)
throws IOException, InterruptedException {
while(top5array.get(4)==null)
{
top5array.add(values.iterator().next().toString());
}
while(values.iterator().hasNext())
{
String currentValues=values.iterator().next().toString();
String currentkey=currentValues.split("\\|")[0];
Integer currentnum=Integer.parseInt(currentValues.split("\\|")[1]);
for(int i=0;i<5;i++)
{
Integer numofArray = Integer.parseInt(top5array.get(i).split("\\|")[1]);
if(top5array.get(i) != null && currentnum < numofArray)
{
break;
}
if(i == 4)
{
String currentKeyValuePair = currentkey + currentnum.toString();
top5array.add(5, currentKeyValuePair);
Collections.sort(top5array);
top5array.remove(0);
}
}
}
String top5StringConca = "";
for(int i=0; i < 5; i++){
top5StringConca = top5StringConca + top5array.get(i);
}
context.write(new Text(" "), new Text(top5StringConca));
}
}
//the second of mapreduce end
public static void main(String[] args) throws Exception {
Configuration conf = new Configuration();
Job job = Job.getInstance(conf);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(Text.class);
job.setMapperClass(Map2.class);
job.setReducerClass(Reduce2.class);
job.setCombinerClass(Combine.class);
job.setInputFormatClass(TextInputFormat.class);
job.setOutputFormatClass(TextOutputFormat.class);
FileInputFormat.addInputPath(job, new Path(args[0]));
FileOutputFormat.setOutputPath(job, new Path(args[1]));
job.waitForCompletion(true);
}
}
运行时出现的问题显示以下异常:
WARN [main] util.NativeCodeLoader (NativeCodeLoader.java:<clinit>(62))
- Unable to load native-hadoop library for your platform using builtin-java classes where applicable`
我怎样才能解决这个问题?
1条答案
按热度按时间cnh2zyt31#
在项目中添加hadoopjar。
如果已经配置了hadoop,那么可以将hdfs指向eclipse内部。为此,您必须包含依赖项。
在pom.xml中添加hadoop依赖项(如果您使用的是maven)。同时为eclipse添加第三方插件。这是导游。这些将在eclipse中启用map reduce透视图。我在项目中添加了以下依赖项:
您将看到依赖项本身包含hadoopjar。现在将取决于您是要使用jar提供的现有配置还是默认配置。
现在试着运行hadoop驱动程序类。您可以轻松地在eclipse中调试代码。现在您的hadoop透视图也被启用了。您可以在这里添加hdfs路径。
您也可以检查这个以进行远程调试。