我正在尝试从map reduce作业编写一个snappy块压缩序列文件。我使用的是hadoop2.0.0-cdh4.5.0和snappyjava1.0.4.1
这是我的密码:
package jinvestor.jhouse.mr;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.util.Arrays;
import java.util.List;
import jinvestor.jhouse.core.House;
import jinvestor.jhouse.core.util.HouseAvroUtil;
import jinvestor.jhouse.download.HBaseHouseDAO;
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.LocatedFileStatus;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.RemoteIterator;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil;
import org.apache.hadoop.hbase.mapreduce.TableMapper;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.SequenceFile;
import org.apache.hadoop.io.compress.CompressionCodec;
import org.apache.hadoop.io.compress.SnappyCodec;
import org.apache.hadoop.mapred.FileOutputFormat;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat;
import org.apache.hadoop.mapreduce.Job;
import org.apache.mahout.math.DenseVector;
import org.apache.mahout.math.NamedVector;
import org.apache.mahout.math.VectorWritable;
/**
* Produces mahout vectors from House entries in HBase.
*
* @author Michael Scott Knapp
*
*/
public class HouseVectorizer {
private final Configuration configuration;
private final House minimumHouse;
private final House maximumHouse;
public HouseVectorizer(final Configuration configuration,
final House minimumHouse, final House maximumHouse) {
this.configuration = configuration;
this.minimumHouse = minimumHouse;
this.maximumHouse = maximumHouse;
}
public void vectorize() throws IOException, ClassNotFoundException, InterruptedException {
JobConf jobConf = new JobConf();
jobConf.setMapOutputKeyClass(LongWritable.class);
jobConf.setMapOutputValueClass(VectorWritable.class);
// we want the vectors written straight to HDFS,
// the order does not matter.
jobConf.setNumReduceTasks(0);
Path outputDir = new Path("/home/cloudera/house_vectors");
FileSystem fs = FileSystem.get(configuration);
if (fs.exists(outputDir)) {
fs.delete(outputDir, true);
}
FileOutputFormat.setOutputPath(jobConf, outputDir);
// I want the mappers to know the max and min value
// so they can normalize the data.
// I will add them as properties in the configuration,
// by serializing them with avro.
String minmax = HouseAvroUtil.toBase64String(Arrays.asList(minimumHouse,
maximumHouse));
jobConf.set("minmax", minmax);
Job job = Job.getInstance(jobConf);
Scan scan = new Scan();
scan.addFamily(Bytes.toBytes("data"));
TableMapReduceUtil.initTableMapperJob("homes", scan,
HouseVectorizingMapper.class, LongWritable.class,
VectorWritable.class, job);
job.setOutputFormatClass(SequenceFileOutputFormat.class);
job.setOutputKeyClass(LongWritable.class);
job.setOutputValueClass(VectorWritable.class);
job.setMapOutputKeyClass(LongWritable.class);
job.setMapOutputValueClass(VectorWritable.class);
SequenceFileOutputFormat.setOutputCompressionType(job, SequenceFile.CompressionType.BLOCK);
SequenceFileOutputFormat.setOutputCompressorClass(job, SnappyCodec.class);
SequenceFileOutputFormat.setOutputPath(job, outputDir);
job.getConfiguration().setClass("mapreduce.map.output.compress.codec",
SnappyCodec.class,
CompressionCodec.class);
job.waitForCompletion(true);
}
当我运行它时,我得到:
java.lang.Exception: java.lang.UnsatisfiedLinkError: org.apache.hadoop.util.NativeCodeLoader.buildSupportsSnappy()Z
at org.apache.hadoop.mapred.LocalJobRunner$Job.run(LocalJobRunner.java:401)
Caused by: java.lang.UnsatisfiedLinkError: org.apache.hadoop.util.NativeCodeLoader.buildSupportsSnappy()Z
at org.apache.hadoop.util.NativeCodeLoader.buildSupportsSnappy(Native Method)
at org.apache.hadoop.io.compress.SnappyCodec.checkNativeCodeLoaded(SnappyCodec.java:62)
at org.apache.hadoop.io.compress.SnappyCodec.getCompressorType(SnappyCodec.java:127)
at org.apache.hadoop.io.compress.CodecPool.getCompressor(CodecPool.java:104)
at org.apache.hadoop.io.compress.CodecPool.getCompressor(CodecPool.java:118)
at org.apache.hadoop.io.SequenceFile$Writer.init(SequenceFile.java:1169)
at org.apache.hadoop.io.SequenceFile$Writer.<init>(SequenceFile.java:1080)
at org.apache.hadoop.io.SequenceFile$BlockCompressWriter.<init>(SequenceFile.java:1400)
at org.apache.hadoop.io.SequenceFile.createWriter(SequenceFile.java:274)
at org.apache.hadoop.io.SequenceFile.createWriter(SequenceFile.java:527)
at org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat.getSequenceWriter(SequenceFileOutputFormat.java:64)
at org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat.getRecordWriter(SequenceFileOutputFormat.java:75)
at org.apache.hadoop.mapred.MapTask$NewDirectOutputCollector.<init>(MapTask.java:617)
at org.apache.hadoop.mapred.MapTask.runNewMapper(MapTask.java:737)
at org.apache.hadoop.mapred.MapTask.run(MapTask.java:338)
at org.apache.hadoop.mapred.LocalJobRunner$Job$MapTaskRunnable.run(LocalJobRunner.java:233)
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471)
at java.util.concurrent.FutureTask.run(FutureTask.java:262)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at java.lang.Thread.run(Thread.java:744)
如果我注解掉这些行,那么我的测试通过了:
SequenceFileOutputFormat.setOutputCompressionType(job, SequenceFile.CompressionType.BLOCK);
SequenceFileOutputFormat.setOutputCompressorClass(job, SnappyCodec.class);
job.getConfiguration().setClass("mapreduce.map.output.compress.coded",
SnappyCodec.class,
CompressionCodec.class);
但是,我确实希望在序列文件中使用snappy压缩。有人能解释一下我做错了什么吗?
6条答案
按热度按时间nafvub8i1#
我需要所有的文件,不仅仅是*.so文件。理想情况下,您还可以将文件夹包含到路径中,而不是从那里复制libs。在此之后,您需要重新启动mapreduce服务,以便获取并使用新库。
尼科
mlmc2os52#
从cloudera社区中找到以下信息
确保ld\u library\u path和java\u library\u path包含包含libsnappy.so**文件的本机目录路径。
确保已在spark环境(spark env.sh)中导出ld_library_path和java_library path。
例如,我使用hortonworks hdp,在spark-env.sh中有以下配置
lbsnaicq3#
检查core-site.xml和mapred-site.xml它们应该包含正确的属性和带有库的文件夹路径
core-site.xml文件
mapred-site.xml文件
ld\u library\u path-必须包含libsnappy.so的路径。
wmomyfyw4#
我的问题是我的jre没有包含适当的本机库。这可能是因为我将jdk从cloudera的预构建vm切换到了jdk1.7。snappy.so文件位于hadoop/lib/native目录中,jre需要它们。将它们添加到类路径似乎并没有解决我的问题。我是这样决定的:
然后我就可以使用snappycodec类了。但你的道路可能不同。
这似乎让我想到了下一个问题:
原因:java.lang.runtimeexception:本机snappy库不可用:尚未加载snappycompressor。
还在努力解决这个问题。
svgewumm5#
在我的例子中,您可以检查配置单元conf文件:mapred-site.xml,并检查键:mapreduce.admin.user.env的值,
我在一个新的datanode中测试了它,在没有本机依赖项的机器上收到unlinked buildsnapy错误(libsnapy.so等)
z6psavjg6#
从windows\system32中删除hadoop.dll(我手动复制了它)并设置hadoop\u home=\hadoop-2.6.4之后,它就可以工作了!!!