org.apache.hadoop.hbase.io.hfile.HFile.getWriterFactoryNoCache()方法的使用及代码示例

x33g5p2x  于2022-01-20 转载在 其他  
字(10.7k)|赞(0)|评价(0)|浏览(122)

本文整理了Java中org.apache.hadoop.hbase.io.hfile.HFile.getWriterFactoryNoCache()方法的一些代码示例，展示了HFile.getWriterFactoryNoCache()的具体用法。这些代码示例主要来源于Github/Stackoverflow/Maven等平台，是从一些精选项目中提取出来的代码，具有较强的参考意义，能在一定程度帮忙到你。HFile.getWriterFactoryNoCache()方法的具体详情如下：
包路径：org.apache.hadoop.hbase.io.hfile.HFile
类名称：HFile
方法名：getWriterFactoryNoCache

HFile.getWriterFactoryNoCache介绍

[英]Returns the factory to be used to create HFile writers. Disables block cache access for all writers created through the returned factory.
[中]返回用于创建HFile写入程序的工厂。禁用通过返回工厂创建的所有写入程序的块缓存访问。

代码示例

代码示例来源：origin: apache/hbase

public static void doSmokeTest(FileSystem fs, Path path, String codec)
throws Exception {
 Configuration conf = HBaseConfiguration.create();
 HFileContext context = new HFileContextBuilder()
             .withCompression(HFileWriterImpl.compressionByName(codec)).build();
 HFile.Writer writer = HFile.getWriterFactoryNoCache(conf)
   .withPath(fs, path)
   .withFileContext(context)
   .create();
 // Write any-old Cell...
 final byte [] rowKey = Bytes.toBytes("compressiontestkey");
 Cell c = CellUtil.createCell(rowKey, Bytes.toBytes("compressiontestval"));
 writer.append(c);
 writer.appendFileInfo(Bytes.toBytes("compressioninfokey"), Bytes.toBytes("compressioninfoval"));
 writer.close();
 Cell cc = null;
 HFile.Reader reader = HFile.createReader(fs, path, CacheConfig.DISABLED, true, conf);
 try {
  reader.loadFileInfo();
  HFileScanner scanner = reader.getScanner(false, true);
  scanner.seekTo(); // position to the start of file
  // Scanner does not do Cells yet. Do below for now till fixed.
  cc = scanner.getCell();
  if (CellComparator.getInstance().compareRows(c, cc) != 0) {
   throw new Exception("Read back incorrect result: " + c.toString() + " vs " + cc.toString());
  }
 } finally {
  reader.close();
 }
}

代码示例来源：origin: apache/hbase

fs.mkdirs(hfilePath);
Path path = new Path(pathStr);
HFile.WriterFactory wf = HFile.getWriterFactoryNoCache(TEST_UTIL.getConfiguration());
Assert.assertNotNull(wf);
HFileContext context = new HFileContext();

代码示例来源：origin: apache/hbase

private String createHFileForFamilies(Path testPath, byte[] family,
  byte[] valueBytes) throws IOException {
 HFile.WriterFactory hFileFactory = HFile.getWriterFactoryNoCache(TEST_UTIL.getConfiguration());
 // TODO We need a way to do this without creating files
 Path testFile = new Path(testPath, TEST_UTIL.getRandomUUID().toString());
 FSDataOutputStream out = TEST_UTIL.getTestFileSystem().create(testFile);
 try {
  hFileFactory.withOutputStream(out);
  hFileFactory.withFileContext(new HFileContext());
  HFile.Writer writer = hFileFactory.create();
  try {
   writer.append(new KeyValue(CellUtil.createCell(valueBytes, family, valueBytes, 0L,
    KeyValue.Type.Put.getCode(), valueBytes)));
  } finally {
   writer.close();
  }
 } finally {
  out.close();
 }
 return testFile.toString();
}

代码示例来源：origin: apache/hbase

private String createHFileForFamilies(byte[] family) throws IOException {
 HFile.WriterFactory hFileFactory = HFile.getWriterFactoryNoCache(conf);
 // TODO We need a way to do this without creating files
 File hFileLocation = testFolder.newFile();
 FSDataOutputStream out = new FSDataOutputStream(new FileOutputStream(hFileLocation), null);
 try {
  hFileFactory.withOutputStream(out);
  hFileFactory.withFileContext(new HFileContext());
  HFile.Writer writer = hFileFactory.create();
  try {
   writer.append(new KeyValue(CellUtil.createCell(randomBytes,
     family,
     randomBytes,
     0L,
     KeyValue.Type.Put.getCode(),
     randomBytes)));
  } finally {
   writer.close();
  }
 } finally {
  out.close();
 }
 return hFileLocation.getAbsoluteFile().getAbsolutePath();
}

代码示例来源：origin: apache/hbase

@Override
void setUp() throws Exception {
 HFileContextBuilder builder = new HFileContextBuilder()
   .withCompression(HFileWriterImpl.compressionByName(codec))
   .withBlockSize(RFILE_BLOCKSIZE);
 
 if (cipher == "aes") {
  byte[] cipherKey = new byte[AES.KEY_LENGTH];
  new SecureRandom().nextBytes(cipherKey);
  builder.withEncryptionContext(Encryption.newContext(conf)
    .setCipher(Encryption.getCipher(conf, cipher))
    .setKey(cipherKey));
 } else if (!"none".equals(cipher)) {
  throw new IOException("Cipher " + cipher + " not supported.");
 }
 
 HFileContext hFileContext = builder.build();
 writer = HFile.getWriterFactoryNoCache(conf)
   .withPath(fs, mf)
   .withFileContext(hFileContext)
   .withComparator(CellComparator.getInstance())
   .create();
}

代码示例来源：origin: apache/hbase

.withCompression(HFileWriterImpl.compressionByName(options.compress))
           .build();
Writer writer = HFile.getWriterFactoryNoCache(conf)
  .withOutputStream(fout)
  .withFileContext(context)

代码示例来源：origin: apache/hbase

Path makeNewFile(TagUsage tagUsage) throws IOException {
 Path ncTFile = new Path(TEST_UTIL.getDataTestDir(), "basic.hfile");
 FSDataOutputStream fout = TEST_UTIL.getTestFileSystem().create(ncTFile);
 int blocksize = toKV("a", tagUsage).getLength() * 3;
 HFileContext context = new HFileContextBuilder().withBlockSize(blocksize)
   .withDataBlockEncoding(encoding)
   .withIncludesTags(true).build();
 Configuration conf = TEST_UTIL.getConfiguration();
 HFile.Writer writer = HFile.getWriterFactoryNoCache(conf).withOutputStream(fout)
   .withFileContext(context)
   .withComparator(CellComparatorImpl.COMPARATOR).create();
 // 4 bytes * 3 * 2 for each key/value +
 // 3 for keys, 15 for values = 42 (woot)
 writer.append(toKV("c", tagUsage));
 writer.append(toKV("e", tagUsage));
 writer.append(toKV("g", tagUsage));
 // block transition
 writer.append(toKV("i", tagUsage));
 writer.append(toKV("k", tagUsage));
 writer.close();
 fout.close();
 return ncTFile;
}

代码示例来源：origin: co.cask.hbase/hbase

public static void doSmokeTest(FileSystem fs, Path path, String codec)
throws Exception {
 Configuration conf = HBaseConfiguration.create();
 HFile.Writer writer = HFile.getWriterFactoryNoCache(conf)
   .withPath(fs, path)
   .withCompression(codec)
   .create();
 writer.append(Bytes.toBytes("testkey"), Bytes.toBytes("testval"));
 writer.appendFileInfo(Bytes.toBytes("infokey"), Bytes.toBytes("infoval"));
 writer.close();
 HFile.Reader reader = HFile.createReader(fs, path, new CacheConfig(conf));
 reader.loadFileInfo();
 byte[] key = reader.getFirstKey();
 boolean rc = Bytes.toString(key).equals("testkey");
 reader.close();
 if (!rc) {
  throw new Exception("Read back incorrect result: " +
            Bytes.toStringBinary(key));
 }
}

代码示例来源：origin: harbby/presto-connectors

public static void doSmokeTest(FileSystem fs, Path path, String codec)
throws Exception {
 Configuration conf = HBaseConfiguration.create();
 HFileContext context = new HFileContextBuilder()
             .withCompression(AbstractHFileWriter.compressionByName(codec)).build();
 HFile.Writer writer = HFile.getWriterFactoryNoCache(conf)
   .withPath(fs, path)
   .withFileContext(context)
   .create();
 // Write any-old Cell...
 final byte [] rowKey = Bytes.toBytes("compressiontestkey");
 Cell c = CellUtil.createCell(rowKey, Bytes.toBytes("compressiontestval"));
 writer.append(c);
 writer.appendFileInfo(Bytes.toBytes("compressioninfokey"), Bytes.toBytes("compressioninfoval"));
 writer.close();
 Cell cc = null;
 HFile.Reader reader = HFile.createReader(fs, path, new CacheConfig(conf), conf);
 try {
  reader.loadFileInfo();
  HFileScanner scanner = reader.getScanner(false, true);
  scanner.seekTo(); // position to the start of file
  // Scanner does not do Cells yet. Do below for now till fixed.
  cc = scanner.getKeyValue();
  if (CellComparator.compareRows(c, cc) != 0) {
   throw new Exception("Read back incorrect result: " + c.toString() + " vs " + cc.toString());
  }
 } finally {
  reader.close();
 }
}

代码示例来源：origin: org.apache.hbase/hbase-server

fs.mkdirs(hfilePath);
Path path = new Path(pathStr);
HFile.WriterFactory wf = HFile.getWriterFactoryNoCache(TEST_UTIL.getConfiguration());
Assert.assertNotNull(wf);
HFileContext context = new HFileContext();

代码示例来源：origin: org.apache.hbase/hbase-server

private String createHFileForFamilies(byte[] family) throws IOException {
 HFile.WriterFactory hFileFactory = HFile.getWriterFactoryNoCache(conf);
 // TODO We need a way to do this without creating files
 File hFileLocation = testFolder.newFile();
 FSDataOutputStream out = new FSDataOutputStream(new FileOutputStream(hFileLocation), null);
 try {
  hFileFactory.withOutputStream(out);
  hFileFactory.withFileContext(new HFileContext());
  HFile.Writer writer = hFileFactory.create();
  try {
   writer.append(new KeyValue(CellUtil.createCell(randomBytes,
     family,
     randomBytes,
     0L,
     KeyValue.Type.Put.getCode(),
     randomBytes)));
  } finally {
   writer.close();
  }
 } finally {
  out.close();
 }
 return hFileLocation.getAbsoluteFile().getAbsolutePath();
}

代码示例来源：origin: org.apache.hbase/hbase-server

private String createHFileForFamilies(Path testPath, byte[] family,
  byte[] valueBytes) throws IOException {
 HFile.WriterFactory hFileFactory = HFile.getWriterFactoryNoCache(TEST_UTIL.getConfiguration());
 // TODO We need a way to do this without creating files
 Path testFile = new Path(testPath, TEST_UTIL.getRandomUUID().toString());
 FSDataOutputStream out = TEST_UTIL.getTestFileSystem().create(testFile);
 try {
  hFileFactory.withOutputStream(out);
  hFileFactory.withFileContext(new HFileContext());
  HFile.Writer writer = hFileFactory.create();
  try {
   writer.append(new KeyValue(CellUtil.createCell(valueBytes, family, valueBytes, 0L,
    KeyValue.Type.Put.getCode(), valueBytes)));
  } finally {
   writer.close();
  }
 } finally {
  out.close();
 }
 return testFile.toString();
}

代码示例来源：origin: org.apache.hbase/hbase-server

@Override
void setUp() throws Exception {
 HFileContextBuilder builder = new HFileContextBuilder()
   .withCompression(HFileWriterImpl.compressionByName(codec))
   .withBlockSize(RFILE_BLOCKSIZE);
 
 if (cipher == "aes") {
  byte[] cipherKey = new byte[AES.KEY_LENGTH];
  new SecureRandom().nextBytes(cipherKey);
  builder.withEncryptionContext(Encryption.newContext(conf)
    .setCipher(Encryption.getCipher(conf, cipher))
    .setKey(cipherKey));
 } else if (!"none".equals(cipher)) {
  throw new IOException("Cipher " + cipher + " not supported.");
 }
 
 HFileContext hFileContext = builder.build();
 writer = HFile.getWriterFactoryNoCache(conf)
   .withPath(fs, mf)
   .withFileContext(hFileContext)
   .withComparator(CellComparator.getInstance())
   .create();
}

代码示例来源：origin: org.apache.hbase/hbase-server

.withCompression(HFileWriterImpl.compressionByName(options.compress))
           .build();
Writer writer = HFile.getWriterFactoryNoCache(conf)
  .withOutputStream(fout)
  .withFileContext(context)

代码示例来源：origin: org.apache.hbase/hbase-server

Path makeNewFile(TagUsage tagUsage) throws IOException {
 Path ncTFile = new Path(TEST_UTIL.getDataTestDir(), "basic.hfile");
 FSDataOutputStream fout = TEST_UTIL.getTestFileSystem().create(ncTFile);
 int blocksize = toKV("a", tagUsage).getLength() * 3;
 HFileContext context = new HFileContextBuilder().withBlockSize(blocksize)
   .withDataBlockEncoding(encoding)
   .withIncludesTags(true).build();
 Configuration conf = TEST_UTIL.getConfiguration();
 HFile.Writer writer = HFile.getWriterFactoryNoCache(conf).withOutputStream(fout)
   .withFileContext(context)
   .withComparator(CellComparatorImpl.COMPARATOR).create();
 // 4 bytes * 3 * 2 for each key/value +
 // 3 for keys, 15 for values = 42 (woot)
 writer.append(toKV("c", tagUsage));
 writer.append(toKV("e", tagUsage));
 writer.append(toKV("g", tagUsage));
 // block transition
 writer.append(toKV("i", tagUsage));
 writer.append(toKV("k", tagUsage));
 writer.close();
 fout.close();
 return ncTFile;
}

内容来源于网络，如有侵权，请联系作者删除！

HFile

相关文章

    查看更多

    热门标签

    更多
    JavaquerypythonNode开发语言requestUtil数据库Table后端算法LoggerMessageElementParser

    最新文章

    更多
    • 蜀ICP备13028337号-1 大数据知识库 https://www.saoniuhuo.com © All rights reserved
    • 本站内容来源互联网,如果侵犯您的权益请联系我们删除, 联系方式：448109455@qq.com