org.apache.parquet.column.Encoding.usesDictionary()方法的使用及代码示例

x33g5p2x  于2022-01-19 转载在 其他  
字(10.8k)|赞(0)|评价(0)|浏览(110)

本文整理了Java中org.apache.parquet.column.Encoding.usesDictionary()方法的一些代码示例，展示了Encoding.usesDictionary()的具体用法。这些代码示例主要来源于Github/Stackoverflow/Maven等平台，是从一些精选项目中提取出来的代码，具有较强的参考意义，能在一定程度帮忙到你。Encoding.usesDictionary()方法的具体详情如下：
包路径：org.apache.parquet.column.Encoding
类名称：Encoding
方法名：usesDictionary

Encoding.usesDictionary介绍

暂无

代码示例

代码示例来源：origin: apache/hive

private void initDataReader(Encoding dataEncoding, ByteBufferInputStream in, int valueCount)
  throws IOException {
 this.pageValueCount = valueCount;
 this.endOfPageValueCount = valuesRead + pageValueCount;
 if (dataEncoding.usesDictionary()) {
  this.dataColumn = null;
  if (dictionary == null) {
   throw new IOException(
     "could not read page in col " + descriptor +
       " as the dictionary was missing for encoding " + dataEncoding);
  }
  dataColumn = ParquetDataColumnReaderFactory.getDataColumnReaderByType(type.asPrimitiveType(), hiveType,
    dataEncoding.getDictionaryBasedValuesReader(descriptor, VALUES, dictionary
      .getDictionary()), skipTimestampConversion);
  this.isCurrentPageDictionaryEncoded = true;
 } else {
  dataColumn = ParquetDataColumnReaderFactory.getDataColumnReaderByType(type.asPrimitiveType(), hiveType,
    dataEncoding.getValuesReader(descriptor, VALUES), skipTimestampConversion);
  this.isCurrentPageDictionaryEncoded = false;
 }
 try {
  dataColumn.initFromPage(pageValueCount, in);
 } catch (IOException e) {
  throw new IOException("could not read page in col " + descriptor, e);
 }
}

代码示例来源：origin: org.apache.spark/spark-sql_2.11

private void initDataReader(Encoding dataEncoding, ByteBufferInputStream in) throws IOException {
 this.endOfPageValueCount = valuesRead + pageValueCount;
 if (dataEncoding.usesDictionary()) {
  this.dataColumn = null;
  if (dictionary == null) {
   throw new IOException(
     "could not read page in col " + descriptor +
       " as the dictionary was missing for encoding " + dataEncoding);
  }
  @SuppressWarnings("deprecation")
  Encoding plainDict = Encoding.PLAIN_DICTIONARY; // var to allow warning suppression
  if (dataEncoding != plainDict && dataEncoding != Encoding.RLE_DICTIONARY) {
   throw new UnsupportedOperationException("Unsupported encoding: " + dataEncoding);
  }
  this.dataColumn = new VectorizedRleValuesReader();
  this.isCurrentPageDictionaryEncoded = true;
 } else {
  if (dataEncoding != Encoding.PLAIN) {
   throw new UnsupportedOperationException("Unsupported encoding: " + dataEncoding);
  }
  this.dataColumn = new VectorizedPlainValuesReader();
  this.isCurrentPageDictionaryEncoded = false;
 }
 try {
  dataColumn.initFromPage(pageValueCount, in);
 } catch (IOException e) {
  throw new IOException("could not read page in col " + descriptor, e);
 }
}

代码示例来源：origin: org.apache.spark/spark-sql

private void initDataReader(Encoding dataEncoding, ByteBufferInputStream in) throws IOException {
 this.endOfPageValueCount = valuesRead + pageValueCount;
 if (dataEncoding.usesDictionary()) {
  this.dataColumn = null;
  if (dictionary == null) {
   throw new IOException(
     "could not read page in col " + descriptor +
       " as the dictionary was missing for encoding " + dataEncoding);
  }
  @SuppressWarnings("deprecation")
  Encoding plainDict = Encoding.PLAIN_DICTIONARY; // var to allow warning suppression
  if (dataEncoding != plainDict && dataEncoding != Encoding.RLE_DICTIONARY) {
   throw new UnsupportedOperationException("Unsupported encoding: " + dataEncoding);
  }
  this.dataColumn = new VectorizedRleValuesReader();
  this.isCurrentPageDictionaryEncoded = true;
 } else {
  if (dataEncoding != Encoding.PLAIN) {
   throw new UnsupportedOperationException("Unsupported encoding: " + dataEncoding);
  }
  this.dataColumn = new VectorizedPlainValuesReader();
  this.isCurrentPageDictionaryEncoded = false;
 }
 try {
  dataColumn.initFromPage(pageValueCount, in);
 } catch (IOException e) {
  throw new IOException("could not read page in col " + descriptor, e);
 }
}

代码示例来源：origin: org.apache.spark/spark-sql_2.10

private void initDataReader(Encoding dataEncoding, byte[] bytes, int offset) throws IOException {
 this.endOfPageValueCount = valuesRead + pageValueCount;
 if (dataEncoding.usesDictionary()) {
  this.dataColumn = null;
  if (dictionary == null) {
   throw new IOException(
     "could not read page in col " + descriptor +
       " as the dictionary was missing for encoding " + dataEncoding);
  }
  @SuppressWarnings("deprecation")
  Encoding plainDict = Encoding.PLAIN_DICTIONARY; // var to allow warning suppression
  if (dataEncoding != plainDict && dataEncoding != Encoding.RLE_DICTIONARY) {
   throw new UnsupportedOperationException("Unsupported encoding: " + dataEncoding);
  }
  this.dataColumn = new VectorizedRleValuesReader();
  this.isCurrentPageDictionaryEncoded = true;
 } else {
  if (dataEncoding != Encoding.PLAIN) {
   throw new UnsupportedOperationException("Unsupported encoding: " + dataEncoding);
  }
  this.dataColumn = new VectorizedPlainValuesReader();
  this.isCurrentPageDictionaryEncoded = false;
 }
 try {
  dataColumn.initFromPage(pageValueCount, bytes, offset);
 } catch (IOException e) {
  throw new IOException("could not read page in col " + descriptor, e);
 }
}

代码示例来源：origin: dremio/dremio-oss

private boolean isDictionaryEncoded(Collection<Encoding> encodings) {
 for (Encoding encoding : encodings) {
  if (encoding.usesDictionary()) {
   return true;
  }
 }
 return false;
}

代码示例来源：origin: org.lasersonlab.apache.parquet/parquet-column

@Override
public Encoding getEncoding() {
 Encoding encoding = currentWriter.getEncoding();
 if (!fellBackAlready && !initialUsedAndHadDictionary) {
  initialUsedAndHadDictionary = encoding.usesDictionary();
 }
 return encoding;
}

代码示例来源：origin: org.apache.parquet/parquet-column

@Override
public Encoding getEncoding() {
 Encoding encoding = currentWriter.getEncoding();
 if (!fellBackAlready && !initialUsedAndHadDictionary) {
  initialUsedAndHadDictionary = encoding.usesDictionary();
 }
 return encoding;
}

代码示例来源：origin: io.snappydata/snappy-spark-sql

private void initDataReader(Encoding dataEncoding, byte[] bytes, int offset) throws IOException {
 this.endOfPageValueCount = valuesRead + pageValueCount;
 if (dataEncoding.usesDictionary()) {
  this.dataColumn = null;
  if (dictionary == null) {
   throw new IOException(
     "could not read page in col " + descriptor +
       " as the dictionary was missing for encoding " + dataEncoding);
  }
  @SuppressWarnings("deprecation")
  Encoding plainDict = Encoding.PLAIN_DICTIONARY; // var to allow warning suppression
  if (dataEncoding != plainDict && dataEncoding != Encoding.RLE_DICTIONARY) {
   throw new UnsupportedOperationException("Unsupported encoding: " + dataEncoding);
  }
  this.dataColumn = new VectorizedRleValuesReader();
  this.isCurrentPageDictionaryEncoded = true;
 } else {
  if (dataEncoding != Encoding.PLAIN) {
   throw new UnsupportedOperationException("Unsupported encoding: " + dataEncoding);
  }
  this.dataColumn = new VectorizedPlainValuesReader();
  this.isCurrentPageDictionaryEncoded = false;
 }
 try {
  dataColumn.initFromPage(pageValueCount, bytes, offset);
 } catch (IOException e) {
  throw new IOException("could not read page in col " + descriptor, e);
 }
}

代码示例来源：origin: com.alibaba.blink/flink-table

private void initDataReader(Encoding dataEncoding, byte[] bytes, int offset) throws IOException {
  this.endOfPageValueCount = valuesRead + pageValueCount;
  if (dataEncoding.usesDictionary()) {
    this.dataColumn = null;
    if (dictionary == null) {
      throw new IOException(
          "could not read page in col " + descriptor +
              " as the dictionary was missing for encoding " + dataEncoding);
    }
    this.dataColumn = new VectorizedRleValuesReader();
    this.isCurrentPageDictionaryEncoded = true;
  } else {
    if (dataEncoding != Encoding.PLAIN) {
      throw new UnsupportedOperationException("Unsupported encoding: " + dataEncoding);
    }
    this.dataColumn = new VectorizedPlainValuesReader();
    this.isCurrentPageDictionaryEncoded = false;
  }
  try {
    dataColumn.initFromPage(pageValueCount, bytes, offset);
  } catch (IOException e) {
    throw new IOException("could not read page in col " + descriptor, e);
  }
}

代码示例来源：origin: org.apache.parquet/parquet-column

private void initDataReader(Encoding dataEncoding, ByteBufferInputStream in, int valueCount) {
 ValuesReader previousReader = this.dataColumn;
 this.currentEncoding = dataEncoding;
 this.pageValueCount = valueCount;
 this.endOfPageValueCount = readValues + pageValueCount;
 if (dataEncoding.usesDictionary()) {
  if (dictionary == null) {
   throw new ParquetDecodingException(
     "could not read page in col " + path + " as the dictionary was missing for encoding " + dataEncoding);
  }
  this.dataColumn = dataEncoding.getDictionaryBasedValuesReader(path, VALUES, dictionary);
 } else {
  this.dataColumn = dataEncoding.getValuesReader(path, VALUES);
 }
 if (dataEncoding.usesDictionary() && converter.hasDictionarySupport()) {
  bindToDictionary(dictionary);
 } else {
  bind(path.getType());
 }
 try {
  dataColumn.initFromPage(pageValueCount, in);
 } catch (IOException e) {
  throw new ParquetDecodingException("could not read page in col " + path, e);
 }
 if (CorruptDeltaByteArrays.requiresSequentialReads(writerVersion, dataEncoding) &&
   previousReader != null && previousReader instanceof RequiresPreviousReader) {
  // previous reader can only be set if reading sequentially
  ((RequiresPreviousReader) dataColumn).setPreviousReader(previousReader);
 }
}

代码示例来源：origin: org.lasersonlab.apache.parquet/parquet-column

private void initDataReader(Encoding dataEncoding, ByteBufferInputStream in, int valueCount) {
 ValuesReader previousReader = this.dataColumn;
 this.currentEncoding = dataEncoding;
 this.pageValueCount = valueCount;
 this.endOfPageValueCount = readValues + pageValueCount;
 if (dataEncoding.usesDictionary()) {
  if (dictionary == null) {
   throw new ParquetDecodingException(
     "could not read page in col " + path + " as the dictionary was missing for encoding " + dataEncoding);
  }
  this.dataColumn = dataEncoding.getDictionaryBasedValuesReader(path, VALUES, dictionary);
 } else {
  this.dataColumn = dataEncoding.getValuesReader(path, VALUES);
 }
 if (dataEncoding.usesDictionary() && converter.hasDictionarySupport()) {
  bindToDictionary(dictionary);
 } else {
  bind(path.getType());
 }
 try {
  dataColumn.initFromPage(pageValueCount, in);
 } catch (IOException e) {
  throw new ParquetDecodingException("could not read page in col " + path, e);
 }
 if (CorruptDeltaByteArrays.requiresSequentialReads(writerVersion, dataEncoding) &&
   previousReader != null && previousReader instanceof RequiresPreviousReader) {
  // previous reader can only be set if reading sequentially
  ((RequiresPreviousReader) dataColumn).setPreviousReader(previousReader);
 }
}

代码示例来源：origin: Netflix/iceberg

if (dataEncoding.usesDictionary()) {
 if (dict == null) {
  throw new ParquetDecodingException(

代码示例来源：origin: dremio/dremio-oss

definitionLevels.initFromPage(currentPageCount, pageDataBuffer, (int) readPosInBytes);
 readPosInBytes = definitionLevels.getNextOffset();
 if (!valueEncoding.usesDictionary()) {
  valueReader = valueEncoding.getValuesReader(parentColumnReader.columnDescriptor, ValuesType.VALUES);
  valueReader.initFromPage(currentPageCount, pageDataBuffer, (int) readPosInBytes);
 valueReader.initFromPage(currentPageCount, pageDataBuffer, (int) readPosInBytes);
if (valueEncoding.usesDictionary()) {

代码示例来源：origin: org.apache.drill.exec/drill-java-exec

definitionLevels.initFromPage(currentPageCount, in);
 readPosInBytes = in.position();
 if (!valueEncoding.usesDictionary()) {
  valueReader = valueEncoding.getValuesReader(parentColumnReader.columnDescriptor, ValuesType.VALUES);
  valueReader.initFromPage(currentPageCount, in);
 valueReader.initFromPage(currentPageCount, in);
if (valueEncoding.usesDictionary()) {

内容来源于网络，如有侵权，请联系作者删除！

Encoding

相关文章

    查看更多

    热门标签

    更多
    JavaquerypythonNode开发语言requestUtil数据库Table后端算法LoggerMessageElementParser

    最新文章

    更多
    • 蜀ICP备13028337号-1 大数据知识库 https://www.saoniuhuo.com © All rights reserved
    • 本站内容来源互联网,如果侵犯您的权益请联系我们删除, 联系方式：448109455@qq.com