本文整理了Java中org.apache.avro.file.FileReader.getSchema()
方法的一些代码示例,展示了FileReader.getSchema()
的具体用法。这些代码示例主要来源于Github
/Stackoverflow
/Maven
等平台,是从一些精选项目中提取出来的代码,具有较强的参考意义,能在一定程度帮忙到你。FileReader.getSchema()
方法的具体详情如下:
包路径:org.apache.avro.file.FileReader
类名称:FileReader
方法名:getSchema
[英]Return the schema for data in this file.
[中]返回此文件中数据的架构。
代码示例来源:origin: org.apache.hadoop/hadoop-common
public AvroFileInputStream(FileStatus status) throws IOException {
pos = 0;
buffer = new byte[0];
GenericDatumReader<Object> reader = new GenericDatumReader<Object>();
FileContext fc = FileContext.getFileContext(new Configuration());
fileReader =
DataFileReader.openReader(new AvroFSInput(fc, status.getPath()),reader);
Schema schema = fileReader.getSchema();
writer = new GenericDatumWriter<Object>(schema);
output = new ByteArrayOutputStream();
encoder = EncoderFactory.get().jsonEncoder(schema, output);
}
代码示例来源:origin: apache/hive
@Override
public boolean next(NullWritable nullWritable, AvroGenericRecordWritable record) throws IOException {
if(isEmptyInput || !reader.hasNext() || reader.pastSync(stop)) {
return false;
}
GenericData.Record r = (GenericData.Record)reader.next();
record.setRecord(r);
record.setRecordReaderID(recordReaderID);
record.setFileSchema(reader.getSchema());
return true;
}
代码示例来源:origin: Netflix/iceberg
@Override
public Schema getSchema() {
return reader.getSchema();
}
代码示例来源:origin: qubole/streamx
@Override
public Schema getSchema(Configuration conf, Path path) throws IOException {
SeekableInput input = new FsInput(path, conf);
DatumReader<Object> reader = new GenericDatumReader<>();
FileReader<Object> fileReader = DataFileReader.openReader(input, reader);
org.apache.avro.Schema schema = fileReader.getSchema();
fileReader.close();
return avroData.toConnectSchema(schema);
}
代码示例来源:origin: com.github.jiayuhan-it/hadoop-common
public AvroFileInputStream(FileStatus status) throws IOException {
pos = 0;
buffer = new byte[0];
GenericDatumReader<Object> reader = new GenericDatumReader<Object>();
FileContext fc = FileContext.getFileContext(new Configuration());
fileReader =
DataFileReader.openReader(new AvroFSInput(fc, status.getPath()),reader);
Schema schema = fileReader.getSchema();
writer = new GenericDatumWriter<Object>(schema);
output = new ByteArrayOutputStream();
JsonGenerator generator =
new JsonFactory().createJsonGenerator(output, JsonEncoding.UTF8);
MinimalPrettyPrinter prettyPrinter = new MinimalPrettyPrinter();
prettyPrinter.setRootValueSeparator(System.getProperty("line.separator"));
generator.setPrettyPrinter(prettyPrinter);
encoder = EncoderFactory.get().jsonEncoder(schema, generator);
}
代码示例来源:origin: io.hops/hadoop-common
public AvroFileInputStream(FileStatus status) throws IOException {
pos = 0;
buffer = new byte[0];
GenericDatumReader<Object> reader = new GenericDatumReader<Object>();
FileContext fc = FileContext.getFileContext(new Configuration());
fileReader =
DataFileReader.openReader(new AvroFSInput(fc, status.getPath()),reader);
Schema schema = fileReader.getSchema();
writer = new GenericDatumWriter<Object>(schema);
output = new ByteArrayOutputStream();
JsonGenerator generator =
new JsonFactory().createJsonGenerator(output, JsonEncoding.UTF8);
MinimalPrettyPrinter prettyPrinter = new MinimalPrettyPrinter();
prettyPrinter.setRootValueSeparator(System.getProperty("line.separator"));
generator.setPrettyPrinter(prettyPrinter);
encoder = EncoderFactory.get().jsonEncoder(schema, generator);
}
代码示例来源:origin: unchartedsoftware/aperture-tiles
FileReader<Object> fileReader = DataFileReader.openReader(input, reader);
try {
Schema schema = fileReader.getSchema();
DatumWriter<Object> writer = new GenericDatumWriter<>(schema);
JsonEncoder encoder = EncoderFactory.get().jsonEncoder(schema, output);
代码示例来源:origin: ch.cern.hadoop/hadoop-common
public AvroFileInputStream(FileStatus status) throws IOException {
pos = 0;
buffer = new byte[0];
GenericDatumReader<Object> reader = new GenericDatumReader<Object>();
FileContext fc = FileContext.getFileContext(new Configuration());
fileReader =
DataFileReader.openReader(new AvroFSInput(fc, status.getPath()),reader);
Schema schema = fileReader.getSchema();
writer = new GenericDatumWriter<Object>(schema);
output = new ByteArrayOutputStream();
JsonGenerator generator =
new JsonFactory().createJsonGenerator(output, JsonEncoding.UTF8);
MinimalPrettyPrinter prettyPrinter = new MinimalPrettyPrinter();
prettyPrinter.setRootValueSeparator(System.getProperty("line.separator"));
generator.setPrettyPrinter(prettyPrinter);
encoder = EncoderFactory.get().jsonEncoder(schema, generator);
}
代码示例来源:origin: linkedin/TonY
&& !this.schema.equals(dataFileReader.getSchema())) {
LOG.warn("Input file have different schema");
this.schema = dataFileReader.getSchema();
dataFileReader.sync(startOffset);
代码示例来源:origin: com.google.cloud.bigdataoss/bigquery-connector
schema = dataFileReader.getSchema();
currentRecord = new GenericData.Record(schema);
代码示例来源:origin: com.expedia.dsp/datasqueeze
private boolean isSchemaCompatible(Schema inputSchema, Path filePath) throws IOException {
GenericDatumReader<Object> reader = new GenericDatumReader<Object>();
FileContext fc = FileContext.getFileContext(configuration);
FileReader fileReader = DataFileReader.openReader(new AvroFSInput(fc, filePath), reader);
Schema fileSchema = fileReader.getSchema();
if (Compatibility.checkThat(fileSchema).canRead(inputSchema).getResult().getCompatibility().equals(patched.org.apache.avro.SchemaCompatibility.SchemaCompatibilityType.COMPATIBLE)) {
return true;
}
return false;
}
代码示例来源:origin: com.facebook.presto.hive/hive-apache
@Override
public boolean next(NullWritable nullWritable, AvroGenericRecordWritable record) throws IOException {
if(!reader.hasNext() || reader.pastSync(stop)) {
return false;
}
GenericData.Record r = (GenericData.Record)reader.next();
record.setRecord(r);
record.setRecordReaderID(recordReaderID);
record.setFileSchema(reader.getSchema());
return true;
}
代码示例来源:origin: kite-sdk/kite
List<GenericData.Record> expecteds = new ArrayList();
FileReader<GenericData.Record> reader = new DataFileReader(file, new GenericDatumReader());
Schema schema = reader.getSchema();
while (reader.hasNext()) {
GenericData.Record expected = reader.next();
代码示例来源:origin: kite-sdk/kite
List<GenericData.Record> expecteds = new ArrayList();
FileReader<GenericData.Record> reader = new DataFileReader(file, new GenericDatumReader());
Schema schema = reader.getSchema();
while (reader.hasNext()) {
GenericData.Record expected = reader.next();
内容来源于网络,如有侵权,请联系作者删除!