org.apache.avro.io.DatumWriter - java examples

Here are the examples of the java api org.apache.avro.io.DatumWriter taken from open source projects. By voting up you can indicate which examples are most useful and appropriate.

155 Examples 7

19 View Complete Implementation : MessageSerializerImpl.java
Copyright Apache License 2.0
Author : apache
/**
 * Implementation of Avro message specific serializer.
 * @param <TMessage> Java type of the message the instantiation can serialize.
 */
public clreplaced MessageSerializerImpl<TMessage> implements IMessageSerializer {

    private final String msgMetaClreplacedName;

    // Writers for header and message.
    private final DatumWriter<Header> headerWriter = new SpecificDatumWriter<>(Header.clreplaced);

    private final DatumWriter<TMessage> messageWriter;

    /**
     * Initialize message specific serializer.
     * @param msgMetaClreplaced The reflection clreplaced for the message.
     */
    public MessageSerializerImpl(final Clreplaced<TMessage> msgMetaClreplaced) {
        this.msgMetaClreplacedName = ProtocolSerializer.getClreplacedId(msgMetaClreplaced);
        this.messageWriter = new SpecificDatumWriter<>(msgMetaClreplaced);
    }

    /**
     * Deserialize messages of type TMessage from input outputStream.
     * @param outputStream A ByteArrayOutputStream where the message to
     *                     be serialized will be written.
     * @param message An Avro message clreplaced which implements the Avro SpcificRecord interface.
     * @param sequence The numerical position of the message in the outgoing message stream.
     * @throws IOException An error occurred writing the message to the outputStream.
     */
    public void serialize(final ByteArrayOutputStream outputStream, final SpecificRecord message, final long sequence) throws IOException {
        // Binary encoder for both the header and message.
        final BinaryEncoder encoder = EncoderFactory.get().binaryEncoder(outputStream, null);
        // Write the header and the message.
        headerWriter.write(new Header(sequence, msgMetaClreplacedName), encoder);
        messageWriter.write((TMessage) message, encoder);
        encoder.flush();
    }
}

19 View Complete Implementation : AvroExternalJarProgram.java
Copyright Apache License 2.0
Author : ljygz
public static void writeTestData(File testFile, int numRecords) throws IOException {
    DatumWriter<MyUser> userDatumWriter = new ReflectDatumWriter<MyUser>(MyUser.clreplaced);
    DataFileWriter<MyUser> dataFileWriter = new DataFileWriter<MyUser>(userDatumWriter);
    dataFileWriter.create(ReflectData.get().getSchema(MyUser.clreplaced), testFile);
    Generator generator = new Generator();
    for (int i = 0; i < numRecords; i++) {
        MyUser user = generator.nextUser();
        dataFileWriter.append(user);
    }
    dataFileWriter.close();
}

19 View Complete Implementation : MultiAvroSchemaJoinIT.java
Copyright Apache License 2.0
Author : apache
@Before
public void setUp() throws Exception {
    this.personFile = File.createTempFile("person", ".avro");
    this.employeeFile = File.createTempFile("employee", ".avro");
    DatumWriter<Person> pdw = new SpecificDatumWriter<Person>();
    DataFileWriter<Person> pfw = new DataFileWriter<Person>(pdw);
    pfw.create(Person.SCHEMA$, personFile);
    Person p1 = new Person();
    p1.name = "Josh";
    p1.age = 19;
    p1.siblingnames = ImmutableList.<CharSequence>of("Kate", "Mike");
    pfw.append(p1);
    Person p2 = new Person();
    p2.name = "Kate";
    p2.age = 17;
    ;
    p2.siblingnames = ImmutableList.<CharSequence>of("Josh", "Mike");
    pfw.append(p2);
    Person p3 = new Person();
    p3.name = "Mike";
    p3.age = 12;
    p3.siblingnames = ImmutableList.<CharSequence>of("Josh", "Kate");
    pfw.append(p3);
    pfw.close();
    DatumWriter<Employee> edw = new SpecificDatumWriter<Employee>();
    DataFileWriter<Employee> efw = new DataFileWriter<Employee>(edw);
    efw.create(Employee.SCHEMA$, employeeFile);
    Employee e1 = new Employee();
    e1.name = "Kate";
    e1.salary = 100000;
    e1.department = "Marketing";
    efw.append(e1);
    efw.close();
}

19 View Complete Implementation : AvroExternalJarProgram.java
Copyright Apache License 2.0
Author : apache
public static void writeTestData(File testFile, int numRecords) throws IOException {
    DatumWriter<MyUser> userDatumWriter = new ReflectDatumWriter<MyUser>(MyUser.clreplaced);
    DataFileWriter<MyUser> dataFileWriter = new DataFileWriter<MyUser>(userDatumWriter);
    dataFileWriter.create(ReflectData.get().getSchema(MyUser.clreplaced), testFile);
    Generator generator = new Generator();
    for (int i = 0; i < numRecords; i++) {
        MyUser user = generator.nextUser();
        dataFileWriter.append(user);
    }
    dataFileWriter.close();
}

19 View Complete Implementation : AvroExternalJarProgram.java
Copyright Apache License 2.0
Author : ljygz
public static void writeTestData(File testFile, int numRecords) throws IOException {
    DatumWriter<MyUser> userDatumWriter = new ReflectDatumWriter<MyUser>(MyUser.clreplaced);
    DataFileWriter<MyUser> dataFileWriter = new DataFileWriter<MyUser>(userDatumWriter);
    dataFileWriter.create(ReflectData.get().getSchema(MyUser.clreplaced), testFile);
    Generator generator = new Generator();
    for (int i = 0; i < numRecords; i++) {
        MyUser user = generator.nextUser();
        dataFileWriter.append(user);
    }
    dataFileWriter.close();
}

18 View Complete Implementation : SafeAvroSerialization.java
Copyright Apache License 2.0
Author : apache
/**
 * Returns the specified output serializer.
 */
public Serializer<AvroWrapper<T>> getSerializer(Clreplaced<AvroWrapper<T>> c) {
    // AvroWrapper used for final output, AvroKey or AvroValue for map output
    boolean isFinalOutput = c.equals(AvroWrapper.clreplaced);
    Configuration conf = getConf();
    Schema schema = isFinalOutput ? AvroJob.getOutputSchema(conf) : (AvroKey.clreplaced.isreplacedignableFrom(c) ? Pair.getKeySchema(AvroJob.getMapOutputSchema(conf)) : Pair.getValueSchema(AvroJob.getMapOutputSchema(conf)));
    ReaderWriterFactory factory = AvroMode.fromShuffleConfiguration(conf);
    DatumWriter<T> writer = factory.getWriter(schema);
    return new AvroWrapperSerializer(writer);
}

18 View Complete Implementation : AvroOutputFormat.java
Copyright Apache License 2.0
Author : apache
@Override
public void open(int taskNumber, int numTasks) throws IOException {
    super.open(taskNumber, numTasks);
    DatumWriter<E> datumWriter;
    Schema schema;
    if (org.apache.avro.specific.SpecificRecordBase.clreplaced.isreplacedignableFrom(avroValueType)) {
        datumWriter = new SpecificDatumWriter<E>(avroValueType);
        try {
            schema = ((org.apache.avro.specific.SpecificRecordBase) avroValueType.newInstance()).getSchema();
        } catch (InstantiationException | IllegalAccessException e) {
            throw new RuntimeException(e.getMessage());
        }
    } else if (org.apache.avro.generic.GenericRecord.clreplaced.isreplacedignableFrom(avroValueType)) {
        if (userDefinedSchema == null) {
            throw new IllegalStateException("Schema must be set when using Generic Record");
        }
        datumWriter = new GenericDatumWriter<E>(userDefinedSchema);
        schema = userDefinedSchema;
    } else {
        datumWriter = new ReflectDatumWriter<E>(avroValueType);
        schema = ReflectData.get().getSchema(avroValueType);
    }
    dataFileWriter = new DataFileWriter<E>(datumWriter);
    if (codec != null) {
        dataFileWriter.setCodec(codec.getCodecFactory());
    }
    if (userDefinedSchema == null) {
        dataFileWriter.create(schema, stream);
    } else {
        dataFileWriter.create(userDefinedSchema, stream);
    }
}

18 View Complete Implementation : YarnSubmissionParametersFileGenerator.java
Copyright Apache License 2.0
Author : apache
static void writeAvroYarnAppSubmissionParametersToOutputStream(final YarnClusterSubmissionFromCS yarnClusterSubmissionFromCS, final OutputStream outputStream) throws IOException {
    final DatumWriter<AvroYarnAppSubmissionParameters> datumWriter = new SpecificDatumWriter<>(AvroYarnAppSubmissionParameters.clreplaced);
    final AvroYarnAppSubmissionParameters appSubmissionParameters = yarnClusterSubmissionFromCS.getYarnAppSubmissionParameters();
    final JsonEncoder encoder = EncoderFactory.get().jsonEncoder(appSubmissionParameters.getSchema(), outputStream);
    datumWriter.write(appSubmissionParameters, encoder);
    encoder.flush();
    outputStream.flush();
}

18 View Complete Implementation : AvroUtilsTest.java
Copyright Apache License 2.0
Author : kite-sdk
@Test
public void testWriteAvroEnreplacedy() throws Exception {
    String schemaString = "{ \"type\": \"int\" }";
    InputStream is = new ByteArrayInputStream(schemaString.getBytes());
    Schema schema = parser.parse(is);
    DatumWriter<Integer> writer = new GenericDatumWriter<Integer>(schema);
    byte[] bytes = AvroUtils.writeAvroEnreplacedy(1, writer);
    replacedertArrayEquals(new byte[] { (byte) 2 }, bytes);
}

18 View Complete Implementation : AvroOutputFormat.java
Copyright Apache License 2.0
Author : ljygz
@Override
public void open(int taskNumber, int numTasks) throws IOException {
    super.open(taskNumber, numTasks);
    DatumWriter<E> datumWriter;
    Schema schema;
    if (org.apache.avro.specific.SpecificRecordBase.clreplaced.isreplacedignableFrom(avroValueType)) {
        datumWriter = new SpecificDatumWriter<E>(avroValueType);
        try {
            schema = ((org.apache.avro.specific.SpecificRecordBase) avroValueType.newInstance()).getSchema();
        } catch (InstantiationException | IllegalAccessException e) {
            throw new RuntimeException(e.getMessage());
        }
    } else if (org.apache.avro.generic.GenericRecord.clreplaced.isreplacedignableFrom(avroValueType)) {
        if (userDefinedSchema == null) {
            throw new IllegalStateException("Schema must be set when using Generic Record");
        }
        datumWriter = new GenericDatumWriter<E>(userDefinedSchema);
        schema = userDefinedSchema;
    } else {
        datumWriter = new ReflectDatumWriter<E>(avroValueType);
        schema = ReflectData.get().getSchema(avroValueType);
    }
    dataFileWriter = new DataFileWriter<E>(datumWriter);
    if (codec != null) {
        dataFileWriter.setCodec(codec.getCodecFactory());
    }
    if (userDefinedSchema == null) {
        dataFileWriter.create(schema, stream);
    } else {
        dataFileWriter.create(userDefinedSchema, stream);
    }
}

18 View Complete Implementation : AvroSchemasTest.java
Copyright Apache License 2.0
Author : dkmfbk
@Test
public void test() throws Throwable {
    System.out.println(AvroSchemas.RECORD.toString());
    final Record resource = HBaseTestUtils.getMockResource();
    // final Object generic = AvroSerialization.toGenericData(resource);
    final Object generic = null;
    final ByteArrayOutputStream jsonStream = new ByteArrayOutputStream();
    final ByteArrayOutputStream binaryStream = new ByteArrayOutputStream();
    final Encoder jsonEncoder = EncoderFactory.get().jsonEncoder(AvroSchemas.RECORD, jsonStream);
    final Encoder binaryEncoder = EncoderFactory.get().directBinaryEncoder(binaryStream, null);
    final DatumWriter<Object> writer = new GenericDatumWriter<Object>(AvroSchemas.RECORD);
    writer.write(generic, jsonEncoder);
    writer.write(generic, binaryEncoder);
    binaryEncoder.flush();
    jsonEncoder.flush();
    final byte[] bytes = binaryStream.toByteArray();
    final String json = new String(jsonStream.toByteArray(), Charsets.UTF_8);
    System.out.println(bytes.length + " bytes: " + BaseEncoding.base16().encode(bytes));
    System.out.println("JSON:\n" + json);
}

18 View Complete Implementation : AvroEvaluatorListSerializer.java
Copyright Apache License 2.0
Author : apache
/**
 * Convert AvroEvaluatorList to JSON string.
 */
@Override
public String toString(final AvroEvaluatorList avroEvaluatorList) {
    final DatumWriter<AvroEvaluatorList> evaluatorWriter = new SpecificDatumWriter<>(AvroEvaluatorList.clreplaced);
    try (final ByteArrayOutputStream out = new ByteArrayOutputStream()) {
        final JsonEncoder encoder = EncoderFactory.get().jsonEncoder(avroEvaluatorList.getSchema(), out);
        evaluatorWriter.write(avroEvaluatorList, encoder);
        encoder.flush();
        return out.toString(AvroHttpSerializer.JSON_CHARSET);
    } catch (final IOException e) {
        throw new RuntimeException(e);
    }
}

18 View Complete Implementation : AvroEvaluatorInfoSerializer.java
Copyright Apache License 2.0
Author : apache
/**
 * Convert AvroEvaluatorsInfo object to JSON string.
 */
@Override
public String toString(final AvroEvaluatorsInfo avroEvaluatorsInfo) {
    final DatumWriter<AvroEvaluatorsInfo> evaluatorWriter = new SpecificDatumWriter<>(AvroEvaluatorsInfo.clreplaced);
    try (final ByteArrayOutputStream out = new ByteArrayOutputStream()) {
        final JsonEncoder encoder = EncoderFactory.get().jsonEncoder(avroEvaluatorsInfo.getSchema(), out);
        evaluatorWriter.write(avroEvaluatorsInfo, encoder);
        encoder.flush();
        return out.toString(AvroHttpSerializer.JSON_CHARSET);
    } catch (final IOException e) {
        throw new RuntimeException(e);
    }
}

18 View Complete Implementation : AvroEntitySerDe.java
Copyright Apache License 2.0
Author : kite-sdk
@Override
public byte[] serializeKeyAsColumnValueToBytes(String fieldName, CharSequence columnKey, Object columnValue) {
    Field field = avroSchema.getAvroSchema().getField(fieldName);
    if (field == null) {
        throw new ValidationException("Invalid field name " + fieldName + " for schema " + avroSchema.toString());
    }
    Schema.Type schemaType = field.schema().getType();
    if (schemaType == Schema.Type.MAP) {
        DatumWriter<Object> datumWriter = fieldDatumWriters.get(fieldName);
        if (datumWriter == null) {
            throw new ValidationException("No datum writer for field name: " + fieldName);
        }
        return AvroUtils.writeAvroEnreplacedy(columnValue, datumWriter);
    } else if (schemaType == Schema.Type.RECORD) {
        if (!kacRecordDatumWriters.containsKey(fieldName)) {
            throw new ValidationException("Invalid field name " + fieldName + " for schema " + avroSchema.toString());
        }
        if (!kacRecordDatumWriters.get(fieldName).containsKey(columnKey.toString())) {
            throw new ValidationException("Invalid key in record: " + fieldName + "." + columnKey);
        }
        DatumWriter<Object> datumWriter = kacRecordDatumWriters.get(fieldName).get(columnKey.toString());
        return AvroUtils.writeAvroEnreplacedy(columnValue, datumWriter);
    } else {
        throw new ValidationException("Unsupported type for keyAsColumn: " + schemaType);
    }
}

18 View Complete Implementation : AvroOutputFormat.java
Copyright Apache License 2.0
Author : ljygz
@Override
public void open(int taskNumber, int numTasks) throws IOException {
    super.open(taskNumber, numTasks);
    DatumWriter<E> datumWriter;
    Schema schema;
    if (org.apache.avro.specific.SpecificRecordBase.clreplaced.isreplacedignableFrom(avroValueType)) {
        datumWriter = new SpecificDatumWriter<E>(avroValueType);
        try {
            schema = ((org.apache.avro.specific.SpecificRecordBase) avroValueType.newInstance()).getSchema();
        } catch (InstantiationException | IllegalAccessException e) {
            throw new RuntimeException(e.getMessage());
        }
    } else if (org.apache.avro.generic.GenericRecord.clreplaced.isreplacedignableFrom(avroValueType)) {
        if (userDefinedSchema == null) {
            throw new IllegalStateException("Schema must be set when using Generic Record");
        }
        datumWriter = new GenericDatumWriter<E>(userDefinedSchema);
        schema = userDefinedSchema;
    } else {
        datumWriter = new ReflectDatumWriter<E>(avroValueType);
        schema = ReflectData.get().getSchema(avroValueType);
    }
    dataFileWriter = new DataFileWriter<E>(datumWriter);
    if (codec != null) {
        dataFileWriter.setCodec(codec.getCodecFactory());
    }
    if (userDefinedSchema == null) {
        dataFileWriter.create(schema, stream);
    } else {
        dataFileWriter.create(userDefinedSchema, stream);
    }
}

17 View Complete Implementation : FromTest.java
Copyright Apache License 2.0
Author : apache
@Test
public void testAvroFile_GlobWithSchemaInferenceIsSupported() throws IOException {
    Schema schema = SchemaBuilder.record("record").fields().endRecord();
    DatumWriter<GenericRecord> datumWriter = new GenericDatumWriter<>(schema);
    try (DataFileWriter<GenericRecord> writer = new DataFileWriter<>(datumWriter)) {
        writer.create(schema, tmp.newFile("1"));
        writer.append(new GenericData.Record(schema));
    }
    Source<GenericData.Record> source = From.avroFile(new Path(tmp.getRoot().toString() + "/*"));
    replacedertEquals(source.getType(), Avros.generics(schema));
}

17 View Complete Implementation : AvroEntitySerDe.java
Copyright Apache License 2.0
Author : kite-sdk
@Override
public byte[] serializeColumnValueToBytes(String fieldName, Object columnValue) {
    Field field = avroSchema.getAvroSchema().getField(fieldName);
    DatumWriter<Object> datumWriter = fieldDatumWriters.get(fieldName);
    if (field == null) {
        throw new ValidationException("Invalid field name " + fieldName + " for schema " + avroSchema.toString());
    }
    if (datumWriter == null) {
        throw new ValidationException("No datum writer for field name: " + fieldName);
    }
    ByteArrayOutputStream byteOut = new ByteArrayOutputStream();
    Encoder encoder = getColumnEncoder(field.schema(), byteOut);
    AvroUtils.writeAvroEnreplacedy(columnValue, encoder, fieldDatumWriters.get(fieldName));
    return byteOut.toByteArray();
}

17 View Complete Implementation : AvroCellEncoder.java
Copyright Apache License 2.0
Author : kijiproject
/**
 * Gets a datum writer for a schema and caches it.
 *
 * <p> Not thread-safe, calls to this method must be externally synchronized. </p>
 *
 * @param schema The writer schema.
 * @return A datum writer for the given schema.
 */
private DatumWriter<Object> getDatumWriter(final Schema schema) {
    final DatumWriter<Object> existing = mCachedDatumWriters.get(schema);
    if (null != existing) {
        return existing;
    }
    final DatumWriter<Object> newWriter = new SpecificDatumWriter<Object>(schema);
    mCachedDatumWriters.put(schema, newWriter);
    return newWriter;
}

17 View Complete Implementation : TestExtractAvroMetadata.java
Copyright Apache License 2.0
Author : wangrenlei
private ByteArrayOutputStream getOutputStreamWithOneUser(Schema schema) throws IOException {
    final GenericRecord user = new GenericData.Record(schema);
    user.put("name", "Alyssa");
    user.put("favorite_number", 256);
    final DatumWriter<GenericRecord> datumWriter = new GenericDatumWriter<>(schema);
    return AvroTestUtil.serializeAvroRecord(schema, datumWriter, user);
}

17 View Complete Implementation : TestExtractAvroMetadata.java
Copyright Apache License 2.0
Author : apache
private ByteArrayOutputStream getOutputStreamWithMultipleUsers(Schema schema, int numUsers) throws IOException {
    final GenericRecord[] users = new GenericRecord[numUsers];
    for (int i = 0; i < numUsers; i++) {
        final GenericRecord user = new GenericData.Record(schema);
        user.put("name", "user" + i);
        user.put("favorite_number", i);
        users[i] = user;
    }
    final DatumWriter<GenericRecord> datumWriter = new GenericDatumWriter<>(schema);
    return AvroTestUtil.serializeAvroRecord(schema, datumWriter, users);
}

17 View Complete Implementation : TestAvroFileHdfsReader.java
Copyright Apache License 2.0
Author : apache
public static void writeTestEventsToFile(String path, int numEvents) throws Exception {
    Schema schema = Schema.parse(TestAvroFileHdfsReader.clreplaced.getResourcereplacedtream("/reader/TestEvent.avsc"));
    File file = new File(path);
    DatumWriter<GenericRecord> writer = new GenericDatumWriter<>(schema);
    DataFileWriter<GenericRecord> dataFileWriter = new DataFileWriter<>(writer);
    dataFileWriter.create(schema, file);
    for (int i = 0; i < numEvents; i++) {
        GenericRecord datum = new GenericData.Record(schema);
        datum.put(FIELD_1, i);
        datum.put(FIELD_2, "string_" + i);
        dataFileWriter.append(datum);
    }
    dataFileWriter.close();
}

17 View Complete Implementation : JsonUtils.java
Copyright Apache License 2.0
Author : wangrenlei
/**
 * Writes provided {@link GenericRecord} into the provided
 * {@link OutputStream} as JSON.
 */
public static void write(GenericRecord record, OutputStream out) {
    try {
        DatumWriter<GenericRecord> writer = new GenericDatumWriter<GenericRecord>(record.getSchema());
        JsonEncoder encoder = EncoderFactory.get().jsonEncoder(record.getSchema(), out);
        writer.write(record, encoder);
        encoder.flush();
    } catch (Exception e) {
        throw new IllegalStateException("Failed to read GenericRecord", e);
    }
}

17 View Complete Implementation : TestAvroRelConversion.java
Copyright Apache License 2.0
Author : apache
private static byte[] bytesFromGenericRecord(GenericRecord record) throws IOException {
    DatumWriter<IndexedRecord> datumWriter;
    datumWriter = new GenericDatumWriter<>(record.getSchema());
    ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
    BinaryEncoder encoder = EncoderFactory.get().binaryEncoder(outputStream, null);
    datumWriter.write(record, encoder);
    encoder.flush();
    outputStream.close();
    return outputStream.toByteArray();
}

17 View Complete Implementation : YarnSubmissionParametersFileGenerator.java
Copyright Apache License 2.0
Author : apache
static void writeAvroYarnJobSubmissionParametersToOutputStream(final YarnClusterSubmissionFromCS yarnClusterSubmissionFromCS, final String jobFolderOnDFSPath, final OutputStream outputStream) throws IOException {
    final DatumWriter<AvroYarnJobSubmissionParameters> datumWriter = new SpecificDatumWriter<>(AvroYarnJobSubmissionParameters.clreplaced);
    final AvroYarnJobSubmissionParameters jobSubmissionParameters = yarnClusterSubmissionFromCS.getYarnJobSubmissionParameters();
    jobSubmissionParameters.setDfsJobSubmissionFolder(jobFolderOnDFSPath);
    final JsonEncoder encoder = EncoderFactory.get().jsonEncoder(jobSubmissionParameters.getSchema(), outputStream);
    datumWriter.write(jobSubmissionParameters, encoder);
    encoder.flush();
    outputStream.flush();
}

17 View Complete Implementation : FromTest.java
Copyright Apache License 2.0
Author : apache
@Test
public void testAvroFile_DirectoryWithSchemaInferenceIsSupported() throws IOException {
    Schema schema = SchemaBuilder.record("record").fields().endRecord();
    DatumWriter<GenericRecord> datumWriter = new GenericDatumWriter<>(schema);
    try (DataFileWriter<GenericRecord> writer = new DataFileWriter<>(datumWriter)) {
        writer.create(schema, tmp.newFile("1"));
        writer.append(new GenericData.Record(schema));
    }
    Source<GenericData.Record> source = From.avroFile(new Path(tmp.getRoot().toString()));
    replacedertEquals(source.getType(), Avros.generics(schema));
}

17 View Complete Implementation : RecordSetUtil.java
Copyright Apache License 2.0
Author : Talend
/**
 * Writes all records from the test set into a single Avro file on the file system.
 *
 * @param fs The filesystem.
 * @param path The path of the file on the filesystem.
 * @param td The test data to write.
 * @throws IOException If there was an exception writing to the filesystem.
 */
public static void writeRandomAvroFile(FileSystem fs, String path, RecordSet td) throws IOException {
    try (OutputStream out = fs.create(new Path(path))) {
        DatumWriter<IndexedRecord> datumWriter = new GenericDatumWriter<>(td.getSchema());
        DataFileWriter<IndexedRecord> dataFileWriter = new DataFileWriter<>(datumWriter);
        dataFileWriter.create(td.getSchema(), out);
        for (List<IndexedRecord> parreplacedion : td.getParreplacedions()) {
            for (IndexedRecord record : parreplacedion) {
                dataFileWriter.append(record);
            }
        }
        dataFileWriter.close();
    }
}

17 View Complete Implementation : FsSpecProducer.java
Copyright Apache License 2.0
Author : apache
private void writeAvroJobSpec(AvroJobSpec jobSpec) throws IOException {
    DatumWriter<AvroJobSpec> datumWriter = new SpecificDatumWriter<>(AvroJobSpec.SCHEMA$);
    DataFileWriter<AvroJobSpec> dataFileWriter = new DataFileWriter<>(datumWriter);
    Path jobSpecPath = new Path(this.specConsumerPath, jobSpec.getUri());
    FileSystem fs = jobSpecPath.getFileSystem(new Configuration());
    OutputStream out = fs.create(jobSpecPath);
    dataFileWriter.create(AvroJobSpec.SCHEMA$, out);
    dataFileWriter.append(jobSpec);
    dataFileWriter.close();
}

17 View Complete Implementation : TestExtractAvroMetadata.java
Copyright Apache License 2.0
Author : wangrenlei
private ByteArrayOutputStream getOutputStreamWithMultipleUsers(Schema schema, int numUsers) throws IOException {
    final GenericRecord[] users = new GenericRecord[numUsers];
    for (int i = 0; i < numUsers; i++) {
        final GenericRecord user = new GenericData.Record(schema);
        user.put("name", "user" + i);
        user.put("favorite_number", i);
        users[i] = user;
    }
    final DatumWriter<GenericRecord> datumWriter = new GenericDatumWriter<>(schema);
    return AvroTestUtil.serializeAvroRecord(schema, datumWriter, users);
}

17 View Complete Implementation : SafeAvroSerializationIT.java
Copyright Apache License 2.0
Author : apache
private File populateRightSide() throws IOException {
    File file = tmpDir.getFile("rightSide.avro");
    DatumWriter<String> datumWriter = new GenericDatumWriter<String>(Avros.strings().getSchema());
    DataFileWriter<String> dataFileWriter = new DataFileWriter<String>(datumWriter);
    dataFileWriter.create(Avros.strings().getSchema(), file);
    // will join successfully to RECORD 2 from left side
    dataFileWriter.append("c");
    dataFileWriter.close();
    return file;
}

17 View Complete Implementation : FromTest.java
Copyright Apache License 2.0
Author : apache
@Test
public void testAvroFile_FileWithSchemaInferenceIsSupported() throws IOException {
    Schema schema = SchemaBuilder.record("record").fields().endRecord();
    DatumWriter<GenericRecord> datumWriter = new GenericDatumWriter<>(schema);
    try (DataFileWriter<GenericRecord> writer = new DataFileWriter<>(datumWriter)) {
        writer.create(schema, tmp.newFile("1"));
        writer.append(new GenericData.Record(schema));
    }
    Source<GenericData.Record> source = From.avroFile(new Path(tmp.getRoot().toString(), "1"));
    replacedertEquals(source.getType(), Avros.generics(schema));
}

17 View Complete Implementation : TestExtractAvroMetadata.java
Copyright Apache License 2.0
Author : apache
private ByteArrayOutputStream getOutputStreamWithOneUser(Schema schema) throws IOException {
    final GenericRecord user = new GenericData.Record(schema);
    user.put("name", "Alyssa");
    user.put("favorite_number", 256);
    final DatumWriter<GenericRecord> datumWriter = new GenericDatumWriter<>(schema);
    return AvroTestUtil.serializeAvroRecord(schema, datumWriter, user);
}

16 View Complete Implementation : SerializerAvro.java
Copyright Apache License 2.0
Author : dkmfbk
public void toStream(final OutputStream stream, final Object object) throws IOException {
    final Object generic = encodeNode(object);
    final Encoder encoder = EncoderFactory.get().directBinaryEncoder(stream, null);
    final DatumWriter<Object> writer = new GenericDatumWriter<Object>(Schemas.NODE);
    writer.write(generic, encoder);
    encoder.flush();
}

16 View Complete Implementation : SerializerAvro.java
Copyright Apache License 2.0
Author : dkmfbk
public void toStream(final OutputStream stream, final Record object, @Nullable final Set<URI> propertiesToSerialize) throws IOException {
    final Object generic = encodeRecord(object, propertiesToSerialize);
    final Encoder encoder = EncoderFactory.get().directBinaryEncoder(stream, null);
    final DatumWriter<Object> writer = new GenericDatumWriter<Object>(Schemas.NODE);
    writer.write(generic, encoder);
    encoder.flush();
}

16 View Complete Implementation : InlineSchemaAvroBytesDecoderTest.java
Copyright Apache License 2.0
Author : apache
@Test
public void testParse() throws Exception {
    GenericRecord someAvroDatum = AvroStreamInputRowParserTest.buildSomeAvroDatum();
    Schema schema = SomeAvroDatum.getClreplacedSchema();
    ByteArrayOutputStream out = new ByteArrayOutputStream();
    DatumWriter<GenericRecord> writer = new SpecificDatumWriter<>(schema);
    writer.write(someAvroDatum, EncoderFactory.get().directBinaryEncoder(out, null));
    GenericRecord actual = new InlineSchemaAvroBytesDecoder(schema).parse(ByteBuffer.wrap(out.toByteArray()));
    replacedert.replacedertEquals(someAvroDatum.get("id"), actual.get("id"));
}

16 View Complete Implementation : AvroSerializer.java
Copyright Apache License 2.0
Author : dkmfbk
public void toStream(final OutputStream stream, final Object object) throws IOException {
    final Object generic = encodeNode(object);
    final Encoder encoder = EncoderFactory.get().directBinaryEncoder(stream, null);
    final DatumWriter<Object> writer = new GenericDatumWriter<Object>(AvroSchemas.NODE);
    writer.write(generic, encoder);
    encoder.flush();
}

16 View Complete Implementation : InlineSchemasAvroBytesDecoderTest.java
Copyright Apache License 2.0
Author : apache
@Test
public void testParse() throws Exception {
    GenericRecord someAvroDatum = AvroStreamInputRowParserTest.buildSomeAvroDatum();
    Schema schema = SomeAvroDatum.getClreplacedSchema();
    ByteArrayOutputStream out = new ByteArrayOutputStream();
    out.write(new byte[] { 1 });
    out.write(ByteBuffer.allocate(4).putInt(10).array());
    DatumWriter<GenericRecord> writer = new SpecificDatumWriter<>(schema);
    writer.write(someAvroDatum, EncoderFactory.get().directBinaryEncoder(out, null));
    GenericRecord actual = new InlineSchemasAvroBytesDecoder(ImmutableMap.of(10, schema)).parse(ByteBuffer.wrap(out.toByteArray()));
    replacedert.replacedertEquals(someAvroDatum.get("id"), actual.get("id"));
}

16 View Complete Implementation : AvroDriverInfoSerializer.java
Copyright Apache License 2.0
Author : apache
/**
 * Convert AvroDriverInfo object to JSON string.
 */
@Override
public String toString(final AvroDriverInfo avroDriverInfo) {
    final DatumWriter<AvroDriverInfo> driverWriter = new SpecificDatumWriter<>(AvroDriverInfo.clreplaced);
    try (final ByteArrayOutputStream out = new ByteArrayOutputStream()) {
        final JsonEncoder encoder = EncoderFactory.get().jsonEncoder(avroDriverInfo.getSchema(), out);
        driverWriter.write(avroDriverInfo, encoder);
        encoder.flush();
        return out.toString(AvroHttpSerializer.JSON_CHARSET);
    } catch (final IOException e) {
        throw new RuntimeException(e);
    }
}

16 View Complete Implementation : AvroSerializer.java
Copyright Apache License 2.0
Author : dkmfbk
public void toStream(final OutputStream stream, final Record object, @Nullable final Set<URI> propertiesToSerialize) throws IOException {
    final Object generic = encodeRecord(object, propertiesToSerialize);
    final Encoder encoder = EncoderFactory.get().directBinaryEncoder(stream, null);
    final DatumWriter<Object> writer = new GenericDatumWriter<Object>(AvroSchemas.NODE);
    writer.write(generic, encoder);
    encoder.flush();
}

16 View Complete Implementation : MemPipeline.java
Copyright Apache License 2.0
Author : apache
@SuppressWarnings({ "rawtypes", "unchecked" })
private void writeAvroFile(FSDataOutputStream outputStream, PCollection recordCollection) throws IOException {
    AvroType avroType = (AvroType) recordCollection.getPType();
    if (avroType == null) {
        throw new IllegalStateException("Can't write a non-typed Avro collection");
    }
    DatumWriter datumWriter = Avros.newWriter((AvroType) recordCollection.getPType());
    DataFileWriter dataFileWriter = new DataFileWriter(datumWriter);
    dataFileWriter.create(avroType.getSchema(), outputStream);
    for (Object record : recordCollection.materialize()) {
        dataFileWriter.append(avroType.getOutputMapFn().map(record));
    }
    dataFileWriter.close();
    outputStream.close();
}

16 View Complete Implementation : SerializerAvro.java
Copyright Apache License 2.0
Author : dkmfbk
public byte[] compressURI(final URI uri) {
    Preconditions.checkNotNull(uri);
    try {
        final ByteArrayOutputStream stream = new ByteArrayOutputStream();
        final Encoder encoder = EncoderFactory.get().directBinaryEncoder(stream, null);
        final DatumWriter<Object> writer = new GenericDatumWriter<Object>(Schemas.COMPRESSED_IDENTIFIER);
        // ensure a compressed version of URI is available
        this.dictionary.keyFor(uri);
        final Object generic = encodeIdentifier(uri);
        writer.write(generic, encoder);
        return stream.toByteArray();
    } catch (final IOException ex) {
        throw new Error("Unexpected exception (!): " + ex.getMessage(), ex);
    }
}

16 View Complete Implementation : AvroSerializer.java
Copyright Apache License 2.0
Author : dkmfbk
public byte[] compressURI(final URI uri) {
    Preconditions.checkNotNull(uri);
    try {
        final ByteArrayOutputStream stream = new ByteArrayOutputStream();
        final Encoder encoder = EncoderFactory.get().directBinaryEncoder(stream, null);
        final DatumWriter<Object> writer = new GenericDatumWriter<Object>(AvroSchemas.COMPRESSED_IDENTIFIER);
        // ensure a compressed version of URI is available
        this.dictionary.keyFor(uri);
        final Object generic = encodeIdentifier(uri);
        writer.write(generic, encoder);
        return stream.toByteArray();
    } catch (final IOException ex) {
        throw new Error("Unexpected exception (!): " + ex.getMessage(), ex);
    }
}

16 View Complete Implementation : AvroKeySerDe.java
Copyright Apache License 2.0
Author : kite-sdk
@Override
public byte[] serialize(ParreplacedionKey key) {
    ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
    Encoder encoder = new MemcmpEncoder(outputStream);
    Schema schemaToUse;
    if (key.getLength() == schema.getFields().size()) {
        schemaToUse = schema;
    } else {
        schemaToUse = partialSchemas[key.getLength() - 1];
    }
    DatumWriter<GenericRecord> datumWriter = new GenericDatumWriter<GenericRecord>(schemaToUse);
    GenericRecord record = new GenericData.Record(schemaToUse);
    for (int i = 0; i < key.getLength(); i++) {
        Object keyPart = key.get(i);
        if (keyPart == null) {
            // keyPart is null, let's make sure we check that the key can support a
            // null value so we can throw a friendly exception if it can't.
            Schema fieldSchema = schemaToUse.getFields().get(i).schema();
            if (fieldSchema.getType() != Schema.Type.NULL && fieldSchema.getType() != Schema.Type.UNION) {
                throw new DatasetException("Null key field only supported in null type or union type that has a null type.");
            } else if (fieldSchema.getType() == Schema.Type.UNION) {
                boolean foundNullInUnion = false;
                for (Schema unionSchema : fieldSchema.getTypes()) {
                    if (unionSchema.getType() == Schema.Type.NULL) {
                        foundNullInUnion = true;
                    }
                }
                if (!foundNullInUnion) {
                    throw new DatasetException("Null key field only supported in union type that has a null type.");
                }
            }
        }
        record.put(i, keyPart);
    }
    AvroUtils.writeAvroEnreplacedy(record, encoder, datumWriter);
    return outputStream.toByteArray();
}

16 View Complete Implementation : AvroSerDeFactory.java
Copyright Apache License 2.0
Author : apache
private byte[] encodeAvroGenericRecord(Schema schema, GenericRecord record) throws IOException {
    DatumWriter<IndexedRecord> msgDatumWriter = new GenericDatumWriter<>(schema);
    ByteArrayOutputStream os = new ByteArrayOutputStream();
    Encoder encoder = EncoderFactory.get().binaryEncoder(os, null);
    msgDatumWriter.write(record, encoder);
    encoder.flush();
    return os.toByteArray();
}

16 View Complete Implementation : AvroUtils.java
Copyright Apache License 2.0
Author : apache
/**
 * Convert a GenericRecord to a byte array.
 */
public static byte[] recordToByteArray(GenericRecord record) throws IOException {
    try (ByteArrayOutputStream out = new ByteArrayOutputStream()) {
        Encoder encoder = EncoderFactory.get().directBinaryEncoder(out, null);
        DatumWriter<GenericRecord> writer = new GenericDatumWriter<>(record.getSchema());
        writer.write(record, encoder);
        byte[] byteArray = out.toByteArray();
        return byteArray;
    }
}

15 View Complete Implementation : AvroUtils.java
Copyright Apache License 2.0
Author : wangrenlei
/**
 * Writes provided {@link GenericRecord} into the provided
 * {@link OutputStream}.
 */
public static void write(GenericRecord record, OutputStream out) {
    BinaryEncoder encoder = EncoderFactory.get().binaryEncoder(out, null);
    DatumWriter<GenericRecord> writer = new GenericDatumWriter<>(record.getSchema());
    try {
        writer.write(record, encoder);
        encoder.flush();
    } catch (Exception e) {
        throw new IllegalStateException("Failed to write AVRO record", e);
    }
}

15 View Complete Implementation : SafeAvroSerializationIT.java
Copyright Apache License 2.0
Author : apache
private File populateLeftSide(Schema schema) throws IOException {
    File file = tmpDir.getFile("leftSide.avro");
    DatumWriter<GenericRecord> datumWriter = new GenericDatumWriter<GenericRecord>(schema);
    DataFileWriter<GenericRecord> dataFileWriter = new DataFileWriter<GenericRecord>(datumWriter);
    dataFileWriter.create(schema, file);
    GenericRecord record = new GenericData.Record(schema);
    // RECORD 1
    record.put("tag", "b");
    record.put("data1", createString('b', 996100));
    // buffer space has to run out on a write of less than 512 bytes for the
    // issue to occur
    record.put("data2", createString('b', 250));
    dataFileWriter.append(record);
    // RECORD 2 -- this record will be corrupted with overflow from RECORD 1
    record.put("tag", "c");
    record.put("data1", createString('c', 40));
    record.put("data2", null);
    dataFileWriter.append(record);
    dataFileWriter.close();
    return file;
}

15 View Complete Implementation : TestAvroRelConversion.java
Copyright Apache License 2.0
Author : apache
public static <T> byte[] encodeAvroSpecificRecord(Clreplaced<T> clazz, T record) throws IOException {
    DatumWriter<T> msgDatumWriter = new SpecificDatumWriter<>(clazz);
    ByteArrayOutputStream os = new ByteArrayOutputStream();
    Encoder encoder = EncoderFactory.get().binaryEncoder(os, null);
    msgDatumWriter.write(record, encoder);
    encoder.flush();
    return os.toByteArray();
}

15 View Complete Implementation : JdbcCommon.java
Copyright Apache License 2.0
Author : apache
public static void createEmptyAvroStream(final OutputStream outStream) throws IOException {
    final Fieldreplacedembler<Schema> builder = SchemaBuilder.record("NiFi_ExecuteSQL_Record").namespace("any.data").fields();
    final Schema schema = builder.endRecord();
    final DatumWriter<GenericRecord> datumWriter = new GenericDatumWriter<>(schema);
    try (final DataFileWriter<GenericRecord> dataFileWriter = new DataFileWriter<>(datumWriter)) {
        dataFileWriter.create(schema, outStream);
    }
}

15 View Complete Implementation : WriteAvroResultWithExternalSchema.java
Copyright Apache License 2.0
Author : apache
public clreplaced WriteAvroResultWithExternalSchema extends AbstractRecordSetWriter {

    private final SchemaAccessWriter schemaAccessWriter;

    private final RecordSchema recordSchema;

    private final Schema avroSchema;

    private final BinaryEncoder encoder;

    private final OutputStream buffered;

    private final DatumWriter<GenericRecord> datumWriter;

    private final BlockingQueue<BinaryEncoder> recycleQueue;

    public WriteAvroResultWithExternalSchema(final Schema avroSchema, final RecordSchema recordSchema, final SchemaAccessWriter schemaAccessWriter, final OutputStream out, final BlockingQueue<BinaryEncoder> recycleQueue, final ComponentLog logger) {
        super(out);
        this.recordSchema = recordSchema;
        this.schemaAccessWriter = schemaAccessWriter;
        this.avroSchema = avroSchema;
        this.buffered = new BufferedOutputStream(out);
        this.recycleQueue = recycleQueue;
        BinaryEncoder reusableEncoder = recycleQueue.poll();
        if (reusableEncoder == null) {
            logger.debug("Was not able to obtain a BinaryEncoder from reuse pool. This is normal for the first X number of iterations (where X is equal to the max size of the pool), " + "but if this continues, it indicates that increasing the size of the pool will likely yield better performance for this Avro Writer.");
        }
        encoder = EncoderFactory.get().blockingBinaryEncoder(buffered, reusableEncoder);
        datumWriter = new GenericDatumWriter<>(avroSchema);
    }

    @Override
    protected void onBeginRecordSet() throws IOException {
        schemaAccessWriter.writeHeader(recordSchema, buffered);
    }

    @Override
    protected Map<String, String> onFinishRecordSet() throws IOException {
        flush();
        return schemaAccessWriter.getAttributes(recordSchema);
    }

    @Override
    public Map<String, String> writeRecord(final Record record) throws IOException {
        // If we are not writing an active record set, then we need to ensure that we write the
        // schema information.
        if (!isActiveRecordSet()) {
            flush();
            schemaAccessWriter.writeHeader(recordSchema, getOutputStream());
        }
        final GenericRecord rec = AvroTypeUtil.createAvroRecord(record, avroSchema);
        datumWriter.write(rec, encoder);
        return schemaAccessWriter.getAttributes(recordSchema);
    }

    @Override
    public void flush() throws IOException {
        encoder.flush();
        buffered.flush();
    }

    @Override
    public String getMimeType() {
        return "application/avro-binary";
    }

    @Override
    public void close() throws IOException {
        if (encoder != null) {
            flush();
            recycleQueue.offer(encoder);
        }
        super.close();
    }
}

15 View Complete Implementation : ParquetReader.java
Copyright Apache License 2.0
Author : apache
/**
 * Serialize Avro data to a local file.
 * @param file Local destination file for serialization.
 * @throws IOException if the parquet file couldn't be parsed correctly.
 */
public void serializeToDisk(final File file) throws IOException {
    final DatumWriter datumWriter = new GenericDatumWriter<GenericRecord>();
    final DataFileWriter fileWriter = new DataFileWriter<GenericRecord>(datumWriter);
    final AvroParquetReader<GenericRecord> reader = createAvroReader();
    fileWriter.create(createAvroSchema(), file);
    GenericRecord record = reader.read();
    while (record != null) {
        fileWriter.append(record);
        record = reader.read();
    }
    try {
        reader.close();
    } catch (IOException ex) {
        LOG.log(Level.SEVERE, ex.getMessage());
        throw ex;
    }
    try {
        fileWriter.close();
    } catch (IOException ex) {
        LOG.log(Level.SEVERE, ex.getMessage());
        throw ex;
    }
}