org.apache.avro.generic.GenericDatumWriter - java examples

Here are the examples of the java api org.apache.avro.generic.GenericDatumWriter taken from open source projects. By voting up you can indicate which examples are most useful and appropriate.

81 Examples 7

19 View Complete Implementation : RecordBenchmarkBase.java
Copyright Apache License 2.0
Author : RTBHOUSE
@Setup
public void init() throws Exception {
    final GenericDatumWriter<GenericData.Record> datumWriter = new GenericDatumWriter<>(specificRecordSchema);
    for (int i = 0; i < 1000; i++) {
        ByteArrayOutputStream baos = new ByteArrayOutputStream();
        Encoder encoder = EncoderFactory.get().binaryEncoder(baos, null);
        genericRecords.add(FastSerdeBenchmarkSupport.generateRandomRecordData(specificRecordSchema));
        specificRecords.add(FastSerdeBenchmarkSupport.toSpecificRecord(genericRecords.get(genericRecords.size() - 1)));
        datumWriter.write(genericRecords.get(genericRecords.size() - 1), encoder);
        encoder.flush();
        recordBytes.add(baos.toByteArray());
    }
    fastGenericDatumReader = new FastGenericDatumReader<>(specificRecordSchema, cache);
    fastGenericDatumWriter = new FastGenericDatumWriter<>(specificRecordSchema, cache);
    genericDatumReader = new GenericDatumReader<>(specificRecordSchema);
    genericDatumWriter = new GenericDatumWriter<>(specificRecordSchema);
    fastSpecificDatumReader = new FastSpecificDatumReader<>(specificRecordSchema, cache);
    fastSpecificDatumWriter = new FastSpecificDatumWriter<>(specificRecordSchema, cache);
    specificDatumReader = new SpecificDatumReader<>(specificRecordSchema);
    specificDatumWriter = new SpecificDatumWriter<>(specificRecordSchema);
}

19 View Complete Implementation : AvroRecordReaderTest.java
Copyright Apache License 2.0
Author : GoogleCloudDataproc
@Before
public void setup() throws IOException {
    Schema schema = SchemaBuilder.record("BigQueryRecord").fields().name("key").type().stringBuilder().endString().noDefault().name("value1").type().stringBuilder().endString().noDefault().name("value2").type().intBuilder().endInt().noDefault().endRecord();
    GenericDatumWriter<GenericData.Record> recordWriter = new GenericDatumWriter<>(schema);
    testAvroFile = temporaryFolder.newFile("TestAvroFile");
    if (testAvroFile.exists()) {
        testAvroFile.delete();
    }
    DataFileWriter<GenericData.Record> dataFileWriter = new DataFileWriter<>(recordWriter).create(schema, testAvroFile);
    dataFileWriter.setSyncInterval(AUTO_SYNC_INTERVAL);
    ImmutableList.Builder<String> addedKeysBuilder = ImmutableList.builder();
    for (int idx = 0; idx < RECORD_COUNT; idx++) {
        GenericData.Record record = new GenericData.Record(schema);
        String key = String.format("key-%s", idx);
        record.put("key", key);
        record.put("value1", String.format("value-%s", idx));
        record.put("value2", idx * RECORD_COUNT);
        dataFileWriter.append(record);
        addedKeysBuilder.add(key);
    }
    dataFileWriter.close();
    allAddedKeys = addedKeysBuilder.build();
}

19 View Complete Implementation : OracleGenericSchemaDecoder.java
Copyright Apache License 2.0
Author : BriData
/**
 * osource generic schema getInstance
 * Created by Shrimp on 16/5/5.
 */
public clreplaced OracleGenericSchemaDecoder {

    private Logger logger = LoggerFactory.getLogger(getClreplaced());

    private Schema genericSchema;

    private Schema fullPullSchema;

    private int fullPullHash;

    private Schema heartbeatSchema;

    private int heartbeatHash;

    private Schema syncEventSchema;

    private int syncEventHash;

    private GenericDatumReader<GenericRecord> datumReader;

    private GenericDatumWriter<GenericRecord> datumWriter;

    private static OracleGenericSchemaDecoder instance = new OracleGenericSchemaDecoder();

    private OracleGenericSchemaDecoder() {
        initDecoder();
    }

    public static OracleGenericSchemaDecoder getInstance() {
        return instance;
    }

    private void initDecoder() {
        try {
            genericSchema = OracleGenericSchemaProvider.getInstance().getSchema("generic_wrapper.avsc");
            fullPullSchema = OracleGenericSchemaProvider.getInstance().getSchema("DBUS.DB_FULL_PULL_REQUESTS.avsc");
            fullPullHash = OracleGenericSchemaProvider.getInstance().getSchemaHash("DBUS.DB_FULL_PULL_REQUESTS.avsc");
            syncEventSchema = OracleGenericSchemaProvider.getInstance().getSchema("DBUS.META_SYNC_EVENT.avsc");
            syncEventHash = OracleGenericSchemaProvider.getInstance().getSchemaHash("DBUS.META_SYNC_EVENT.avsc");
            heartbeatSchema = OracleGenericSchemaProvider.getInstance().getSchema("DBUS.DB_HEARTBEAT_MONITOR.avsc");
            heartbeatHash = OracleGenericSchemaProvider.getInstance().getSchemaHash("DBUS.DB_HEARTBEAT_MONITOR.avsc");
            datumReader = new GenericDatumReader<>(genericSchema);
            datumWriter = new GenericDatumWriter<>(genericSchema);
        } catch (Exception e) {
            logger.error("OracleGenericSchemaDecoder Initialization Error!", e);
            e.printStackTrace();
        }
    }

    /**
     * 解析被generic schema封装的实际数据
     *
     * @param schema  schema对象
     * @param payload 实际数据
     * @return List<GenericRecord>
     * @throws Exception
     */
    public List<GenericRecord> decode(Schema schema, byte[] payload) throws IOException {
        logger.debug("Schema:" + schema.toString() + " schema payload:" + new String(payload, "utf-8"));
        List<GenericRecord> list = new LinkedList<>();
        DatumReader<GenericRecord> reader = new GenericDatumReader<>(schema);
        BinaryDecoder decoder = getBinaryDecoder(payload);
        while (!decoder.isEnd()) {
            list.add(reader.read(null, decoder));
        }
        return list;
    }

    public List<GenericRecord> decodeSyncEvent(int hash, byte[] payload) throws IOException {
        if (syncEventHash != hash) {
            throw new RuntimeException(String.format("syncEvent schema hash 不一致, 期待的是%s, 实际是%s", syncEventHash, hash));
        }
        return decode(syncEventSchema, payload);
    }

    public List<GenericRecord> decodeFullPull(int hash, byte[] payload) throws IOException {
        if (fullPullHash != hash) {
            throw new RuntimeException(String.format("fullPull schema hash 不一致, 期待的是%s, 实际是%s", fullPullHash, hash));
        }
        return decode(fullPullSchema, payload);
    }

    public List<GenericRecord> decodeHeartBeat(int hash, byte[] payload) throws IOException {
        if (heartbeatHash != hash) {
            throw new RuntimeException(String.format("HeartBeat schema hash 不一致, 期待的是%s, 实际是%s", heartbeatHash, hash));
        }
        return decode(heartbeatSchema, payload);
    }

    public byte[] wrap(List<IGenericMessage> list) throws IOException {
        ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
        BinaryEncoder encoder = EncoderFactory.get().blockingBinaryEncoder(outputStream, null);
        for (IGenericMessage obj : list) {
            OracleGenericMessage msg = (OracleGenericMessage) obj;
            GenericRecord record = msg.generateRecord(genericSchema);
            datumWriter.write(record, encoder);
        }
        encoder.flush();
        return outputStream.toByteArray();
    }

    public List<IGenericMessage> unwrap(byte[] input) throws IOException {
        List<IGenericMessage> list = new LinkedList<>();
        BinaryDecoder decoder = getBinaryDecoder(input);
        while (!decoder.isEnd()) {
            GenericRecord record = datumReader.read(null, decoder);
            OracleGenericMessage msg = new OracleGenericMessage();
            Utf8 utf8 = (Utf8) record.get(OracleGenericMessage.NAMESAPCE);
            msg.setNameSpace(utf8.toString());
            msg.setSchemaHash((Integer) record.get(OracleGenericMessage.SCHEMA_HASH));
            ByteBuffer buffer = (ByteBuffer) record.get(OracleGenericMessage.PAYLOAD);
            msg.setPayload(buffer.array());
            logger.debug(String.format("TAble: %s, HASH: %d\n", msg.getNameSpace(), msg.getSchemaHash()));
            list.add((IGenericMessage) msg);
        }
        return list;
    }

    private BinaryDecoder getBinaryDecoder(byte[] bytes) {
        return DecoderFactory.get().binaryDecoder(bytes, null);
    }
}

19 View Complete Implementation : DailyTrackingWriter.java
Copyright Apache License 2.0
Author : apache
public void open(int year, int month, int day) throws IOException {
    if (_dataWriter != null) {
        throw new RuntimeException("Already have data writer");
    }
    Path dailyPath = _outputPath;
    Path path = new Path(dailyPath, String.format("%04d/%02d/%02d", year, month, day));
    _outputStream = _fs.create(new Path(path, "part-00000.avro"));
    GenericDatumWriter<GenericRecord> writer = new GenericDatumWriter<GenericRecord>();
    _dataWriter = new DataFileWriter<GenericRecord>(writer);
    _dataWriter.create(_schema, _outputStream);
}

19 View Complete Implementation : AvroContainerOutputFormat.java
Copyright Apache License 2.0
Author : jghoman
@Override
public FileSinkOperator.RecordWriter getHiveRecordWriter(JobConf jobConf, Path path, Clreplaced<? extends Writable> valueClreplaced, boolean isCompressed, Properties properties, Progressable progressable) throws IOException {
    Schema schema;
    try {
        schema = HaivvreoUtils.determineSchemaOrThrowException(jobConf, properties);
    } catch (HaivvreoException e) {
        throw new IOException(e);
    }
    GenericDatumWriter<GenericRecord> gdw = new GenericDatumWriter<GenericRecord>(schema);
    DataFileWriter<GenericRecord> dfw = new DataFileWriter<GenericRecord>(gdw);
    if (isCompressed) {
        int level = jobConf.getInt(DEFLATE_LEVEL_KEY, DEFAULT_DEFLATE_LEVEL);
        String codecName = jobConf.get(OUTPUT_CODEC, DEFLATE_CODEC);
        CodecFactory factory = codecName.equals(DEFLATE_CODEC) ? CodecFactory.deflateCodec(level) : CodecFactory.fromString(codecName);
        dfw.setCodec(factory);
    }
    dfw.create(schema, path.getFileSystem(jobConf).create(path));
    return new AvroGenericRecordWriter(dfw);
}

19 View Complete Implementation : AvroContainerOutputFormat.java
Copyright Apache License 2.0
Author : bunnyg
@Override
public FileSinkOperator.RecordWriter getHiveRecordWriter(JobConf jobConf, Path path, Clreplaced<? extends Writable> valueClreplaced, boolean isCompressed, Properties properties, Progressable progressable) throws IOException {
    Schema schema;
    try {
        schema = AvroSerdeUtils.determineSchemaOrThrowException(properties);
    } catch (AvroSerdeException e) {
        throw new IOException(e);
    }
    GenericDatumWriter<GenericRecord> gdw = new GenericDatumWriter<GenericRecord>(schema);
    DataFileWriter<GenericRecord> dfw = new DataFileWriter<GenericRecord>(gdw);
    if (isCompressed) {
        int level = jobConf.getInt(DEFLATE_LEVEL_KEY, DEFAULT_DEFLATE_LEVEL);
        String codecName = jobConf.get(OUTPUT_CODEC, DEFLATE_CODEC);
        CodecFactory factory = codecName.equals(DEFLATE_CODEC) ? CodecFactory.deflateCodec(level) : CodecFactory.fromString(codecName);
        dfw.setCodec(factory);
    }
    dfw.create(schema, path.getFileSystem(jobConf).create(path));
    return new AvroGenericRecordWriter(dfw);
}

19 View Complete Implementation : NamespaceValidationTest.java
Copyright BSD 2-Clause "Simplified" License
Author : linkedin
@Test
public void testAvro14DoesntValidateNamespace() throws Exception {
    AvroVersion runtimeVersion = AvroCompatibilityHelper.getRuntimeAvroVersion();
    if (runtimeVersion != AvroVersion.AVRO_1_4) {
        throw new SkipException("only supported under " + AvroVersion.AVRO_1_4 + ". runtime version detected as " + runtimeVersion);
    }
    String withAvsc = TestUtil.load("HasNamespace.avsc");
    Schema with = Schema.parse(withAvsc);
    String withoutAvsc = TestUtil.load("HasNoNamespace.avsc");
    Schema without = Schema.parse(withoutAvsc);
    GenericData.Record record = new GenericData.Record(without);
    record.put("f", AvroCompatibilityHelper.newEnumSymbol(without.getField("f").schema(), "B"));
    ByteArrayOutputStream os = new ByteArrayOutputStream();
    GenericDatumWriter writer = new GenericDatumWriter(without);
    BinaryEncoder encoder = AvroCompatibilityHelper.newBinaryEncoder(os);
    // noinspection unchecked
    writer.write(record, encoder);
    encoder.flush();
    byte[] bytes = os.toByteArray();
    GenericDatumReader<GenericData.Record> reader = new GenericDatumReader<>(without, with);
    BinaryDecoder decoder = DecoderFactory.defaultFactory().createBinaryDecoder(bytes, null);
    GenericData.Record read = reader.read(null, decoder);
    String value = String.valueOf(read.get("f"));
    replacedert.replacedertEquals(value, "B");
}

19 View Complete Implementation : JsonCodersTest.java
Copyright Apache License 2.0
Author : openaire
private static String toJson(List<GenericRecord> records, Schema schema) throws IOException {
    ByteArrayOutputStream output = new ByteArrayOutputStream();
    HackedJsonEncoder jsonEncoder = new HackedJsonEncoder(schema, output);
    GenericDatumWriter<GenericRecord> writer = new GenericDatumWriter<GenericRecord>(schema);
    for (GenericRecord record : records) {
        writer.write(record, jsonEncoder);
    }
    jsonEncoder.flush();
    output.flush();
    return output.toString();
}

19 View Complete Implementation : DataGeneratorWriter.java
Copyright Apache License 2.0
Author : Hanmourang
public void generate() throws IOException {
    Schema schemaJson = Schema.parse(config.schemaFile);
    final GenericDatumWriter<GenericData.Record> datum = new GenericDatumWriter<GenericData.Record>(schemaJson);
    DataFileWriter<GenericData.Record> recordWriter = new DataFileWriter<GenericData.Record>(datum);
    Map<String, Double> minMap = new HashMap<String, Double>();
    Map<String, Double> maxMap = new HashMap<String, Double>();
    for (DimensionSpec dimension : config.starTreeConfig.getDimensions()) {
        minMap.put(dimension.getName(), (double) 0);
        maxMap.put(dimension.getName(), (double) (dimensionValues.get(dimension.getName()).size() - 1));
    }
    int recordNo = 0;
    int fileNo = 0;
    for (long time = config.startTime; time <= config.endTime; time++) {
        for (long record = 0; record < config.numRecordsPerTimeUnit; record++) {
            if (recordNo % config.numRecordsPerFile == 0) {
                File avroFile = new File(PATH_JOINER.join(config.outputDataDirectory, "part-" + fileNo + ".avro"));
                if (avroFile.exists())
                    avroFile.delete();
                if (fileNo > 0)
                    recordWriter.close();
                recordWriter.create(schemaJson, avroFile);
                fileNo++;
            }
            recordNo++;
            GenericData.Record outRecord = new GenericData.Record(schemaJson);
            // Write time
            outRecord.put(config.starTreeConfig.getTime().getColumnName(), time);
            Random rand = new Random();
            // Write dimensions
            for (DimensionSpec dimensionSpec : config.starTreeConfig.getDimensions()) {
                String dimension = dimensionSpec.getName();
                int cardinality = dimensionValues.get(dimension).size();
                double mean = (cardinality) / 2;
                double variance = cardinality / 2;
                double gaussianPick = mean + rand.nextGaussian() * variance;
                if (gaussianPick < minMap.get(dimension))
                    minMap.put(dimension, gaussianPick);
                else if (gaussianPick > maxMap.get(dimension))
                    maxMap.put(dimension, gaussianPick);
                int normalizedValue = (int) ((cardinality - 1) / (maxMap.get(dimension) - minMap.get(dimension)) * (gaussianPick - maxMap.get(dimension)) + (cardinality - 1));
                outRecord.put(dimension, dimensionValues.get(dimension).get(normalizedValue));
            }
            // Write metrics
            for (MetricSpec metricSpec : config.starTreeConfig.getMetrics()) {
                String metric = metricSpec.getName();
                long metricValue = rand.nextInt(100);
                outRecord.put(metric, metricValue);
            }
            recordWriter.append(outRecord);
        }
    }
    recordWriter.close();
}

19 View Complete Implementation : TestAvroEventParser.java
Copyright Apache License 2.0
Author : flumebase
/**
 * Make an Avro record with the given schema.
 */
private Event makeEvent(GenericData.Record record, Schema schema) throws IOException {
    GenericDatumWriter<GenericRecord> datumWriter = new GenericDatumWriter<GenericRecord>(schema);
    ByteArrayOutputStream outBytes = new ByteArrayOutputStream();
    BinaryEncoder encoder = new BinaryEncoder(outBytes);
    datumWriter.write(record, encoder);
    return new EventImpl(outBytes.toByteArray());
}

19 View Complete Implementation : AvroRecursionEliminatingConverterTest.java
Copyright Apache License 2.0
Author : apache
public File generateRecord() throws IOException {
    Schema inputSchema = new Schema.Parser().parse(getClreplaced().getResourcereplacedtream("/converter/recursive.avsc"));
    GenericDatumWriter<GenericRecord> datumWriter = new GenericDatumWriter<GenericRecord>(inputSchema);
    GenericRecord record = new GenericData.Record(inputSchema);
    record.put("name", "John");
    record.put("date_of_birth", 1234L);
    record.put("last_modified", 4567L);
    record.put("created", 6789L);
    GenericRecord addressRecord = new GenericData.Record(inputSchema.getField("address").schema());
    addressRecord.put("city", "Los Angeles");
    addressRecord.put("street_number", 1234);
    GenericRecord innerAddressRecord = new GenericData.Record(inputSchema.getField("address").schema());
    innerAddressRecord.put("city", "San Francisco");
    innerAddressRecord.put("street_number", 3456);
    addressRecord.put("previous_address", innerAddressRecord);
    record.put("address", addressRecord);
    File recordFile = File.createTempFile(this.getClreplaced().getSimpleName(), "avsc");
    DataFileWriter<GenericRecord> dataFileWriter = new DataFileWriter<GenericRecord>(datumWriter);
    dataFileWriter.create(inputSchema, recordFile);
    dataFileWriter.append(record);
    dataFileWriter.close();
    recordFile.deleteOnExit();
    return recordFile;
}

19 View Complete Implementation : TestAvroMapReader.java
Copyright Apache License 2.0
Author : kijiproject
@Test
public void testReload() throws IOException {
    // Create an Avro "map" using serialization/deserialization.
    final Map<CharSequence, Integer> originalMap = new TreeMap<CharSequence, Integer>(new AvroCharSequenceComparator<CharSequence>());
    originalMap.put("to be deleted", 42);
    final ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
    final GenericDatumWriter<Map<CharSequence, Integer>> writer = new GenericDatumWriter<Map<CharSequence, Integer>>(Schema.createMap(Schema.create(Schema.Type.INT)));
    writer.write(originalMap, EncoderFactory.get().directBinaryEncoder(outputStream, null));
    final ByteArrayInputStream inputStream = new ByteArrayInputStream(outputStream.toByteArray());
    final GenericDatumReader<Map<CharSequence, Integer>> reader = new GenericDatumReader<Map<CharSequence, Integer>>(Schema.createMap(Schema.create(Schema.Type.INT)));
    final Map<CharSequence, Integer> deserializedMap = reader.read(null, DecoderFactory.get().binaryDecoder(inputStream, null));
    // Create a reader for it.
    final AvroMapReader<Integer> mapReader = AvroMapReader.create(deserializedMap);
    // Tests before modifying the original map.
    replacedertEquals(42, mapReader.get(new Utf8("to be deleted")).intValue());
    replacedertFalse(mapReader.containsKey(new Utf8("to be added")));
    // Tests after modifying the avro map but before calling reload.
    // The reader shouldn't yet reflect the changes to the 'wrapped' map.
    deserializedMap.put("to be added", 23);
    deserializedMap.remove(new Utf8("to be deleted"));
    replacedertTrue(deserializedMap.containsKey("to be added"));
    replacedertEquals(23, deserializedMap.get("to be added").intValue());
    replacedertFalse(deserializedMap.containsKey("to be deleted"));
    replacedertEquals(42, mapReader.get(new Utf8("to be deleted")).intValue());
    replacedertFalse(mapReader.containsKey(new Utf8("to be added")));
    // Tests after calling reload.
    mapReader.reload();
    replacedertFalse(mapReader.containsKey(new Utf8("to be deleted")));
    replacedertEquals(23, mapReader.get(new Utf8("to be added")).intValue());
}

19 View Complete Implementation : AvroImport.java
Copyright Apache License 2.0
Author : usc-isi-i2
@Override
public Worksheet generateWorksheet() throws JSONException, IOException, KarmaException {
    DataFileReader<Void> schemareader = new DataFileReader<>(file, new GenericDatumReader<Void>());
    Schema schema = schemareader.getSchema();
    schemareader.close();
    DataFileReader<GenericRecord> reader = new DataFileReader<>(file, new GenericDatumReader<GenericRecord>(schema));
    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    baos.write('[');
    baos.write('\n');
    GenericDatumWriter<GenericRecord> writer = new GenericDatumWriter<>(reader.getSchema());
    while (reader.hasNext()) {
        GenericRecord record = reader.next();
        JsonEncoder encoder = EncoderFactory.get().jsonEncoder(reader.getSchema(), new JsonFactory().createJsonGenerator(baos)).configure(baos);
        writer.write(record, encoder);
        encoder.flush();
        if (reader.hasNext()) {
            baos.write(',');
        }
    }
    reader.close();
    baos.write('\n');
    baos.write(']');
    baos.flush();
    baos.close();
    String json = new String(baos.toByteArray());
    JsonImport jsonImport = new JsonImport(json, this.getFactory(), this.getWorksheet(), workspace, maxNumLines);
    return jsonImport.generateWorksheet();
}

19 View Complete Implementation : TimestampDataWriter.java
Copyright Apache License 2.0
Author : apache
public void open(int year, int month, int day) throws IOException {
    if (_dataWriter != null) {
        throw new RuntimeException("Already have data writer");
    }
    Path path = new Path(_outputPath, String.format("%04d%02d%02d", year, month, day));
    _outputStream = _fs.create(new Path(path, "part-00000.avro"));
    GenericDatumWriter<GenericRecord> writer = new GenericDatumWriter<GenericRecord>();
    _dataWriter = new DataFileWriter<GenericRecord>(writer);
    _dataWriter.create(_schema, _outputStream);
}

19 View Complete Implementation : AvroGenericRecordSerializer.java
Copyright Apache License 2.0
Author : apache
/**
 * A {@link JsonSerializer} for {@link GenericRecord} objects.
 */
@Slf4j
public clreplaced AvroGenericRecordSerializer implements JsonSerializer<GenericRecord> {

    private final ByteArrayOutputStream byteArrayOutputStream;

    private final DataOutputStream out;

    private final GenericDatumWriter<GenericRecord> writer;

    private final Closer closer;

    public AvroGenericRecordSerializer() {
        this.closer = Closer.create();
        this.byteArrayOutputStream = new ByteArrayOutputStream();
        this.out = this.closer.register(new DataOutputStream(this.byteArrayOutputStream));
        this.writer = new GenericDatumWriter<GenericRecord>();
    }

    @Override
    public void configure(Config config) {
    }

    @Override
    public synchronized byte[] serializeToJson(GenericRecord serializable) throws SerializationException {
        try {
            /**
             * We use the toString method of Avro to flatten the JSON for optional nullable types.
             * Otherwise the JSON has an additional level of nesting to encode the type.
             * e.g. "id": {"string": "id-value"} versus "id": "id-value"
             * See {@link: https://issues.apache.org/jira/browse/AVRO-1582} for a good discussion on this.
             */
            String serialized = serializable.toString();
            return serialized.getBytes(Charset.forName("UTF-8"));
        } catch (Exception exception) {
            throw new SerializationException("Could not serializeToJson Avro record", exception);
        }
    }

    @Override
    public void close() throws IOException {
        this.closer.close();
    }
}

19 View Complete Implementation : KafkaEventStore.java
Copyright GNU Affero General Public License v3.0
Author : rakam-io
@Override
public void store(Event event) {
    GenericDatumWriter writer = new SourceFilteredRecordWriter(event.properties().getSchema(), GenericData.get(), sourceFields);
    ByteBuf buffer = Unpooled.buffer(100);
    BinaryEncoder encoder = EncoderFactory.get().directBinaryEncoder(new ByteBufOutputStream(buffer), null);
    try {
        writer.write(event.properties(), encoder);
    } catch (Exception e) {
        throw new RuntimeException("Couldn't serialize event", e);
    }
    try {
        producer.send(new KeyedMessage<>(event.project() + "_" + event.collection(), buffer.array()));
    } catch (FailedToSendMessageException e) {
        throw new RuntimeException("Couldn't send event to Kafka", e);
    }
}

19 View Complete Implementation : Db2GenericSchemaDecoder.java
Copyright Apache License 2.0
Author : BriData
/**
 * osource generic schema getInstance
 * Created by Shrimp on 16/5/5.
 */
public clreplaced Db2GenericSchemaDecoder {

    private Logger logger = LoggerFactory.getLogger(getClreplaced());

    private Schema genericSchema;

    private SchemaRegistryClient schemaRegistry;

    private GenericDatumReader<GenericRecord> datumReader;

    private GenericDatumWriter<GenericRecord> datumWriter;

    private static Db2GenericSchemaDecoder instance = new Db2GenericSchemaDecoder();

    private Db2GenericSchemaDecoder() {
        initDecoder();
    }

    public static Db2GenericSchemaDecoder getInstance(String restUrl) {
        instance.schemaRegistry = new CachedSchemaRegistryClient(restUrl, 20, null);
        return instance;
    }

    private void initDecoder() {
        try {
            genericSchema = Db2GenericSchemaProvider.getInstance().getSchema("db2_generic_wrapper.avsc");
            datumReader = new GenericDatumReader<>(genericSchema);
            datumWriter = new GenericDatumWriter<>(genericSchema);
        } catch (Exception e) {
            logger.error("Db2GenericSchemaDecoder Initialization Error!", e);
            e.printStackTrace();
        }
    }

    /**
     * 解析被generic schema封装的实际数据
     *
     * @param schema  schema对象
     * @param payload 实际数据
     * @return List<GenericRecord>
     * @throws Exception
     */
    public List<GenericRecord> decode(Db2GenericMessage msg) throws IOException {
        ByteBuffer buffer = ByteBuffer.wrap(msg.getPayload());
        if (buffer.get() != Constants.MAGIC_BYTE) {
            logger.error("Unknown magic byte!");
        }
        int id = buffer.getInt();
        Schema schema = null;
        try {
            schema = ThreadLocalCache.get(Constants.CacheNames.TABLE_SCHEMA_VERSION_CACHE, Utils.buildDataTableSchemaCacheKey(msg.getSchemaName(), msg.getTableName(), String.valueOf(id)));
            if (schema == null) {
                schema = schemaRegistry.getById(id);
                ThreadLocalCache.put(Constants.CacheNames.TABLE_SCHEMA_VERSION_CACHE, Utils.buildDataTableSchemaCacheKey(msg.getSchemaName(), msg.getTableName(), String.valueOf(id)), schema);
            }
        } catch (RestClientException e) {
            logger.error("Schema Registry RestClientException: " + e);
        }
        int length = buffer.limit() - 5;
        int start = buffer.position() + buffer.arrayOffset();
        // logger.debug("Schema:" + schema.toString() + " schema payload:" + new String(msg.getPayload(), "utf-8"));
        List<GenericRecord> list = new LinkedList<>();
        DatumReader<GenericRecord> reader = new GenericDatumReader<>(schema);
        BinaryDecoder decoder = DecoderFactory.get().binaryDecoder(buffer.array(), start, length, null);
        GenericRecord genericRecord = reader.read(null, decoder);
        list.add((GenericRecord) genericRecord);
        return list;
    }

    public List<GenericRecord> decodeFullPull(Db2GenericMessage msg) throws IOException {
        return decode(msg);
    }

    public List<GenericRecord> decodeHeartBeat(Db2GenericMessage msg) throws IOException {
        return decode(msg);
    }

    public byte[] wrap(List<IGenericMessage> list) throws IOException {
        ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
        BinaryEncoder encoder = EncoderFactory.get().blockingBinaryEncoder(outputStream, null);
        for (IGenericMessage obj : list) {
            Db2GenericMessage msg = (Db2GenericMessage) obj;
            GenericRecord record = msg.generateRecord(genericSchema);
            datumWriter.write(record, encoder);
        }
        encoder.flush();
        return outputStream.toByteArray();
    }

    public List<IGenericMessage> unwrap(DBusConsumerRecord record) throws IOException {
        List<IGenericMessage> list = new LinkedList<>();
        String[] schemaTableInfo = StringUtils.split(record.topic(), '.');
        if (schemaTableInfo.length != 3) {
            logger.error("record topic is error: [" + record.topic() + "]");
        }
        String dbSchema = schemaTableInfo[1];
        String tableName = schemaTableInfo[2];
        if (record.value() == null) {
            logger.error("DBusConsumerRecord record.value() is null!");
        } else {
            ByteBuffer buffer = ByteBuffer.wrap((byte[]) record.value());
            if (buffer.get() != Constants.MAGIC_BYTE) {
                logger.error("Unknown magic byte!");
            }
            int id = buffer.getInt();
            Schema schema = null;
            try {
                // 这里先去cache中查询数据的schema,如果cache中没有,则再去schema registry中查询,并写入cache中
                schema = ThreadLocalCache.get(Constants.CacheNames.TABLE_SCHEMA_VERSION_CACHE, Utils.buildDataTableSchemaCacheKey(dbSchema, tableName, String.valueOf(id)));
                if (schema == null) {
                    schema = schemaRegistry.getById(id);
                    ThreadLocalCache.put(Constants.CacheNames.TABLE_SCHEMA_VERSION_CACHE, Utils.buildDataTableSchemaCacheKey(dbSchema, tableName, String.valueOf(id)), schema);
                }
            } catch (RestClientException e) {
                logger.error("Schema Registry RestClientException: " + e);
            }
            int length = buffer.limit() - 5;
            int start = buffer.position() + buffer.arrayOffset();
            DatumReader<GenericRecord> reader = new GenericDatumReader<>(schema);
            BinaryDecoder decoder = DecoderFactory.get().binaryDecoder(buffer.array(), start, length, null);
            GenericRecord genericRecord = reader.read(null, decoder);
            if (!StringUtils.equals(Constants.DB2MessageBoreplacedy.RR, (CharSequence) genericRecord.get(Constants.DB2MessageBoreplacedy.DB2_ENTTYP))) {
                Db2GenericMessage msg = new Db2GenericMessage();
                String[] arrList = StringUtils.split(schema.getFullName(), ".");
                msg.setNameSpace(StringUtils.joinWith(".", arrList[2], arrList[3]));
                msg.setPayload((byte[]) record.value());
                msg.setSchemaId(id);
                msg.setOffset(String.valueOf(record.offset()));
                list.add(msg);
            }
        }
        return list;
    }

    private BinaryDecoder getBinaryDecoder(byte[] bytes) {
        return DecoderFactory.get().binaryDecoder(bytes, null);
    }
}

19 View Complete Implementation : BoomWriter.java
Copyright Apache License 2.0
Author : blackberry
private DataFileWriter<GenericRecord> newWriter() throws IOException {
    GenericDatumWriter<GenericRecord> datumWriter = new GenericDatumWriter<GenericRecord>(logBlockSchema);
    writer = new DataFileWriter<GenericRecord>(datumWriter);
    writer.setCodec(CodecFactory.deflateCodec(getDeflateLevel()));
    writer.setSyncInterval(getAvroBlockSize());
    writer.create(logBlockSchema, out);
    return writer;
}

19 View Complete Implementation : AvroToBytesConverter.java
Copyright Apache License 2.0
Author : apache
/**
 * Convert an Avro GenericRecord back to its byte representation. Note: This converter returns
 * the raw bytes for a record - it does not return a container file. If you want to write
 * Avro records out to a container file do not use this converter; instead use the AvroDataWriter
 * writer.
 */
public clreplaced AvroToBytesConverter extends Converter<Schema, String, GenericRecord, byte[]> {

    private GenericDatumWriter<GenericRecord> writer;

    private ThreadLocal<BinaryEncoder> encoderCache = new ThreadLocal<BinaryEncoder>() {

        @Override
        protected BinaryEncoder initialValue() {
            return null;
        }
    };

    @Override
    public String convertSchema(Schema inputSchema, WorkUnitState workUnit) throws SchemaConversionException {
        writer = new GenericDatumWriter<GenericRecord>(inputSchema);
        return inputSchema.toString();
    }

    @Override
    public Iterable<byte[]> convertRecord(String outputSchema, GenericRecord inputRecord, WorkUnitState workUnit) throws DataConversionException {
        try {
            ByteArrayOutputStream bytesOut = new ByteArrayOutputStream();
            BinaryEncoder encoder = EncoderFactory.get().binaryEncoder(bytesOut, encoderCache.get());
            encoderCache.set(encoder);
            writer.write(inputRecord, encoder);
            encoder.flush();
            return Collections.singleton(bytesOut.toByteArray());
        } catch (IOException e) {
            throw new DataConversionException("Error serializing record", e);
        }
    }
}

19 View Complete Implementation : TestAvroMapReader.java
Copyright Apache License 2.0
Author : kijiproject
@Test
public void testMap() throws IOException {
    // Create a map.
    final Map<CharSequence, Integer> originalMap = new TreeMap<CharSequence, Integer>(new AvroCharSequenceComparator<CharSequence>());
    originalMap.put("foo", 42);
    replacedertTrue(originalMap.containsKey(new Utf8("foo")));
    replacedertTrue(originalMap.containsKey("foo"));
    final AvroMapReader<Integer> originalMapReader = AvroMapReader.create(originalMap);
    replacedertTrue(originalMapReader.containsKey(new Utf8("foo")));
    replacedertEquals(42, originalMapReader.get(new Utf8("foo")).intValue());
    replacedertTrue(originalMapReader.containsKey("foo"));
    replacedertEquals(42, originalMapReader.get("foo").intValue());
    // Serialize it to a stream.
    final ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
    final GenericDatumWriter<Map<CharSequence, Integer>> writer = new GenericDatumWriter<Map<CharSequence, Integer>>(Schema.createMap(Schema.create(Schema.Type.INT)));
    writer.write(originalMap, EncoderFactory.get().directBinaryEncoder(outputStream, null));
    // Read from serialized stream.
    final ByteArrayInputStream inputStream = new ByteArrayInputStream(outputStream.toByteArray());
    final GenericDatumReader<Map<CharSequence, Integer>> reader = new GenericDatumReader<Map<CharSequence, Integer>>(Schema.createMap(Schema.create(Schema.Type.INT)));
    // Deserialize the map.
    final Map<CharSequence, Integer> deserializedMap = reader.read(null, DecoderFactory.get().binaryDecoder(inputStream, null));
    // Both of these *should* work, but Avro is broken, so the check for a String key "foo" fails.
    replacedertTrue(deserializedMap.containsKey(new Utf8("foo")));
    replacedertFalse("Avro has been fixed! AvroMapReader is no longer necessary.", deserializedMap.containsKey("foo"));
    // Use the reader.  It should work just fine with Strings or Utf8's.
    final AvroMapReader<Integer> mapReader = AvroMapReader.create(deserializedMap);
    replacedertTrue(mapReader.containsKey(new Utf8("foo")));
    replacedertEquals(42, mapReader.get(new Utf8("foo")).intValue());
    replacedertTrue(mapReader.containsKey("foo"));
    replacedertEquals(42, mapReader.get("foo").intValue());
}

18 View Complete Implementation : SimpleAvroTest.java
Copyright Apache License 2.0
Author : datafibers-community
public static void main(String[] args) throws InterruptedException {
    Schema.Parser parser = new Schema.Parser();
    Schema schema = parser.parse(USER_SCHEMA);
    GenericRecord user1 = new GenericData.Record(schema);
    user1.put("name", "Alyssa");
    user1.put("symbol", "CHINA");
    user1.put("exchangecode", "TEST");
    try {
        final ByteArrayOutputStream out = new ByteArrayOutputStream();
        GenericDatumWriter<GenericRecord> writer = new GenericDatumWriter<GenericRecord>(schema);
        BinaryEncoder encoder = EncoderFactory.get().directBinaryEncoder(out, null);
        writer.write(user1, encoder);
        encoder.flush();
        GenericDatumReader<GenericRecord> reader = new GenericDatumReader<GenericRecord>(schema);
        BinaryDecoder decoder = DecoderFactory.get().binaryDecoder(out.toByteArray(), null);
        GenericRecord gr = reader.read(null, decoder);
        System.out.println(gr.toString());
        System.out.println("******Get Fields Names");
        List<String> stringList = new ArrayList<String>();
        if (RECORD.equals(schema.getType()) && schema.getFields() != null && !schema.getFields().isEmpty()) {
            for (org.apache.avro.Schema.Field field : schema.getFields()) {
                stringList.add(field.name());
            }
        }
        String[] fieldNames = stringList.toArray(new String[] {});
        for (String element : fieldNames) {
            System.out.println(element);
        }
        System.out.println("******Get Fields Types");
        int fieldsLen = schema.getFields().size();
        Clreplaced<?>[] fieldTypes = new Clreplaced[fieldsLen];
        int index = 0;
        String typeName;
        try {
            if (RECORD.equals(schema.getType()) && schema.getFields() != null && !schema.getFields().isEmpty()) {
                for (org.apache.avro.Schema.Field field : schema.getFields()) {
                    typeName = field.schema().getType().getName().toLowerCase();
                    // Mapping Avro type to Java type - TODO Complex type is not supported yet
                    switch(typeName) {
                        case "boolean":
                        case "string":
                        case "long":
                        case "float":
                            fieldTypes[index] = Clreplaced.forName("java.lang." + StringUtils.capitalize(typeName));
                            break;
                        case "bytes":
                            fieldTypes[index] = Clreplaced.forName("java.util.Byte");
                            break;
                        case "int":
                            fieldTypes[index] = Clreplaced.forName("java.lang.Integer");
                            break;
                        default:
                            fieldTypes[index] = Clreplaced.forName("java.lang." + StringUtils.capitalize(typeName));
                    }
                    index++;
                }
            }
        } catch (ClreplacedNotFoundException cnf) {
            cnf.printStackTrace();
        }
        for (Clreplaced<?> element : fieldTypes) {
            System.out.println(element);
        }
        System.out.println("TestCase_Test Schema Register Client");
        Properties properties = new Properties();
        properties.setProperty("bootstrap.servers", "localhost:9092");
        properties.setProperty("group.id", "consumer_test");
        properties.setProperty("schema.subject", "test-value");
        properties.setProperty("schema.registry", "localhost:8081");
        try {
            Schema schema1 = SchemaRegistryClient.getLatestSchemaFromProperty(properties);
            System.out.println("raw schema1 for name is " + schema1.getField("name"));
            String USER_SCHEMA = "{" + "\"type\":\"record\"," + "\"name\":\"test\"," + "\"fields\":[" + "  { \"name\":\"name\", \"type\":\"string\" }," + "  { \"name\":\"symbol\", \"type\":\"string\" }," + "  { \"name\":\"exchangecode\", \"type\":\"string\" }" + "]}";
            Schema.Parser parser2 = new Schema.Parser();
            Schema schema2 = parser2.parse(USER_SCHEMA);
            System.out.println("raw schema2 for name is " + schema.getField("name"));
        } catch (Exception e) {
            e.printStackTrace();
        }
    } catch (IOException ioe) {
        ioe.printStackTrace();
    }
}

18 View Complete Implementation : AvroOutputElementImpl.java
Copyright Apache License 2.0
Author : flumebase
/**
 * Abstract clreplaced that allows nodes to emit output records which
 * are serialized in avro format.
 */
public abstract clreplaced AvroOutputElementImpl extends FlowElementImpl {

    private static final Logger LOG = LoggerFactory.getLogger(AvroOutputElementImpl.clreplaced.getName());

    // Avro encoder components reused in our internal workflow.
    private BinaryEncoder mEncoder;

    private GenericDatumWriter<GenericRecord> mDatumWriter;

    private ByteArrayOutputStream mOutputBytes;

    private Schema mOutputSchema;

    public AvroOutputElementImpl(FlowElementContext ctxt, Schema outputSchema) {
        super(ctxt);
        mDatumWriter = new GenericDatumWriter<GenericRecord>(outputSchema);
        mOutputBytes = new ByteArrayOutputStream();
        mEncoder = new BinaryEncoder(mOutputBytes);
        mOutputSchema = outputSchema;
    }

    /**
     * Create a new output Event that encapsulates the specified record,
     * and emit it to the output context.
     * @param record the avro record to emit to the output context.
     * @param inEvent the input event to the current FlowElement; properties
     * of this event are propagated forward into the output event.
     */
    protected void emitAvroRecord(GenericData.Record record, Event inEvent) throws IOException, InterruptedException {
        emitAvroRecord(record, inEvent, inEvent.getTimestamp());
    }

    protected void emitAvroRecord(GenericData.Record record, Event inEvent, long timestamp) throws IOException, InterruptedException {
        emitAvroRecord(record, inEvent, inEvent.getTimestamp(), getContext());
    }

    protected void emitAvroRecord(GenericData.Record record, Event inEvent, long timestamp, FlowElementContext context) throws IOException, InterruptedException {
        // TODO: BAOS.toByteArray() creates a new byte array, as does the
        // creation of the event. That's at least one more array copy than
        // necessary.
        mOutputBytes.reset();
        try {
            LOG.debug("Emitting record");
            for (Schema.Field f : record.getSchema().getFields()) {
                StringBuilder sb = new StringBuilder();
                sb.append("  " + f.name() + " => " + record.get(f.name()));
                sb.append(" (");
                Object out = record.get(f.name());
                if (null == out) {
                    sb.append("Null");
                } else {
                    sb.append(out.getClreplaced().getName());
                }
                sb.append(")");
                LOG.debug(sb.toString());
            }
            mDatumWriter.write(record, mEncoder);
        } catch (NullPointerException npe) {
            // Schema error - the user tried to put a null in a field declared non-null.
            // We silently elide the entire record.
            LOG.debug("Omitting record with NULL value in non-null field: " + npe.toString());
            return;
        }
        Event out = new EventImpl(mOutputBytes.toByteArray(), timestamp, inEvent.getPriority(), inEvent.getNanos(), inEvent.getHost());
        AvroEventWrapper outWrapper = new AvroEventWrapper(mOutputSchema);
        outWrapper.reset(out);
        emit(outWrapper, context);
    }

    protected Schema getOutputSchema() {
        return mOutputSchema;
    }

    /**
     * Given a native Java object, return the object that represents this
     * value in an Avro GenericRecord where we control the schema.
     *
     * @param val the value (held in a native Java object), to convert to
     * an object representing this same value in an Avro GenericRecord.
     * @param outputType the rtsql type replacedociated with the type we're
     * converting this object to.
     *
     * @see AvroEventParser.avroToNative().
     */
    protected Object nativeToAvro(Object val, Type outputType) {
        if (null == val) {
            return null;
        } else if (val instanceof BigDecimal) {
            if (outputType.getPrimitiveTypeName().equals(Type.TypeName.PRECISE)) {
                // If we know the scale replacedociated with the result PRECISE object,
                // make sure we use the appropriate scale (the scale used during
                // computation may be overly precise as a result of math operations).
                PreciseType preciseType = PreciseType.toPreciseType(outputType);
                BigDecimal bigDec = (BigDecimal) val;
                return bigDec.setScale(preciseType.getScale(), RoundingMode.HALF_EVEN).toString();
            } else {
                // Not sure what would cause a BigDecimal to be coerced to some
                // non-PRECISE output type, but in any case, it will be represented
                // as a string.
                return val.toString();
            }
        } else {
            // For all other value types, we use the same type internally
            // as Avro does.
            return val;
        }
    }
}

18 View Complete Implementation : AvroSupport.java
Copyright Apache License 2.0
Author : ggj2010
public static byte[] dataToByteArray(Schema schema, GenericRecord datum) throws IOException {
    GenericDatumWriter<GenericRecord> writer = new GenericDatumWriter<GenericRecord>(schema);
    ByteArrayOutputStream os = new ByteArrayOutputStream();
    try {
        Encoder e = EncoderFactory.get().binaryEncoder(os, null);
        writer.write(datum, e);
        e.flush();
        byte[] byteData = os.toByteArray();
        return byteData;
    } finally {
        os.close();
    }
}

18 View Complete Implementation : TestThatEvolvedSchemasActAsWeWant.java
Copyright Apache License 2.0
Author : jghoman
@Test
public void resolvedSchemreplacedhouldReturnReaderSchema() throws IOException {
    // Need to verify that when reading a datum with an updated reader schema
    // that the datum then returns the reader schema as its own, since we
    // depend on this behavior in order to avoid re-encoding the datum
    // in the serde.
    String v0 = "{\n" + "    \"namespace\": \"com.linkedin.haivvreo\",\n" + "    \"name\": \"SomeStuff\",\n" + "    \"type\": \"record\",\n" + "    \"fields\": [\n" + "        {\n" + "            \"name\":\"v0\",\n" + "            \"type\":\"string\"\n" + "        }\n" + "    ]\n" + "}";
    String v1 = "{\n" + "    \"namespace\": \"com.linkedin.haivvreo\",\n" + "    \"name\": \"SomeStuff\",\n" + "    \"type\": \"record\",\n" + "    \"fields\": [\n" + "        {\n" + "            \"name\":\"v0\",\n" + "            \"type\":\"string\"\n" + "        },\n" + "        {\n" + "            \"name\":\"v1\",\n" + "            \"type\":\"string\",\n" + "            \"default\":\"v1_default\"" + "        }\n" + "    ]\n" + "}";
    Schema[] schemas = { Schema.parse(v0), Schema.parse(v1) };
    // Encode a schema with v0, write out.
    GenericRecord record = new GenericData.Record(schemas[0]);
    record.put("v0", "v0 value");
    replacedertTrue(GenericData.get().validate(schemas[0], record));
    // Write datum out to a stream
    GenericDatumWriter<GenericRecord> gdw = new GenericDatumWriter<GenericRecord>(schemas[0]);
    DataFileWriter<GenericRecord> dfw = new DataFileWriter<GenericRecord>(gdw);
    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    dfw.create(schemas[0], baos);
    dfw.append(record);
    dfw.close();
    ByteArrayInputStream bais = new ByteArrayInputStream(baos.toByteArray());
    GenericDatumReader<GenericRecord> gdr = new GenericDatumReader<GenericRecord>();
    gdr.setExpected(schemas[1]);
    DataFileStream<GenericRecord> dfs = new DataFileStream<GenericRecord>(bais, gdr);
    replacedertTrue(dfs.hasNext());
    GenericRecord next = dfs.next();
    replacedertEquals("v0 value", next.get("v0").toString());
    replacedertEquals("v1_default", next.get("v1").toString());
    // Now the most important check - when we query this record for its schema,
    // we should get back the latest, reader schema:
    replacedertEquals(schemas[1], next.getSchema());
}

18 View Complete Implementation : AvroGenericRecordResolvingSerializer.java
Copyright Apache License 2.0
Author : jingwei
@Override
public byte[] serialize(GenericRecord record) throws SerializationException {
    if (record == null) {
        return null;
    }
    try {
        ByteArrayOutputStream out = new ByteArrayOutputStream();
        Encoder encoder = new BinaryEncoder(out);
        GenericDatumWriter<GenericRecord> datumWriter = new GenericDatumWriter<GenericRecord>(_writer);
        datumWriter.write(record, encoder);
        encoder.flush();
        return out.toByteArray();
    } catch (Exception e) {
        throw new SerializationException("Failed to serialize", e);
    }
}

18 View Complete Implementation : AvroGenericRecordSerializer.java
Copyright Apache License 2.0
Author : jingwei
@Override
public byte[] serialize(GenericRecord record) throws SerializationException {
    if (record == null) {
        return null;
    }
    try {
        ByteArrayOutputStream out = new ByteArrayOutputStream();
        Encoder encoder = new BinaryEncoder(out);
        GenericDatumWriter<GenericRecord> datumWriter = new GenericDatumWriter<GenericRecord>(_schema);
        datumWriter.write(record, encoder);
        encoder.flush();
        return out.toByteArray();
    } catch (Exception e) {
        throw new SerializationException("Failed to serialize", e);
    }
}

18 View Complete Implementation : TestThatEvolvedSchemasActAsWeWant.java
Copyright Apache License 2.0
Author : bunnyg
@Test
public void resolvedSchemreplacedhouldReturnReaderSchema() throws IOException {
    // Need to verify that when reading a datum with an updated reader schema
    // that the datum then returns the reader schema as its own, since we
    // depend on this behavior in order to avoid re-encoding the datum
    // in the serde.
    String v0 = "{\n" + "    \"namespace\": \"org.apache.hadoop.hive\",\n" + "    \"name\": \"SomeStuff\",\n" + "    \"type\": \"record\",\n" + "    \"fields\": [\n" + "        {\n" + "            \"name\":\"v0\",\n" + "            \"type\":\"string\"\n" + "        }\n" + "    ]\n" + "}";
    String v1 = "{\n" + "    \"namespace\": \"org.apache.hadoop.hive\",\n" + "    \"name\": \"SomeStuff\",\n" + "    \"type\": \"record\",\n" + "    \"fields\": [\n" + "        {\n" + "            \"name\":\"v0\",\n" + "            \"type\":\"string\"\n" + "        },\n" + "        {\n" + "            \"name\":\"v1\",\n" + "            \"type\":\"string\",\n" + "            \"default\":\"v1_default\"" + "        }\n" + "    ]\n" + "}";
    Schema[] schemas = { Schema.parse(v0), Schema.parse(v1) };
    // Encode a schema with v0, write out.
    GenericRecord record = new GenericData.Record(schemas[0]);
    record.put("v0", "v0 value");
    replacedertTrue(GenericData.get().validate(schemas[0], record));
    // Write datum out to a stream
    GenericDatumWriter<GenericRecord> gdw = new GenericDatumWriter<GenericRecord>(schemas[0]);
    DataFileWriter<GenericRecord> dfw = new DataFileWriter<GenericRecord>(gdw);
    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    dfw.create(schemas[0], baos);
    dfw.append(record);
    dfw.close();
    ByteArrayInputStream bais = new ByteArrayInputStream(baos.toByteArray());
    GenericDatumReader<GenericRecord> gdr = new GenericDatumReader<GenericRecord>();
    gdr.setExpected(schemas[1]);
    DataFileStream<GenericRecord> dfs = new DataFileStream<GenericRecord>(bais, gdr);
    replacedertTrue(dfs.hasNext());
    GenericRecord next = dfs.next();
    replacedertEquals("v0 value", next.get("v0").toString());
    replacedertEquals("v1_default", next.get("v1").toString());
    // Now the most important check - when we query this record for its schema,
    // we should get back the latest, reader schema:
    replacedertEquals(schemas[1], next.getSchema());
}

18 View Complete Implementation : AvroTeeWriter.java
Copyright Apache License 2.0
Author : linkedin
@Override
public void open(Configuration conf, JsonNode json, BlockSchema schema, Path root, String filename) throws IOException {
    Path teePath = new Path(root, filename + ".avro");
    FileSystem fs = FileSystem.get(conf);
    Schema avroSchema = AvroUtils.convertFromBlockSchema("record", schema);
    GenericDatumWriter<Object> datumWriter = new PigAvroDatumWriter(avroSchema);
    dataFileWriter = new DataFileWriter<Object>(datumWriter);
    // if compression is requested, set the proper compression codec
    if (PhaseContext.getConf().getBoolean("mapred.output.compress", false)) {
        int level = conf.getInt(DEFLATE_LEVEL_KEY, DEFAULT_DEFLATE_LEVEL);
        String codecName = conf.get(OUTPUT_CODEC, DEFLATE_CODEC);
        CodecFactory factory = codecName.equals(DEFLATE_CODEC) ? CodecFactory.deflateCodec(level) : CodecFactory.fromString(codecName);
        dataFileWriter.setCodec(factory);
    }
    dataFileWriter.create(avroSchema, fs.create(teePath));
}

18 View Complete Implementation : AbstractAvroSerializer.java
Copyright Apache License 2.0
Author : alibaba
public void write(Kryo kryo, Output output, GenericContainer record) {
    String fingerPrint = this.getFingerprint(record.getSchema());
    output.writeString(fingerPrint);
    GenericDatumWriter<GenericContainer> writer = new GenericDatumWriter<>(record.getSchema());
    BinaryEncoder encoder = EncoderFactory.get().directBinaryEncoder(output, null);
    try {
        writer.write(record, encoder);
    } catch (IOException e) {
        throw new RuntimeException(e);
    }
}

18 View Complete Implementation : AvroSerde.java
Copyright Apache License 2.0
Author : milinda
@Override
public byte[] toBytes(GenericRecord genericRecord) {
    GenericDatumWriter<GenericRecord> serveWriter = new GenericDatumWriter<GenericRecord>(avroSchema);
    ByteArrayOutputStream out = new ByteArrayOutputStream();
    try {
        serveWriter.write(genericRecord, EncoderFactory.get().binaryEncoder(out, null));
        return out.toByteArray();
    } catch (IOException e) {
        log.error("Cannot serialize GenericRecord.");
    }
    return new byte[0];
}

18 View Complete Implementation : AvroEventDeserializer.java
Copyright Apache License 2.0
Author : javachen
/**
 * A deserializer that parses Avro container files, generating one Flume event
 * per record in the Avro file, and storing binary avro-encoded records in
 * the Flume event body.
 */
public clreplaced AvroEventDeserializer implements EventDeserializer {

    private static final Logger logger = LoggerFactory.getLogger(AvroEventDeserializer.clreplaced);

    private final AvroSchemaType schemaType;

    private final ResettableInputStream ris;

    private Schema schema;

    private byte[] schemaHash;

    private String schemaHashString;

    private DataFileReader<GenericRecord> fileReader;

    private GenericDatumWriter datumWriter;

    private GenericRecord record;

    private ByteArrayOutputStream out;

    private BinaryEncoder encoder;

    @VisibleForTesting
    public static enum AvroSchemaType {

        HASH, LITERAL
    }

    public static final String CONFIG_SCHEMA_TYPE_KEY = "schemaType";

    public static final String AVRO_SCHEMA_HEADER_HASH = "flume.avro.schema.hash";

    public static final String AVRO_SCHEMA_HEADER_LITERAL = "flume.avro.schema.literal";

    private AvroEventDeserializer(Context context, ResettableInputStream ris) {
        this.ris = ris;
        schemaType = AvroSchemaType.valueOf(context.getString(CONFIG_SCHEMA_TYPE_KEY, AvroSchemaType.HASH.toString()).toUpperCase());
        if (schemaType == AvroSchemaType.LITERAL) {
            logger.warn(CONFIG_SCHEMA_TYPE_KEY + " set to " + AvroSchemaType.LITERAL.toString() + ", so storing full Avro " + "schema in the header of each event, which may be inefficient. " + "Consider using the hash of the schema " + "instead of the literal schema.");
        }
    }

    private void initialize() throws IOException, NoSuchAlgorithmException {
        SeekableResettableInputBridge in = new SeekableResettableInputBridge(ris);
        long pos = in.tell();
        in.seek(0L);
        fileReader = new DataFileReader<GenericRecord>(in, new GenericDatumReader<GenericRecord>());
        fileReader.sync(pos);
        schema = fileReader.getSchema();
        datumWriter = new GenericDatumWriter(schema);
        out = new ByteArrayOutputStream();
        encoder = EncoderFactory.get().binaryEncoder(out, encoder);
        schemaHash = SchemaNormalization.parsingFingerprint("CRC-64-AVRO", schema);
        schemaHashString = Hex.encodeHexString(schemaHash);
    }

    @Override
    public Event readEvent() throws IOException {
        if (fileReader.hasNext()) {
            record = fileReader.next(record);
            out.reset();
            datumWriter.write(record, encoder);
            encoder.flush();
            // annotate header with 64-bit schema CRC hash in hex
            Event event = EventBuilder.withBody(out.toByteArray());
            if (schemaType == AvroSchemaType.HASH) {
                event.getHeaders().put(AVRO_SCHEMA_HEADER_HASH, schemaHashString);
            } else {
                event.getHeaders().put(AVRO_SCHEMA_HEADER_LITERAL, schema.toString());
            }
            return event;
        }
        return null;
    }

    @Override
    public List<Event> readEvents(int numEvents) throws IOException {
        List<Event> events = Lists.newArrayList();
        for (int i = 0; i < numEvents && fileReader.hasNext(); i++) {
            Event event = readEvent();
            if (event != null) {
                events.add(event);
            }
        }
        return events;
    }

    @Override
    public void mark() throws IOException {
        long pos = fileReader.previousSync() - DataFileConstants.SYNC_SIZE;
        if (pos < 0)
            pos = 0;
        ((RemoteMarkable) ris).markPosition(pos);
    }

    @Override
    public void reset() throws IOException {
        long pos = ((RemoteMarkable) ris).getMarkPosition();
        fileReader.sync(pos);
    }

    @Override
    public void close() throws IOException {
        ris.close();
    }

    public static clreplaced Builder implements EventDeserializer.Builder {

        @Override
        public EventDeserializer build(Context context, ResettableInputStream in) {
            if (!(in instanceof RemoteMarkable)) {
                throw new IllegalArgumentException("Cannot use this deserializer " + "without a RemoteMarkable input stream");
            }
            AvroEventDeserializer deserializer = new AvroEventDeserializer(context, in);
            try {
                deserializer.initialize();
            } catch (Exception e) {
                throw new FlumeException("Cannot instantiate deserializer", e);
            }
            return deserializer;
        }
    }

    private static clreplaced SeekableResettableInputBridge implements SeekableInput {

        ResettableInputStream ris;

        public SeekableResettableInputBridge(ResettableInputStream ris) {
            this.ris = ris;
        }

        @Override
        public void seek(long p) throws IOException {
            ris.seek(p);
        }

        @Override
        public long tell() throws IOException {
            return ris.tell();
        }

        @Override
        public long length() throws IOException {
            if (ris instanceof LengthMeasurable) {
                return ((LengthMeasurable) ris).length();
            } else {
                // FIXME: Avro doesn't seem to complain about this,
                // but probably not a great idea...
                return Long.MAX_VALUE;
            }
        }

        @Override
        public int read(byte[] b, int off, int len) throws IOException {
            return ris.read(b, off, len);
        }

        @Override
        public void close() throws IOException {
            ris.close();
        }
    }
}

18 View Complete Implementation : AvroPipelineTest.java
Copyright Apache License 2.0
Author : apache
private void populateGenericFile(List<GenericRecord> genericRecords, Schema schema) throws IOException {
    FileOutputStream outputStream = new FileOutputStream(this.inputFile);
    GenericDatumWriter<GenericRecord> genericDatumWriter = new GenericDatumWriter<>(schema);
    try (DataFileWriter<GenericRecord> dataFileWriter = new DataFileWriter<>(genericDatumWriter)) {
        dataFileWriter.create(schema, outputStream);
        for (GenericRecord record : genericRecords) {
            dataFileWriter.append(record);
        }
    }
    outputStream.close();
}

18 View Complete Implementation : MockSchemaRegistryClient.java
Copyright Apache License 2.0
Author : snowflakedb
private static byte[] jsonToAvro(String json, Schema schema) throws IOException {
    InputStream input = new ByteArrayInputStream(json.getBytes(StandardCharsets.UTF_8));
    GenericDatumWriter<GenericRecord> writer;
    Encoder encoder;
    ByteArrayOutputStream output;
    try {
        DatumReader<GenericRecord> reader = new GenericDatumReader<>(schema);
        output = new ByteArrayOutputStream();
        writer = new GenericDatumWriter<>(schema);
        DataInputStream din = new DataInputStream(input);
        Decoder decoder = DecoderFactory.get().jsonDecoder(schema, din);
        encoder = EncoderFactory.get().binaryEncoder(output, null);
        GenericRecord datum;
        while (true) {
            try {
                datum = reader.read(null, decoder);
            } catch (EOFException eof) {
                break;
            }
            writer.write(datum, encoder);
        }
        encoder.flush();
        return output.toByteArray();
    } finally {
        try {
            input.close();
        } catch (IOException e) {
            e.printStackTrace();
        }
    }
}

18 View Complete Implementation : DBInputAvroMapper.java
Copyright Apache License 2.0
Author : sonalgoyal
public clreplaced DBInputAvroMapper extends MapReduceBase implements Mapper<LongWritable, GenericDBWritable, AvroValue<Pair>, NullWritable> {

    final static Logger logger = Logger.getLogger(co.nubetech.hiho.mapreduce.DBInputAvroMapper.clreplaced);

    GenericDatumWriter<Pair> writer;

    private NullWritable nullWritable = NullWritable.get();

    Pair pair = null;

    Schema pairSchema = null;

    Schema keySchema = null;

    Schema valueSchema = null;

    @Override
    public void map(LongWritable key, GenericDBWritable val, OutputCollector<AvroValue<Pair>, NullWritable> output, Reporter reporter) throws IOException {
        logger.debug("Key, val are " + key + " val " + val.getColumns());
        if (pairSchema == null) {
            logger.debug("Creating schema for MR");
            logger.debug("MR columns are " + val.getColumns());
            for (ColumnInfo column : val.getColumns()) {
                logger.debug("Column is " + column.getIndex() + " " + column.getName());
            }
            pairSchema = DBMapper.getPairSchema(val.getColumns());
            keySchema = Pair.getKeySchema(pairSchema);
            valueSchema = Pair.getValueSchema(pairSchema);
            pair = new Pair<GenericRecord, GenericRecord>(pairSchema);
        }
        // writer = new GenericDatumWriter<Pair>(pairSchema);
        GenericRecord keyRecord = this.getKeyRecord(keySchema, key);
        logger.debug("Key record is " + keyRecord);
        // now generate the avro record
        GenericRecord valueRecord = this.getValueRecord(valueSchema, val);
        logger.debug("Value Record is " + valueRecord);
        /*
		 * ByteArrayOutputStream stream = new ByteArrayOutputStream();
		 * writer.write(record, new BinaryEncoder(stream)); stream.close();
		 */
        pair.key(keyRecord);
        pair.value(valueRecord);
        output.collect(new AvroValue<Pair>(pair), nullWritable);
    }

    public GenericRecord getKeyRecord(Schema keySchema, LongWritable key) {
        GenericRecord keyRecord = new GenericData.Record(keySchema);
        keyRecord.put(0, key.get());
        return keyRecord;
    }

    public GenericRecord getValueRecord(Schema valueSchema, GenericDBWritable val) {
        GenericRecord valueRecord = new GenericData.Record(valueSchema);
        List<Schema.Field> fieldSchemas = valueSchema.getFields();
        for (int i = 0; i < val.getValues().size(); ++i) {
            Schema.Type type = fieldSchemas.get(i).schema().getType();
            if (type.equals(Schema.Type.STRING)) {
                Utf8 utf8 = new Utf8((String) val.getValues().get(i).toString());
                valueRecord.put(i, utf8);
            } else {
                valueRecord.put(i, val.getValues().get(i));
            }
        }
        return valueRecord;
    }
}

18 View Complete Implementation : AvroFileSourceTargetIT.java
Copyright Apache License 2.0
Author : apache
private void populateGenericFile(File outFile, List<GenericRecord> genericRecords, Schema schema) throws IOException {
    FileOutputStream outputStream = new FileOutputStream(outFile);
    GenericDatumWriter<GenericRecord> genericDatumWriter = new GenericDatumWriter<GenericRecord>(schema);
    DataFileWriter<GenericRecord> dataFileWriter = new DataFileWriter<GenericRecord>(genericDatumWriter);
    dataFileWriter.create(schema, outputStream);
    for (GenericRecord record : genericRecords) {
        dataFileWriter.append(record);
    }
    dataFileWriter.close();
    outputStream.close();
}

18 View Complete Implementation : NamespaceValidationTest.java
Copyright BSD 2-Clause "Simplified" License
Author : linkedin
@Test
public void testModernAvroValidatesNamespaces() throws Exception {
    AvroVersion runtimeVersion = AvroCompatibilityHelper.getRuntimeAvroVersion();
    if (!runtimeVersion.laterThan(AvroVersion.AVRO_1_4)) {
        throw new SkipException("only supported under modern avro. runtime version detected as " + runtimeVersion);
    }
    String withAvsc = TestUtil.load("HasNamespace.avsc");
    Schema with = Schema.parse(withAvsc);
    String withoutAvsc = TestUtil.load("HasNoNamespace.avsc");
    Schema without = Schema.parse(withoutAvsc);
    GenericData.Record record = new GenericData.Record(without);
    record.put("f", AvroCompatibilityHelper.newEnumSymbol(without.getField("f").schema(), "B"));
    ByteArrayOutputStream os = new ByteArrayOutputStream();
    GenericDatumWriter writer = new GenericDatumWriter(without);
    BinaryEncoder encoder = AvroCompatibilityHelper.newBinaryEncoder(os);
    // noinspection unchecked
    writer.write(record, encoder);
    encoder.flush();
    byte[] bytes = os.toByteArray();
    GenericDatumReader<GenericData.Record> reader = new GenericDatumReader<>(without, with);
    BinaryDecoder decoder = DecoderFactory.defaultFactory().createBinaryDecoder(bytes, null);
    try {
        GenericData.Record read = reader.read(null, decoder);
        replacedert.fail("deserialization was expected to fail");
    } catch (Exception expected) {
        replacedert.replacedertTrue(expected.getMessage().contains("Found EnumType, expecting com.acme.EnumType"));
    }
}

18 View Complete Implementation : RelayEventGenerator.java
Copyright Apache License 2.0
Author : linkedin
int populateEvents(String source, short id, GenericRecord record, DbusEventKey key, byte[] schemaId, DbusEventsStatisticsCollector statsCollector, DbusEventBufferAppendable buffer) {
    if (record != null && key != null) {
        try {
            // Serialize the row
            ByteArrayOutputStream bos = new ByteArrayOutputStream();
            Encoder encoder = new BinaryEncoder(bos);
            GenericDatumWriter<GenericRecord> writer = new GenericDatumWriter<GenericRecord>(record.getSchema());
            writer.write(record, encoder);
            byte[] serializedValue = bos.toByteArray();
            short pParreplacedionId = RngUtils.randomPositiveShort();
            short lParreplacedionId = RngUtils.randomPositiveShort();
            long timeStamp = System.currentTimeMillis() * 1000000;
            buffer.appendEvent(key, pParreplacedionId, lParreplacedionId, timeStamp, id, schemaId, serializedValue, false, statsCollector);
            return 1;
        } catch (IOException io) {
            LOG.error("Cannot create byte stream payload: " + source);
        }
    }
    return 0;
}

18 View Complete Implementation : AvroPipelineIT.java
Copyright Apache License 2.0
Author : apache
private void populateGenericFile(List<GenericRecord> genericRecords, Schema schema) throws IOException {
    FileOutputStream outputStream = new FileOutputStream(this.avroFile);
    GenericDatumWriter<GenericRecord> genericDatumWriter = new GenericDatumWriter<GenericRecord>(schema);
    DataFileWriter<GenericRecord> dataFileWriter = new DataFileWriter<GenericRecord>(genericDatumWriter);
    dataFileWriter.create(schema, outputStream);
    for (GenericRecord record : genericRecords) {
        dataFileWriter.append(record);
    }
    dataFileWriter.close();
    outputStream.close();
}

18 View Complete Implementation : AvroFileReaderFactoryTest.java
Copyright Apache License 2.0
Author : apache
private void populateGenericFile(List<GenericRecord> genericRecords, Schema outputSchema) throws IOException {
    FileOutputStream outputStream = new FileOutputStream(this.avroFile);
    GenericDatumWriter<GenericRecord> genericDatumWriter = new GenericDatumWriter<GenericRecord>(outputSchema);
    DataFileWriter<GenericRecord> dataFileWriter = new DataFileWriter<GenericRecord>(genericDatumWriter);
    dataFileWriter.create(outputSchema, outputStream);
    for (GenericRecord record : genericRecords) {
        dataFileWriter.append(record);
    }
    dataFileWriter.close();
    outputStream.close();
}

18 View Complete Implementation : JsonStreamWriter.java
Copyright Apache License 2.0
Author : openaire
/**
 * Write JSON stream corresponding a series of Avro datums. Single line in the
 * output corresponds to a single input Avro datum.
 *
 * @author Mateusz Kobos
 */
public clreplaced JsonStreamWriter<T> implements Closeable {

    private final OutputStream out;

    private final HackedJsonEncoder encoder;

    private final GenericDatumWriter<T> writer;

    public JsonStreamWriter(Schema schema, OutputStream out) throws IOException {
        this.out = out;
        this.encoder = new HackedJsonEncoder(schema, out);
        this.writer = new GenericDatumWriter<T>(schema);
    }

    /**
     * Write a datum to given output stream
     *
     * @throws IOException
     */
    public void write(T datum) throws IOException {
        writer.write(datum, encoder);
    }

    /**
     * Close the stream
     */
    @Override
    public void close() throws IOException {
        encoder.flush();
        out.flush();
        out.close();
    }
}

18 View Complete Implementation : AvroSerializationSchema.java
Copyright Apache License 2.0
Author : apache
/**
 * Serialization schema that serializes to Avro binary format.
 *
 * @param <T> the type  to be serialized
 */
public clreplaced AvroSerializationSchema<T> implements SerializationSchema<T> {

    /**
     * Creates {@link AvroSerializationSchema} that serializes {@link SpecificRecord} using provided schema.
     *
     * @param tClreplaced the type to be serialized
     * @return serialized record in form of byte array
     */
    public static <T extends SpecificRecord> AvroSerializationSchema<T> forSpecific(Clreplaced<T> tClreplaced) {
        return new AvroSerializationSchema<>(tClreplaced, null);
    }

    /**
     * Creates {@link AvroSerializationSchema} that serializes {@link GenericRecord} using provided schema.
     *
     * @param schema the schema that will be used for serialization
     * @return serialized record in form of byte array
     */
    public static AvroSerializationSchema<GenericRecord> forGeneric(Schema schema) {
        return new AvroSerializationSchema<>(GenericRecord.clreplaced, schema);
    }

    private static final long serialVersionUID = -8766681879020862312L;

    /**
     * Clreplaced to serialize to.
     */
    private Clreplaced<T> recordClazz;

    private String schemaString;

    private transient Schema schema;

    /**
     * Writer that writes the serialized record to {@link ByteArrayOutputStream}.
     */
    private transient GenericDatumWriter<T> datumWriter;

    /**
     * Output stream to write message to.
     */
    private transient ByteArrayOutputStream arrayOutputStream;

    /**
     * Avro encoder that encodes binary data.
     */
    private transient BinaryEncoder encoder;

    /**
     * Creates an Avro deserialization schema.
     *
     * @param recordClazz clreplaced to serialize. Should be one of:
     *                    {@link org.apache.avro.specific.SpecificRecord},
     *                    {@link org.apache.avro.generic.GenericRecord}.
     * @param schema      writer Avro schema. Should be provided if recordClazz is
     *                    {@link GenericRecord}
     */
    protected AvroSerializationSchema(Clreplaced<T> recordClazz, @Nullable Schema schema) {
        Preconditions.checkNotNull(recordClazz, "Avro record clreplaced must not be null.");
        this.recordClazz = recordClazz;
        this.schema = schema;
        if (schema != null) {
            this.schemaString = schema.toString();
        } else {
            this.schemaString = null;
        }
    }

    public Schema getSchema() {
        return schema;
    }

    protected BinaryEncoder getEncoder() {
        return encoder;
    }

    protected GenericDatumWriter<T> getDatumWriter() {
        return datumWriter;
    }

    protected ByteArrayOutputStream getOutputStream() {
        return arrayOutputStream;
    }

    @Override
    public byte[] serialize(T object) {
        checkAvroInitialized();
        if (object == null) {
            return null;
        } else {
            try {
                datumWriter.write(object, encoder);
                encoder.flush();
                byte[] bytes = arrayOutputStream.toByteArray();
                arrayOutputStream.reset();
                return bytes;
            } catch (IOException e) {
                throw new WrappingRuntimeException("Failed to serialize schema registry.", e);
            }
        }
    }

    protected void checkAvroInitialized() {
        if (datumWriter != null) {
            return;
        }
        ClreplacedLoader cl = Thread.currentThread().getContextClreplacedLoader();
        if (SpecificRecord.clreplaced.isreplacedignableFrom(recordClazz)) {
            Schema schema = SpecificData.get().getSchema(recordClazz);
            this.datumWriter = new SpecificDatumWriter<>(schema);
            this.schema = schema;
        } else {
            this.schema = new Schema.Parser().parse(this.schemaString);
            GenericData genericData = new GenericData(cl);
            this.datumWriter = new GenericDatumWriter<>(schema, genericData);
        }
        this.arrayOutputStream = new ByteArrayOutputStream();
        this.encoder = EncoderFactory.get().directBinaryEncoder(arrayOutputStream, null);
    }
}

18 View Complete Implementation : AvroEventDeserializer.java
Copyright Apache License 2.0
Author : apache
/**
 * A deserializer that parses Avro container files, generating one Flume event
 * per record in the Avro file, and storing binary avro-encoded records in
 * the Flume event body.
 */
public clreplaced AvroEventDeserializer implements EventDeserializer {

    private static final Logger logger = LoggerFactory.getLogger(AvroEventDeserializer.clreplaced);

    private final AvroSchemaType schemaType;

    private final ResettableInputStream ris;

    private Schema schema;

    private byte[] schemaHash;

    private String schemaHashString;

    private DataFileReader<GenericRecord> fileReader;

    private GenericDatumWriter datumWriter;

    private GenericRecord record;

    private ByteArrayOutputStream out;

    private BinaryEncoder encoder;

    @VisibleForTesting
    public static enum AvroSchemaType {

        HASH, LITERAL
    }

    public static final String CONFIG_SCHEMA_TYPE_KEY = "schemaType";

    public static final String AVRO_SCHEMA_HEADER_HASH = "flume.avro.schema.hash";

    public static final String AVRO_SCHEMA_HEADER_LITERAL = "flume.avro.schema.literal";

    private AvroEventDeserializer(Context context, ResettableInputStream ris) {
        this.ris = ris;
        schemaType = AvroSchemaType.valueOf(context.getString(CONFIG_SCHEMA_TYPE_KEY, AvroSchemaType.HASH.toString()).toUpperCase(Locale.ENGLISH));
        if (schemaType == AvroSchemaType.LITERAL) {
            logger.warn(CONFIG_SCHEMA_TYPE_KEY + " set to " + AvroSchemaType.LITERAL.toString() + ", so storing full Avro " + "schema in the header of each event, which may be inefficient. " + "Consider using the hash of the schema " + "instead of the literal schema.");
        }
    }

    private void initialize() throws IOException, NoSuchAlgorithmException {
        SeekableResettableInputBridge in = new SeekableResettableInputBridge(ris);
        long pos = in.tell();
        in.seek(0L);
        fileReader = new DataFileReader<GenericRecord>(in, new GenericDatumReader<GenericRecord>());
        fileReader.sync(pos);
        schema = fileReader.getSchema();
        datumWriter = new GenericDatumWriter(schema);
        out = new ByteArrayOutputStream();
        encoder = EncoderFactory.get().binaryEncoder(out, encoder);
        schemaHash = SchemaNormalization.parsingFingerprint("CRC-64-AVRO", schema);
        schemaHashString = Hex.encodeHexString(schemaHash);
    }

    @Override
    public Event readEvent() throws IOException {
        if (fileReader.hasNext()) {
            record = fileReader.next(record);
            out.reset();
            datumWriter.write(record, encoder);
            encoder.flush();
            // annotate header with 64-bit schema CRC hash in hex
            Event event = EventBuilder.withBody(out.toByteArray());
            if (schemaType == AvroSchemaType.HASH) {
                event.getHeaders().put(AVRO_SCHEMA_HEADER_HASH, schemaHashString);
            } else {
                event.getHeaders().put(AVRO_SCHEMA_HEADER_LITERAL, schema.toString());
            }
            return event;
        }
        return null;
    }

    @Override
    public List<Event> readEvents(int numEvents) throws IOException {
        List<Event> events = Lists.newArrayList();
        for (int i = 0; i < numEvents && fileReader.hasNext(); i++) {
            Event event = readEvent();
            if (event != null) {
                events.add(event);
            }
        }
        return events;
    }

    @Override
    public void mark() throws IOException {
        long pos = fileReader.previousSync() - DataFileConstants.SYNC_SIZE;
        if (pos < 0)
            pos = 0;
        ((RemoteMarkable) ris).markPosition(pos);
    }

    @Override
    public void reset() throws IOException {
        long pos = ((RemoteMarkable) ris).getMarkPosition();
        fileReader.sync(pos);
    }

    @Override
    public void close() throws IOException {
        ris.close();
    }

    public static clreplaced Builder implements EventDeserializer.Builder {

        @Override
        public EventDeserializer build(Context context, ResettableInputStream in) {
            if (!(in instanceof RemoteMarkable)) {
                throw new IllegalArgumentException("Cannot use this deserializer " + "without a RemoteMarkable input stream");
            }
            AvroEventDeserializer deserializer = new AvroEventDeserializer(context, in);
            try {
                deserializer.initialize();
            } catch (Exception e) {
                throw new FlumeException("Cannot instantiate deserializer", e);
            }
            return deserializer;
        }
    }

    private static clreplaced SeekableResettableInputBridge implements SeekableInput {

        ResettableInputStream ris;

        public SeekableResettableInputBridge(ResettableInputStream ris) {
            this.ris = ris;
        }

        @Override
        public void seek(long p) throws IOException {
            ris.seek(p);
        }

        @Override
        public long tell() throws IOException {
            return ris.tell();
        }

        @Override
        public long length() throws IOException {
            if (ris instanceof LengthMeasurable) {
                return ((LengthMeasurable) ris).length();
            } else {
                // FIXME: Avro doesn't seem to complain about this,
                // but probably not a great idea...
                return Long.MAX_VALUE;
            }
        }

        @Override
        public int read(byte[] b, int off, int len) throws IOException {
            return ris.read(b, off, len);
        }

        @Override
        public void close() throws IOException {
            ris.close();
        }
    }
}

18 View Complete Implementation : ClusterIntegrationTestUtils.java
Copyright Apache License 2.0
Author : apache
/**
 * Push the records from the given Avro files into a Kafka stream.
 *
 * @param avroFiles List of Avro files
 * @param kafkaBroker Kafka broker config
 * @param kafkaTopic Kafka topic
 * @param maxNumKafkaMessagesPerBatch Maximum number of Kafka messages per batch
 * @param header Optional Kafka message header
 * @param parreplacedionColumn Optional parreplacedion column
 * @throws Exception
 */
public static void pushAvroIntoKafka(@Nonnull List<File> avroFiles, @Nonnull String kafkaBroker, @Nonnull String kafkaTopic, int maxNumKafkaMessagesPerBatch, @Nullable byte[] header, @Nullable String parreplacedionColumn) throws Exception {
    Properties properties = new Properties();
    properties.put("metadata.broker.list", kafkaBroker);
    properties.put("serializer.clreplaced", "kafka.serializer.DefaultEncoder");
    properties.put("request.required.acks", "1");
    properties.put("parreplacedioner.clreplaced", "kafka.producer.ByteArrayParreplacedioner");
    StreamDataProducer producer = StreamDataProvider.getStreamDataProducer(KafkaStarterUtils.KAFKA_PRODUCER_CLreplaced_NAME, properties);
    try (ByteArrayOutputStream outputStream = new ByteArrayOutputStream(65536)) {
        for (File avroFile : avroFiles) {
            try (DataFileStream<GenericRecord> reader = AvroUtils.getAvroReader(avroFile)) {
                BinaryEncoder binaryEncoder = new EncoderFactory().directBinaryEncoder(outputStream, null);
                GenericDatumWriter<GenericRecord> datumWriter = new GenericDatumWriter<>(reader.getSchema());
                for (GenericRecord genericRecord : reader) {
                    outputStream.reset();
                    if (header != null && 0 < header.length) {
                        outputStream.write(header);
                    }
                    datumWriter.write(genericRecord, binaryEncoder);
                    binaryEncoder.flush();
                    byte[] keyBytes = (parreplacedionColumn == null) ? Longs.toByteArray(System.currentTimeMillis()) : (genericRecord.get(parreplacedionColumn)).toString().getBytes();
                    byte[] bytes = outputStream.toByteArray();
                    producer.produce(kafkaTopic, keyBytes, bytes);
                }
            }
        }
    }
}

18 View Complete Implementation : ClusterIntegrationTestUtils.java
Copyright Apache License 2.0
Author : apache
/**
 * Push random generated
 *
 * @param avroFile Sample Avro file used to extract the Avro schema
 * @param kafkaBroker Kafka broker config
 * @param kafkaTopic Kafka topic
 * @param numKafkaMessagesToPush Number of Kafka messages to push
 * @param maxNumKafkaMessagesPerBatch Maximum number of Kafka messages per batch
 * @param header Optional Kafka message header
 * @param parreplacedionColumn Optional parreplacedion column
 * @throws Exception
 */
@SuppressWarnings("unused")
public static void pushRandomAvroIntoKafka(@Nonnull File avroFile, @Nonnull String kafkaBroker, @Nonnull String kafkaTopic, int numKafkaMessagesToPush, int maxNumKafkaMessagesPerBatch, @Nullable byte[] header, @Nullable String parreplacedionColumn) throws Exception {
    Properties properties = new Properties();
    properties.put("metadata.broker.list", kafkaBroker);
    properties.put("serializer.clreplaced", "kafka.serializer.DefaultEncoder");
    properties.put("request.required.acks", "1");
    properties.put("parreplacedioner.clreplaced", "kafka.producer.ByteArrayParreplacedioner");
    StreamDataProducer producer = StreamDataProvider.getStreamDataProducer(KafkaStarterUtils.KAFKA_PRODUCER_CLreplaced_NAME, properties);
    try (ByteArrayOutputStream outputStream = new ByteArrayOutputStream(65536)) {
        try (DataFileStream<GenericRecord> reader = AvroUtils.getAvroReader(avroFile)) {
            BinaryEncoder binaryEncoder = new EncoderFactory().directBinaryEncoder(outputStream, null);
            Schema avroSchema = reader.getSchema();
            GenericDatumWriter<GenericRecord> datumWriter = new GenericDatumWriter<>(avroSchema);
            GenericRecord genericRecord = new GenericData.Record(avroSchema);
            while (numKafkaMessagesToPush > 0) {
                generateRandomRecord(genericRecord, avroSchema);
                outputStream.reset();
                if (header != null && 0 < header.length) {
                    outputStream.write(header);
                }
                datumWriter.write(genericRecord, binaryEncoder);
                binaryEncoder.flush();
                byte[] keyBytes = (parreplacedionColumn == null) ? Longs.toByteArray(System.currentTimeMillis()) : (genericRecord.get(parreplacedionColumn)).toString().getBytes();
                byte[] bytes = outputStream.toByteArray();
                producer.produce(kafkaTopic, keyBytes, bytes);
                numKafkaMessagesToPush--;
            }
        }
    }
}

18 View Complete Implementation : GenericAvroWriter.java
Copyright Apache License 2.0
Author : apache
public clreplaced GenericAvroWriter implements SchemaWriter<GenericRecord> {

    private final GenericDatumWriter<org.apache.avro.generic.GenericRecord> writer;

    private BinaryEncoder encoder;

    private final ByteArrayOutputStream byteArrayOutputStream;

    public GenericAvroWriter(Schema schema) {
        this.writer = new GenericDatumWriter<>(schema);
        this.byteArrayOutputStream = new ByteArrayOutputStream();
        this.encoder = EncoderFactory.get().binaryEncoder(this.byteArrayOutputStream, encoder);
    }

    @Override
    public synchronized byte[] write(GenericRecord message) {
        try {
            writer.write(((GenericAvroRecord) message).getAvroRecord(), this.encoder);
            this.encoder.flush();
            return this.byteArrayOutputStream.toByteArray();
        } catch (Exception e) {
            throw new SchemaSerializationException(e);
        } finally {
            this.byteArrayOutputStream.reset();
        }
    }
}

18 View Complete Implementation : AvroGenericRecordWritable.java
Copyright Apache License 2.0
Author : bunnyg
@Override
public void write(DataOutput out) throws IOException {
    // Write schema since we need it to pull the data out. (see point #1 above)
    String schemaString = record.getSchema().toString(false);
    out.writeUTF(schemaString);
    // Write record to byte buffer
    GenericDatumWriter<GenericRecord> gdw = new GenericDatumWriter<GenericRecord>();
    BinaryEncoder be = EncoderFactory.get().directBinaryEncoder((DataOutputStream) out, null);
    gdw.setSchema(record.getSchema());
    gdw.write(record, be);
}

17 View Complete Implementation : MapOutputValue.java
Copyright Apache License 2.0
Author : apache
public clreplaced MapOutputValue {

    private static BinaryDecoder binaryDecoder;

    private String schemaName;

    private GenericRecord record;

    private GenericDatumWriter<GenericRecord> WRITER;

    private EncoderFactory factory = EncoderFactory.get();

    private BinaryEncoder binaryEncoder;

    public MapOutputValue(String schemaName, GenericRecord record) {
        this.schemaName = schemaName;
        this.record = record;
    }

    public String getSchemaName() {
        return schemaName;
    }

    public GenericRecord getRecord() {
        return record;
    }

    public byte[] toBytes() throws IOException {
        ByteArrayOutputStream dataStream = new ByteArrayOutputStream();
        Schema schema = record.getSchema();
        if (WRITER == null) {
            WRITER = new GenericDatumWriter<GenericRecord>(schema);
        }
        binaryEncoder = factory.directBinaryEncoder(dataStream, binaryEncoder);
        WRITER.write(record, binaryEncoder);
        // serialize to bytes, we also need to know the schema name when we
        // process this record on the reducer since reducer gets the record from
        // multiple mappers. So we first write the schema/source name and then
        // write the serialized bytes
        ByteArrayOutputStream out = new ByteArrayOutputStream();
        DataOutputStream dos = new DataOutputStream(out);
        dos.writeInt(schema.getName().getBytes().length);
        dos.write(schema.getName().getBytes());
        byte[] dataBytes = dataStream.toByteArray();
        dos.writeInt(dataBytes.length);
        dos.write(dataBytes);
        return out.toByteArray();
    }

    public static MapOutputValue fromBytes(byte[] bytes, Map<String, Schema> schemaMap) throws IOException {
        DataInputStream dataInputStream = new DataInputStream(new ByteArrayInputStream(bytes));
        int length = dataInputStream.readInt();
        byte[] sourceNameBytes = new byte[length];
        dataInputStream.read(sourceNameBytes);
        String schemaName = new String(sourceNameBytes);
        int recordDataLength = dataInputStream.readInt();
        byte[] recordBytes = new byte[recordDataLength];
        dataInputStream.read(recordBytes);
        Schema schema = schemaMap.get(schemaName);
        GenericRecord record = new GenericData.Record(schema);
        binaryDecoder = DecoderFactory.get().binaryDecoder(recordBytes, binaryDecoder);
        GenericDatumReader<GenericRecord> gdr = new GenericDatumReader<GenericRecord>(schema);
        gdr.read(record, binaryDecoder);
        return new MapOutputValue(schemaName, record);
    }
}

17 View Complete Implementation : OutputElement.java
Copyright Apache License 2.0
Author : flumebase
/**
 * FlowElement that sits at the end of a query flow.
 * Outputs records to foreign sources:
 * <ul>
 *   <li>Prints events to the consoles of each subscriber.
 *   <li>Emits Avro records on a named Flume stream.
 * </ul>
 */
public clreplaced OutputElement extends FlowElementImpl {

    private static final Logger LOG = LoggerFactory.getLogger(OutputElement.clreplaced.getName());

    /**
     * Max queue length we deliver to Flume before blocking.
     */
    private static final int MAX_QUEUE_LEN = 512;

    // Used for stringifying ByteBuffers.
    private static bin2str BIN2STR_FN;

    static {
        BIN2STR_FN = new bin2str();
    }

    /**
     * Input fields being delivered to this node.
     */
    private List<TypedField> mInputFields;

    /**
     * Interface with Flume
     */
    private EmbeddedFlumeConfig mFlumeConfig;

    /**
     * Name of Flume logical node that broadcasts the results of this query.
     */
    private String mFlumeNodeName;

    /**
     * Set to true if this OutputNode manages the symbol replacedociated with
     * mFlumeNodeName.
     */
    private boolean mOwnsSymbol;

    /**
     * Queue of events that are delivered to Flume by this OutputElement.
     */
    private BlockingQueue<Event> mOutputQueue;

    /**
     * De-dup'd version of mInputFields, for version emitted to Flume.
     */
    private List<TypedField> mFlumeInputFields;

    /**
     * Output fields emitted to Flume.
     */
    private List<TypedField> mOutputFields;

    /**
     * Output schema emitted to Flume.
     */
    private Schema mOutputSchema;

    /**
     * Symbol table where stream definitions are introduced.
     */
    private SymbolTable mRootSymbolTable;

    // Objects used for Avro serialization.
    private Encoder mEncoder;

    private GenericDatumWriter<GenericRecord> mDatumWriter;

    private ByteArrayOutputStream mOutputBytes;

    public OutputElement(FlowElementContext context, Schema inputSchema, List<TypedField> fields, EmbeddedFlumeConfig flumeConfig, String flumeNodeName, Schema outputSchema, List<TypedField> outputFields, SymbolTable rootSymbolTable) {
        super(context);
        mInputFields = fields;
        mFlumeConfig = flumeConfig;
        mFlumeNodeName = flumeNodeName;
        mOutputQueue = null;
        mOutputSchema = outputSchema;
        mOutputFields = outputFields;
        mRootSymbolTable = rootSymbolTable;
        mOwnsSymbol = false;
        mFlumeInputFields = SQLStatement.distinctFields(mInputFields);
        replacedert mFlumeInputFields.size() == mOutputFields.size();
        mDatumWriter = new GenericDatumWriter<GenericRecord>(outputSchema);
        mOutputBytes = new ByteArrayOutputStream();
        mEncoder = new BinaryEncoder(mOutputBytes);
    }

    private StringBuilder formatHeader() {
        StringBuilder sb = new StringBuilder();
        sb.append("timestamp");
        for (TypedField field : mInputFields) {
            sb.append("\t");
            sb.append(field.getDisplayName());
        }
        return sb;
    }

    @Override
    public void onConnect(UserSession session) {
        // When a user first connects, print the header for our output columns.
        session.sendInfo(formatHeader().toString());
    }

    @Override
    public void open() throws IOException {
        if (null != mFlumeNodeName) {
            if (!mFlumeConfig.isRunning()) {
                mFlumeConfig.start();
            }
            // Open a Flume logical node to host the results of this query.
            // TODO(aaron): What happens if this flume node already exists? This should error...
            mOutputQueue = new ArrayBlockingQueue<Event>(MAX_QUEUE_LEN);
            SourceContext srcContext = new SourceContext(mFlumeNodeName, mOutputQueue);
            SourceContextBindings.get().bindContext(mFlumeNodeName, srcContext);
            try {
                mFlumeConfig.spawnLogicalNode(mFlumeNodeName, "rtsqlsource(\"" + mFlumeNodeName + "\")", "rtsqlmultisink(\"" + mFlumeNodeName + "\")");
                mFlumeConfig.addLocalMultiSink(mFlumeNodeName);
                if (mRootSymbolTable.resolve(mFlumeNodeName) != null) {
                    // TODO(aaron): This should make it back to the UserSession who submitted
                    // the job, if this is the first call to open(), or to the UserSession who
                    // bound the query to the current output name.
                    // Also, should we fail the job? etc etc... check preconditions?
                    LOG.error("Cannot create stream for flow; object already exists at top level: " + mFlumeNodeName);
                    mOwnsSymbol = false;
                    ((LocalContext) getContext()).getFlowData().setStreamName(null);
                } else {
                    FormatSpec formatSpec = new FormatSpec(FormatSpec.FORMAT_AVRO);
                    formatSpec.setParam(AvroEventParser.SCHEMA_PARAM, mOutputSchema.toString());
                    List<Type> outputTypes = new ArrayList<Type>();
                    for (TypedField field : mFlumeInputFields) {
                        outputTypes.add(field.getType());
                    }
                    Type streamType = new StreamType(outputTypes);
                    StreamSymbol streamSym = new StreamSymbol(mFlumeNodeName, StreamSourceType.Node, streamType, mFlumeNodeName, true, mOutputFields, formatSpec);
                    if (!streamSym.getEventParser().validate(streamSym)) {
                        throw new IOException("Could not create valid stream for schema");
                    }
                    mRootSymbolTable.addSymbol(streamSym);
                    mOwnsSymbol = true;
                    ((LocalContext) getContext()).getFlowData().setStreamName(mFlumeNodeName);
                    LOG.info("CREATE STREAM (" + mFlumeNodeName + ")");
                }
            } catch (TException te) {
                throw new IOException(te);
            }
        }
    }

    /**
     * Sets the logical node which should broadcast this flow's output.  If
     * logicalNodeName is null, disables Flume-based output of this flow.  If
     * this FlowElement was previously broadcasting to a particular logical
     * node, this will close any prior logical node before opening a new one.
     */
    public void setFlumeTarget(String logicalNodeName) throws IOException {
        if (mFlumeNodeName != null) {
            if (mFlumeNodeName.equals(logicalNodeName)) {
                // Nothing to do.
                return;
            } else {
                stopFlumeNode();
            }
        }
        mFlumeNodeName = logicalNodeName;
        // Create a new output record schema, with records taking the new stream name.
        if (null != mFlumeNodeName) {
            mOutputSchema = SQLStatement.createFieldSchema(mOutputFields, mFlumeNodeName);
            mDatumWriter = new GenericDatumWriter<GenericRecord>(mOutputSchema);
            open();
        }
    }

    /**
     * Stops the current Flume node broadcasting our output.
     */
    private void stopFlumeNode() throws IOException {
        if (mFlumeNodeName != null) {
            if (mOwnsSymbol) {
                // TODO: Broadcast this DROP STREAM event back to the user who ordered the config change.
                mRootSymbolTable.remove(mFlumeNodeName);
                ((LocalContext) getContext()).getFlowData().setStreamName(null);
                mOwnsSymbol = false;
            }
            try {
                mFlumeConfig.dropLocalMultiSink(mFlumeNodeName);
                mFlumeConfig.decommissionLogicalNode(mFlumeNodeName);
            } catch (TException te) {
                throw new IOException(te);
            } finally {
                SourceContextBindings.get().dropContext(mFlumeNodeName);
                mOutputQueue = null;
            }
        }
    }

    @Override
    public void close() throws IOException, InterruptedException {
        try {
            stopFlumeNode();
        } catch (IOException ioe) {
            LOG.error("TException decommissioning logical node " + mFlumeNodeName + ": " + ioe);
        }
        super.close();
    }

    /**
     * Format the internal event as an Avro record of the output schema,
     * and emit it to the Flume node via our queue.
     */
    private void emitToFlume(EventWrapper e) throws IOException, InterruptedException {
        GenericData.Record record = new GenericData.Record(mOutputSchema);
        for (int i = 0; i < mFlumeInputFields.size(); i++) {
            TypedField inField = mFlumeInputFields.get(i);
            TypedField outField = mOutputFields.get(i);
            Object val = e.getField(inField);
            record.put(outField.getAvroName(), val);
        }
        mOutputBytes.reset();
        try {
            mDatumWriter.write(record, mEncoder);
        } catch (NullPointerException npe) {
            // Schema error -- null output value in non-null field. Drop the record.
            LOG.debug("Dropping output record with NULL value in non-null field: " + npe);
            return;
        }
        Event out = new EventImpl(mOutputBytes.toByteArray());
        mOutputQueue.put(out);
    }

    @Override
    public void takeEvent(EventWrapper e) throws IOException, InterruptedException {
        LocalContext context = (LocalContext) getContext();
        List<UserSession> subscribers = new LinkedList<UserSession>(context.getFlowData().getSubscribers());
        if (mOutputQueue != null) {
            emitToFlume(e);
        }
        if (subscribers.size() == 0) {
            // Nobody is listening on a console; don't waste time formatting it as a string.
            return;
        }
        StringBuilder sb = new StringBuilder();
        long ts = e.getEvent().getTimestamp();
        sb.append(ts);
        // Extract the Avro record from the event.
        for (TypedField field : mInputFields) {
            sb.append('\t');
            Object fieldVal = e.getField(field);
            if (null != fieldVal) {
                LOG.debug("Printing val of clreplaced " + fieldVal.getClreplaced().getName());
            }
            // If we get any GenericRecord types, convert them to our own specific types
            // if we can figure out which to use. This makes toString'ing prettier.
            if (fieldVal instanceof GenericRecord) {
                GenericRecord record = (GenericRecord) fieldVal;
                if (record.getSchema().equals(TimestampBase.SCHEMA$)) {
                    fieldVal = new Timestamp((Long) record.get("milliseconds"), (Long) record.get("nanos"));
                }
            }
            if (null == fieldVal) {
                sb.append("null");
            } else if (fieldVal instanceof ByteBuffer) {
                sb.append("B[");
                String toStr = (String) BIN2STR_FN.eval(null, fieldVal);
                sb.append(toStr);
                sb.append("]");
            } else if (fieldVal instanceof List<?>) {
                sb.append("[");
                StringUtils.formatList(sb, (List<?>) fieldVal);
                sb.append("]");
            } else {
                sb.append(fieldVal);
            }
        }
        // Notify all subscribers of our output.
        String output = sb.toString();
        for (UserSession session : subscribers) {
            session.sendInfo(output);
        }
    }

    @Override
    public String toString() {
        StringBuilder sb = new StringBuilder();
        sb.append("Output(");
        StringUtils.formatList(sb, mInputFields);
        sb.append(")");
        return sb.toString();
    }
}

17 View Complete Implementation : MapOutputValue.java
Copyright Apache License 2.0
Author : Hanmourang
public clreplaced MapOutputValue {

    private static BinaryDecoder binaryDecoder;

    private String schemaName;

    private GenericRecord record;

    private GenericDatumWriter<GenericRecord> WRITER;

    private EncoderFactory factory = EncoderFactory.get();

    private BinaryEncoder binaryEncoder;

    public MapOutputValue(String schemaName, GenericRecord record) {
        this.schemaName = schemaName;
        this.record = record;
    }

    public String getSchemaName() {
        return schemaName;
    }

    public GenericRecord getRecord() {
        return record;
    }

    public byte[] toBytes() throws IOException {
        ByteArrayOutputStream dataStream = new ByteArrayOutputStream();
        Schema schema = record.getSchema();
        if (WRITER == null) {
            WRITER = new GenericDatumWriter<GenericRecord>(schema);
        }
        binaryEncoder = factory.directBinaryEncoder(dataStream, binaryEncoder);
        WRITER.write(record, binaryEncoder);
        // serialize to bytes, we also need to know the schema name when we
        // process this record on the reducer since reducer gets the record from
        // multiple mappers. So we first write the schema/source name and then
        // write the serialized bytes
        ByteArrayOutputStream out = new ByteArrayOutputStream();
        DataOutputStream dos = new DataOutputStream(out);
        System.out.println("schema name:" + schemaName + "  length:" + schemaName.getBytes().length);
        dos.writeInt(schema.getName().getBytes().length);
        dos.write(schema.getName().getBytes());
        byte[] dataBytes = dataStream.toByteArray();
        System.out.println("Data Buffer length:" + dataBytes.length);
        dos.writeInt(dataBytes.length);
        dos.write(dataBytes);
        return out.toByteArray();
    }

    public static MapOutputValue fromBytes(byte[] bytes, Map<String, Schema> schemaMap) throws IOException {
        DataInputStream dataInputStream = new DataInputStream(new ByteArrayInputStream(bytes));
        int length = dataInputStream.readInt();
        byte[] sourceNameBytes = new byte[length];
        dataInputStream.read(sourceNameBytes);
        String schemaName = new String(sourceNameBytes);
        int recordDataLength = dataInputStream.readInt();
        byte[] recordBytes = new byte[recordDataLength];
        dataInputStream.read(recordBytes);
        Schema schema = schemaMap.get(schemaName);
        GenericRecord record = new GenericData.Record(schema);
        binaryDecoder = DecoderFactory.get().binaryDecoder(recordBytes, binaryDecoder);
        GenericDatumReader<GenericRecord> gdr = new GenericDatumReader<GenericRecord>(schema);
        gdr.read(record, binaryDecoder);
        return new MapOutputValue(schemaName, record);
    }
}

17 View Complete Implementation : DataGenerator.java
Copyright Apache License 2.0
Author : linkedin
private static void printAvroJson(GenericRecord record, OutputStream outs) throws IOException {
    JsonEncoder jsonEnc = new JsonEncoder(record.getSchema(), outs);
    GenericDatumWriter<GenericRecord> datumWriter = new GenericDatumWriter<GenericRecord>(record.getSchema());
    datumWriter.write(record, jsonEnc);
    jsonEnc.flush();
}