org.apache.avro.io.BinaryDecoder - java examples

Here are the examples of the java api org.apache.avro.io.BinaryDecoder taken from open source projects. By voting up you can indicate which examples are most useful and appropriate.

99 Examples 7

19 View Complete Implementation : AvroGenericDeserializer.java
Copyright BSD 2-Clause "Simplified" License
Author : linkedin
public V deserialize(BinaryDecoder decoder) throws Exception {
    return deserialize(null, decoder);
}

19 View Complete Implementation : GeoWaveAvroFeatureUtils.java
Copyright Apache License 2.0
Author : locationtech
/**
 * * Deserialize byte stream into an AvroSimpleFeature
 *
 * @param avroData serialized bytes of AvroSimpleFeature
 * @param avroObjectToReuse null or AvroSimpleFeature instance to be re-used. If null a new object
 *        will be allocated.
 * @return instance of AvroSimpleFeature with values parsed from avroData
 * @throws IOException
 */
private static AvroSimpleFeature deserializeASF(final byte[] avroData, AvroSimpleFeature avroObjectToReuse) throws IOException {
    final BinaryDecoder decoder = DECODER_FACTORY.binaryDecoder(avroData, null);
    if (avroObjectToReuse == null) {
        avroObjectToReuse = new AvroSimpleFeature();
    }
    DATUM_READER.setSchema(avroObjectToReuse.getSchema());
    return DATUM_READER.read(avroObjectToReuse, decoder);
}

19 View Complete Implementation : DivolteKafkaConsumer.java
Copyright Apache License 2.0
Author : divolte
private void scheduleReader(final KafkaStream<byte[], byte[]> stream, final BinaryDecoder decoder, final SpecificDatumReader<T> reader) {
    executorService.submit(new Runnable() {

        @Override
        public void run() {
            final EventHandler<T> handler = handlerSupplier.get();
            try {
                handler.setup();
                final ConsumerIterator<byte[], byte[]> iterator = stream.iterator();
                while (iterator.hasNext()) {
                    final byte[] message = iterator.next().message();
                    DecoderFactory.get().binaryDecoder(message, decoder);
                    handler.handle(reader.read(null, decoder));
                }
            } catch (Exception e) {
                logger.warn("Exception in event handler. Re-scheduling.", e);
                scheduleReader(stream, decoder, reader);
            }
            try {
                handler.shutdown();
            } catch (Exception e) {
                logger.warn("Exception in event handler shutdown.", e);
            }
        }
    });
}

19 View Complete Implementation : CountableSkipDataFileStream.java
Copyright Apache License 2.0
Author : jwoschitz
/**
 * Optimized version of org.apache.avro.file.DataFileStream which is only reading the data
 * from the InputStream which is necessary to retrieve the block count
 *
 * The focus of this implementation is counting and it is not providing features for record deserialization.
 * For everything apart counting avro files, use the original DataFileStream implementation.
 *
 * @see org.apache.avro.file.DataFileStream
 */
public clreplaced CountableSkipDataFileStream implements Closeable {

    private BinaryDecoder vin;

    private boolean availableBlock = false;

    private long blockSize;

    private long blockCount;

    private byte[] expectedSync = new byte[DataFileConstants.SYNC_SIZE];

    private byte[] syncBuffer = new byte[DataFileConstants.SYNC_SIZE];

    public CountableSkipDataFileStream(InputStream in) throws IOException {
        initialize(in);
    }

    private void initialize(InputStream in) throws IOException {
        this.vin = DecoderFactory.get().binaryDecoder(in, vin);
        byte[] magic = new byte[DataFileConstants.MAGIC.length];
        try {
            vin.readFixed(magic);
        } catch (IOException e) {
            throw new IOException("Not a data file.", e);
        }
        if (!Arrays.equals(DataFileConstants.MAGIC, magic))
            throw new IOException("Not a data file.");
        long l = vin.readMapStart();
        if (l > 0) {
            do {
                for (long i = 0; i < l; i++) {
                    vin.skipString();
                    vin.skipBytes();
                }
            } while ((l = vin.mapNext()) != 0);
        }
        vin.readFixed(expectedSync);
    }

    public long getBlockCount() {
        return blockCount;
    }

    public boolean hasNextBlock() {
        try {
            if (availableBlock)
                return true;
            if (vin.isEnd())
                return false;
            final long blockRemaining = vin.readLong();
            blockSize = vin.readLong();
            if (blockSize > Integer.MAX_VALUE || blockSize < 0) {
                throw new IOException("Block size invalid or too large for this " + "implementation: " + blockSize);
            }
            blockCount = blockRemaining;
            availableBlock = true;
            return true;
        } catch (EOFException eof) {
            return false;
        } catch (IOException e) {
            throw new AvroRuntimeException(e);
        }
    }

    public void nextBlock() throws IOException {
        if (!hasNextBlock()) {
            throw new NoSuchElementException();
        }
        vin.skipFixed((int) blockSize);
        vin.readFixed(syncBuffer);
        availableBlock = false;
        if (!Arrays.equals(syncBuffer, expectedSync))
            throw new IOException("Invalid sync!");
    }

    @Override
    public void close() throws IOException {
        vin.inputStream().close();
    }
}

19 View Complete Implementation : KryoAvroSerializer.java
Copyright Apache License 2.0
Author : rbrush
@Override
public Object read(Kryo kryo, Input input, Clreplaced type) {
    int length = input.readInt(true);
    byte[] bytes = input.readBytes(length);
    BinaryDecoder decoder = DecoderFactory.get().binaryDecoder(bytes, null);
    try {
        return reader.read(null, decoder);
    } catch (IOException e) {
        throw new AppException(e);
    }
}

19 View Complete Implementation : Util.java
Copyright Apache License 2.0
Author : ArchitectingHBase
public clreplaced Util {

    protected BinaryDecoder decoder = null;

    protected SpecificDatumReader<Event> reader = new SpecificDatumReader<Event>(Event.getClreplacedSchema());

    /**
     * Convert a HBase cell into it's Event AVRO representation.
     * @param cell the HBase cell to convert.
     * @param reuse an existing Event object to reuse, if any. Else, null.
     * @return the AVRO representation of the HBae cell.
     * @throws IOException
     */
    public Event cellToEvent(Cell cell, Event reuse) throws IOException {
        decoder = DecoderFactory.get().binaryDecoder(Bytes.copy(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength()), decoder);
        reuse = reader.read(reuse, decoder);
        return reuse;
    }
}

19 View Complete Implementation : AvroGenericDeserializer.java
Copyright BSD 2-Clause "Simplified" License
Author : linkedin
public V deserialize(V reuseRecord, byte[] bytes) throws Exception {
    // This param is to re-use a decoder instance. TODO: explore GC tuning later.
    BinaryDecoder decoder = DecoderFactory.defaultFactory().createBinaryDecoder(bytes, null);
    return deserialize(reuseRecord, decoder);
}

19 View Complete Implementation : AvroEventWrapper.java
Copyright Apache License 2.0
Author : flumebase
/**
 * An EventWrapper that operates on events containing parsed records
 * that are encoded as Avro record instances.
 */
public clreplaced AvroEventWrapper extends EventWrapperImpl {

    private Event mEvent;

    // true if mEvent has been decoded into mRecord.
    private boolean mIsDecoded;

    // Members used to decode Avro into fields.
    private DecoderFactory mDecoderFactory;

    private BinaryDecoder mDecoder;

    private GenericData.Record mRecord;

    private GenericDatumReader<GenericData.Record> mGenericReader;

    public AvroEventWrapper(Schema inputSchema) {
        this(inputSchema, inputSchema);
    }

    public AvroEventWrapper(Schema inputSchema, Schema outputSchema) {
        mDecoderFactory = new DecoderFactory();
        mRecord = new GenericData.Record(inputSchema);
        mGenericReader = new GenericDatumReader<GenericData.Record>(inputSchema, outputSchema);
    }

    @Override
    public void reset(Event e) {
        mEvent = e;
        mIsDecoded = false;
    }

    /**
     * Decode mEevent into mRecord.
     */
    private void decode() throws IOException {
        mDecoder = mDecoderFactory.createBinaryDecoder(mEvent.getBody(), mDecoder);
        mRecord = mGenericReader.read(mRecord, mDecoder);
        mIsDecoded = true;
    }

    @Override
    public Object getField(TypedField field) throws IOException {
        if (!mIsDecoded) {
            // TODO(aaron): Figure out how to decode more lazily; if we knew which subset
            // of fields we might access, we could project onto a narrower reader schema
            // and only decode those fields...
            decode();
        }
        return mRecord.get(field.getAvroName());
    }

    @Override
    public Event getEvent() {
        return mEvent;
    }

    /**
     * @return the parsed avro record.
     */
    public GenericData.Record getRecord() throws IOException {
        if (!mIsDecoded) {
            decode();
        }
        return mRecord;
    }
}

19 View Complete Implementation : OperatorUtil.java
Copyright Apache License 2.0
Author : pinterest
public static BrokerStats deserializeBrokerStats(ConsumerRecord<byte[], byte[]> record) {
    try {
        BinaryDecoder binaryDecoder = avroDecoderFactory.binaryDecoder(record.value(), null);
        BrokerStats stats = new BrokerStats();
        brokerStatsReader.read(stats, binaryDecoder);
        return stats;
    } catch (Exception e) {
        LOG.debug("Fail to decode an message", e);
        return null;
    }
}

19 View Complete Implementation : AvroObjectInput.java
Copyright Apache License 2.0
Author : apache
public clreplaced AvroObjectInput implements ObjectInput {

    private static DecoderFactory decoderFactory = DecoderFactory.get();

    private BinaryDecoder decoder;

    public AvroObjectInput(InputStream in) {
        decoder = decoderFactory.binaryDecoder(in, null);
    }

    @Override
    public boolean readBool() throws IOException {
        return decoder.readBoolean();
    }

    @Override
    public byte readByte() throws IOException {
        byte[] bytes = new byte[1];
        decoder.readFixed(bytes);
        return bytes[0];
    }

    @Override
    public short readShort() throws IOException {
        return (short) decoder.readInt();
    }

    @Override
    public int readInt() throws IOException {
        return decoder.readInt();
    }

    @Override
    public long readLong() throws IOException {
        return decoder.readLong();
    }

    @Override
    public float readFloat() throws IOException {
        return decoder.readFloat();
    }

    @Override
    public double readDouble() throws IOException {
        return decoder.readDouble();
    }

    @Override
    public String readUTF() throws IOException {
        Utf8 result = new Utf8();
        result = decoder.readString(result);
        return result.toString();
    }

    @Override
    public byte[] readBytes() throws IOException {
        String resultStr = decoder.readString();
        return resultStr.getBytes(StandardCharsets.UTF_8);
    }

    /**
     * will lost all attribute
     */
    @Override
    public Object readObject() throws IOException, ClreplacedNotFoundException {
        ReflectDatumReader<Object> reader = new ReflectDatumReader<>(Object.clreplaced);
        return reader.read(null, decoder);
    }

    @Override
    @SuppressWarnings(value = { "unchecked" })
    public <T> T readObject(Clreplaced<T> cls) throws IOException, ClreplacedNotFoundException {
        // Map interface clreplaced change to HashMap implement
        if (cls == Map.clreplaced) {
            cls = (Clreplaced<T>) HashMap.clreplaced;
        }
        ReflectDatumReader<T> reader = new ReflectDatumReader<>(cls);
        return reader.read(null, decoder);
    }

    @Override
    public <T> T readObject(Clreplaced<T> cls, Type type) throws IOException, ClreplacedNotFoundException {
        ReflectDatumReader<T> reader = new ReflectDatumReader<>(cls);
        return reader.read(null, decoder);
    }
}

19 View Complete Implementation : PersistentDeserializer.java
Copyright Apache License 2.0
Author : apache
/**
 * Hadoop deserializer using {@link SpecificDatumReader}
 * with {@link BinaryDecoder}.
 */
public clreplaced PersistentDeserializer implements Deserializer<PersistentBase> {

    private BinaryDecoder decoder;

    private Clreplaced<? extends PersistentBase> persistentClreplaced;

    private boolean reuseObjects;

    private SpecificDatumReader<PersistentBase> datumReader;

    public PersistentDeserializer(Clreplaced<? extends PersistentBase> c, boolean reuseObjects) {
        this.persistentClreplaced = c;
        this.reuseObjects = reuseObjects;
        try {
            Schema schema = AvroUtils.getSchema(persistentClreplaced);
            datumReader = new SpecificDatumReader<PersistentBase>(schema);
        } catch (Exception ex) {
            throw new RuntimeException(ex);
        }
    }

    @Override
    public void open(InputStream in) throws IOException {
        /* It is very important to use a direct buffer, since Hadoop
* supplies an input stream that is only valid until the end of one
* record serialization. Each time deserialize() is called, the IS
* is advanced to point to the right location, so we should not
* buffer the whole input stream at once.
*/
        decoder = DecoderFactory.get().directBinaryDecoder(in, decoder);
    }

    @Override
    public void close() throws IOException {
    }

    @Override
    public PersistentBase deserialize(PersistentBase persistent) throws IOException {
        persistent = datumReader.read(reuseObjects ? persistent : null, decoder);
        byte[] __g__dirty = new byte[persistent.getFieldsCount()];
        decoder.readFixed(__g__dirty);
        persistent.setDirtyBytes(java.nio.ByteBuffer.wrap(__g__dirty));
        return persistent;
    }
}

19 View Complete Implementation : AvroOutputReader.java
Copyright Apache License 2.0
Author : sunsuk7tp
/**
 * An OutputReader that reads sequential StreamingMutations (from Creplacedandra's Avro client API), and converts them to
 * the objects used by CreplacedandraOutputFormat. This allows Hadoop Streaming to output efficiently to Creplacedandra via
 * a familiar API.
 *
 * Avro requires the reader's and writer's schema: otherwise, it replacedumes they are the same.
 * If the canonical schema that the Creplacedandra side uses changes, and somebody packaged the {{avpr}}
 * up in their application somehow, or generated code, they'd see a runtime failure.
 * We could allow specifying an alternate Avro schema using a Configuration property to work around this.
 */
public clreplaced AvroOutputReader extends OutputReader<ByteBuffer, List<Mutation>> {

    private BinaryDecoder decoder;

    private SpecificDatumReader<StreamingMutation> reader;

    // reusable values
    private final StreamingMutation entry = new StreamingMutation();

    private final ArrayList<Mutation> mutations = new ArrayList<Mutation>(1);

    @Override
    public void initialize(PipeMapRed pmr) throws IOException {
        super.initialize(pmr);
        // set up decoding around the DataInput (hmm) provided by streaming
        InputStream in;
        if (pmr.getClientInput() instanceof InputStream)
            // let's hope this is the case
            in = (InputStream) pmr.getClientInput();
        else
            // ...because this is relatively slow
            in = new FromDataInputStream(pmr.getClientInput());
        decoder = DecoderFactory.defaultFactory().createBinaryDecoder(in, null);
        reader = new SpecificDatumReader<StreamingMutation>(StreamingMutation.SCHEMA$);
    }

    @Override
    public boolean readKeyValue() throws IOException {
        try {
            reader.read(entry, decoder);
        } catch (EOFException e) {
            return false;
        }
        mutations.clear();
        mutations.add(entry.mutation);
        return true;
    }

    @Override
    public ByteBuffer getCurrentKey() throws IOException {
        return entry.key;
    }

    @Override
    public List<Mutation> getCurrentValue() throws IOException {
        return mutations;
    }

    @Override
    public String getLastOutput() {
        return entry.toString();
    }

    /**
     * Wraps a DataInput to extend InputStream. The exception handling in read() is likely to be ridiculous slow.
     */
    private static final clreplaced FromDataInputStream extends InputStream {

        private final DataInput in;

        public FromDataInputStream(DataInput in) {
            this.in = in;
        }

        @Override
        public boolean markSupported() {
            return false;
        }

        @Override
        public int read() throws IOException {
            try {
                return in.readUnsignedByte();
            } catch (EOFException e) {
                return -1;
            }
        }

        @Override
        public long skip(long n) throws IOException {
            FileUtils.skipBytesFully(in, n);
            return n;
        }
    }
}

19 View Complete Implementation : AvroUtils.java
Copyright Apache License 2.0
Author : jveverka
public static Employee deserializeEmployee(byte[] data) throws IOException {
    DatumReader<Employee> employeeDatumReader = new SpecificDatumReader<>(Employee.clreplaced);
    BinaryDecoder binaryDecoder = DecoderFactory.get().binaryDecoder(data, null);
    employeeDatumReader.setSchema(Employee.getClreplacedSchema());
    return employeeDatumReader.read(null, binaryDecoder);
}

19 View Complete Implementation : IOUtils.java
Copyright Apache License 2.0
Author : apache
/**
 * An utility clreplaced for I/O related functionality.
 */
public clreplaced IOUtils {

    public static final int BUFFER_SIZE = 8192;

    private static BinaryDecoder decoder;

    private static Configuration getOrCreateConf(Configuration conf) {
        return conf != null ? conf : new Configuration();
    }

    public static Object readObject(DataInput in) throws ClreplacedNotFoundException, IOException {
        if (in instanceof ObjectInput) {
            return ((ObjectInput) in).readObject();
        } else {
            if (in instanceof InputStream) {
                ObjectInput objIn = new ObjectInputStream((InputStream) in);
                Object obj = objIn.readObject();
                return obj;
            }
        }
        throw new IOException("cannot read from DataInput of instance:" + in.getClreplaced());
    }

    public static void writeObject(DataOutput out, Object obj) throws IOException {
        if (out instanceof ObjectOutput) {
            ((ObjectOutput) out).writeObject(obj);
        } else {
            if (out instanceof OutputStream) {
                ObjectOutput objOut = new ObjectOutputStream((OutputStream) out);
                objOut.writeObject(obj);
            }
        }
        throw new IOException("cannot write to DataOutput of instance:" + out.getClreplaced());
    }

    /**
     * Serializes the object to the given data output using
     * available Hadoop serializations.
     *
     * @param conf Hadoop conf.
     * @param obj object instance to be serialized.
     * @param out data stream which serialized content is written.
     * @param objClreplaced Clreplaced type of the object to be serialized.
     * @param <T> clreplaced type of object to be serialized.
     * @throws IOException occurred while serializing the object to bytes.
     */
    public static <T> void serialize(Configuration conf, DataOutput out, T obj, Clreplaced<T> objClreplaced) throws IOException {
        SerializationFactory serializationFactory = new SerializationFactory(getOrCreateConf(conf));
        Serializer<T> serializer = serializationFactory.getSerializer(objClreplaced);
        try (ByteBufferOutputStream os = new ByteBufferOutputStream()) {
            serializer.open(os);
            serializer.serialize(obj);
            int length = 0;
            List<ByteBuffer> buffers = os.getBufferList();
            for (ByteBuffer buffer : buffers) {
                length += buffer.limit() - buffer.arrayOffset();
            }
            WritableUtils.writeVInt(out, length);
            for (ByteBuffer buffer : buffers) {
                byte[] arr = buffer.array();
                out.write(arr, buffer.arrayOffset(), buffer.limit());
            }
        } finally {
            if (serializer != null)
                serializer.close();
        }
    }

    /**
     * Serializes the object to the given data output using
     * available Hadoop serializations.
     *
     * @param <T> clreplaced type of object to be serialized.
     * @param conf Hadoop conf.
     * @param obj object instance to be serialized.
     * @param out data stream which serialized content is written.
     * @throws IOException occurred while serializing the object to bytes.
     */
    @SuppressWarnings("unchecked")
    public static <T> void serialize(Configuration conf, DataOutput out, T obj) throws IOException {
        Text.writeString(out, obj.getClreplaced().getName());
        serialize(conf, out, obj, (Clreplaced<T>) obj.getClreplaced());
    }

    /**
     * Serializes the object to the given data output using
     * available Hadoop serializations
     *
     * @param conf Hadoop conf.
     * @param <T> clreplaced type of object to be serialized.
     * @param obj object instance to be serialized.
     * @return serialized byte array.
     * @throws IOException occurred while serializing the object to bytes.
     */
    public static <T> byte[] serialize(Configuration conf, T obj) throws IOException {
        DataOutputBuffer buffer = new DataOutputBuffer();
        serialize(conf, buffer, obj);
        return buffer.getData();
    }

    /**
     * Serializes the field object using the datumWriter.
     *
     * @param <T> clreplaced type of object to be serialized.
     * @param datumWriter AVRO datum writer for given schema.
     * @param object object to be serialized.
     * @param os output stream which serialized content is written.
     * @throws IOException occurred while serializing the object to bytes.
     */
    public static <T extends SpecificRecord> void serialize(OutputStream os, SpecificDatumWriter<T> datumWriter, T object) throws IOException {
        BinaryEncoder encoder = EncoderFactory.get().binaryEncoder(os, null);
        datumWriter.write(object, encoder);
        encoder.flush();
    }

    /**
     * Serializes the field object using the datumWriter.
     *
     * @param <T> clreplaced type of object to be serialized.
     * @param datumWriter AVRO datum writer for given schema.
     * @param object object to be serialized.
     * @param os output stream which serialized content is written.
     * @throws IOException occurred while serializing the object to bytes.
     */
    public static <T> void serialize(OutputStream os, SpecificDatumWriter<T> datumWriter, T object) throws IOException {
        BinaryEncoder encoder = EncoderFactory.get().binaryEncoder(os, null);
        datumWriter.write(object, encoder);
        encoder.flush();
    }

    /**
     * Serializes the field object using the datumWriter.
     *
     * @param <T> clreplaced type of object to be serialized.
     * @param datumWriter AVRO datum writer for given schema.
     * @param object object to be serialized.
     * @return serialized byte array.
     * @throws IOException occurred while serializing the object to bytes.
     */
    public static <T extends SpecificRecord> byte[] serialize(SpecificDatumWriter<T> datumWriter, T object) throws IOException {
        ByteArrayOutputStream os = new ByteArrayOutputStream();
        serialize(os, datumWriter, object);
        return os.toByteArray();
    }

    /**
     * Serializes the field object using the datumWriter.
     *
     * @param <T> clreplaced type of object to be serialized.
     * @param datumWriter AVRO datum writer for given schema.
     * @param object object to be serialized.
     * @return serialized byte array.
     * @throws IOException occurred while serializing the object to bytes.
     */
    public static <T> byte[] serialize(SpecificDatumWriter<T> datumWriter, T object) throws IOException {
        ByteArrayOutputStream os = new ByteArrayOutputStream();
        serialize(os, datumWriter, object);
        return os.toByteArray();
    }

    /**
     * Deserializes the object in the given data input using
     * available Hadoop serializations.
     *
     * @param conf Hadoop conf.
     * @param in data input stream where serialized content is read.
     * @param <T> object clreplaced type.
     * @param obj data object.
     * @param objClreplaced object clreplaced type as String.
     * @return deserialized object.
     * @throws IOException occurred while deserializing the byte content.
     * @throws ClreplacedNotFoundException clreplaced definition cannot be found for given clreplaced name.
     */
    @SuppressWarnings("unchecked")
    public static <T> T deserialize(Configuration conf, DataInput in, T obj, String objClreplaced) throws IOException, ClreplacedNotFoundException {
        Clreplaced<T> c = (Clreplaced<T>) ClreplacedLoadingUtils.loadClreplaced(objClreplaced);
        return deserialize(conf, in, obj, c);
    }

    /**
     * Deserializes the object in the given data input using
     * available Hadoop serializations.
     *
     * @param conf Hadoop conf.
     * @param in data input stream where serialized content is read.
     * @param <T> object clreplaced type.
     * @param obj data object.
     * @param objClreplaced object clreplaced type.
     * @throws IOException occurred while deserializing the byte content.
     * @return deserialized object.
     */
    public static <T> T deserialize(Configuration conf, DataInput in, T obj, Clreplaced<T> objClreplaced) throws IOException {
        SerializationFactory serializationFactory = new SerializationFactory(getOrCreateConf(conf));
        Deserializer<T> deserializer = serializationFactory.getDeserializer(objClreplaced);
        int length = WritableUtils.readVInt(in);
        byte[] arr = new byte[length];
        in.readFully(arr);
        List<ByteBuffer> list = new ArrayList<>();
        list.add(ByteBuffer.wrap(arr));
        try (ByteBufferInputStream is = new ByteBufferInputStream(list)) {
            deserializer.open(is);
            return deserializer.deserialize(obj);
        } finally {
            if (deserializer != null)
                deserializer.close();
        }
    }

    /**
     * Deserializes the object in the given data input using
     * available Hadoop serializations.
     *
     * @param conf Hadoop conf.
     * @param in data input stream where serialized content is read.
     * @param <T> object clreplaced type.
     * @param obj data object.
     * @throws IOException occurred while deserializing the byte content.
     * @throws ClreplacedNotFoundException clreplaced definition cannot be found for given clreplaced name.
     * @return deserialized object.
     */
    @SuppressWarnings("unchecked")
    public static <T> T deserialize(Configuration conf, DataInput in, T obj) throws IOException, ClreplacedNotFoundException {
        String clazz = Text.readString(in);
        Clreplaced<T> c = (Clreplaced<T>) ClreplacedLoadingUtils.loadClreplaced(clazz);
        return deserialize(conf, in, obj, c);
    }

    /**
     * Deserializes the object in the given data input using
     * available Hadoop serializations.
     *
     * @param conf Hadoop conf.
     * @param in serialized byte array of  object.
     * @param <T> object clreplaced type.
     * @param obj data object.
     * @throws IOException occurred while deserializing the byte content.
     * @throws ClreplacedNotFoundException clreplaced definition cannot be found for given clreplaced name.
     * @return deserialized object.
     */
    public static <T> T deserialize(Configuration conf, byte[] in, T obj) throws IOException, ClreplacedNotFoundException {
        try (DataInputBuffer buffer = new DataInputBuffer()) {
            buffer.reset(in, in.length);
            return deserialize(conf, buffer, obj);
        }
    }

    /**
     * Deserializes the field object using the datumReader.
     *
     * @param is data input stream.
     * @param datumReader AVRO datum reader replacedociated schema.
     * @param <T> field clreplaced type.
     * @param <K> key clreplaced type.
     * @param object data field object.
     * @return deserialized field object.
     * @throws IOException while deserializing the byte content.
     */
    public static <K, T extends SpecificRecord> T deserialize(InputStream is, SpecificDatumReader<T> datumReader, T object) throws IOException {
        decoder = DecoderFactory.get().binaryDecoder(is, decoder);
        return datumReader.read(object, decoder);
    }

    /**
     * Deserializes the field object using the datumReader.
     *
     * @param bytes serialized byte array of field.
     * @param datumReader AVRO datum reader replacedociated schema.
     * @param <T> field clreplaced type.
     * @param <K> key clreplaced type.
     * @param object data field object.
     * @return deserialized field object.
     * @throws IOException while deserializing the byte content.
     */
    public static <K, T extends SpecificRecord> T deserialize(byte[] bytes, SpecificDatumReader<T> datumReader, T object) throws IOException {
        decoder = DecoderFactory.get().binaryDecoder(bytes, decoder);
        return datumReader.read(object, decoder);
    }

    /**
     * Deserializes the field object using the datumReader.
     *
     * @param bytes serialized byte array of field.
     * @param datumReader AVRO datum reader replacedociated schema.
     * @param object data field object.
     * @param <T> field clreplaced type.
     * @param <K> key clreplaced type.
     * @return deserialized field object.
     * @throws IOException while deserializing the byte content.
     */
    public static <K, T> T deserialize(byte[] bytes, SpecificDatumReader<T> datumReader, T object) throws IOException {
        decoder = DecoderFactory.get().binaryDecoder(bytes, decoder);
        return datumReader.read(object, decoder);
    }

    /**
     * Writes a byte[] to the output, representing whether each given field is null
     * or not. A Vint and ceil( fields.length / 8 ) bytes are written to the output.
     *
     * @param out the output to write to.
     * @param fields the fields to check for null. @see #readNullFieldsInfo(DataInput).
     * @throws IOException when writing the data to the stream.
     */
    public static void writeNullFieldsInfo(DataOutput out, Object... fields) throws IOException {
        boolean[] isNull = new boolean[fields.length];
        for (int i = 0; i < fields.length; i++) {
            isNull[i] = (fields[i] == null);
        }
        writeBoolArray(out, isNull);
    }

    /**
     * Reads the data written by {@link #writeNullFieldsInfo(DataOutput, Object...)}
     * and returns a boolean array representing whether each field is null or not.
     *
     * @param in the input to read from.
     * @return a boolean[] representing whether each field is null or not.
     * @throws IOException when value is too long to fit in integer.
     */
    public static boolean[] readNullFieldsInfo(DataInput in) throws IOException {
        return readBoolArray(in);
    }

    /**
     * Writes a boolean[] to the output.
     *
     * @param out the output to write to.
     * @param boolArray boolean array.
     * @throws IOException when writing the data to the stream.
     */
    public static void writeBoolArray(DataOutput out, boolean[] boolArray) throws IOException {
        WritableUtils.writeVInt(out, boolArray.length);
        byte b = 0;
        int i;
        for (i = 0; i < boolArray.length; i++) {
            if (i % 8 == 0 && i != 0) {
                out.writeByte(b);
                b = 0;
            }
            b >>= 1;
            if (boolArray[i])
                b |= 0x80;
            else
                b &= 0x7F;
        }
        if (i % 8 != 0) {
            for (int j = 0; j < 8 - (i % 8); j++) {
                // shift for the remaining byte
                b >>= 1;
                b &= 0x7F;
            }
        }
        out.writeByte(b);
    }

    /**
     * Reads a boolean[] from input.
     *
     * @param in data input stream to read values.
     * @return boolean array.
     * @throws IOException when value too long to fit in integer.
     */
    public static boolean[] readBoolArray(DataInput in) throws IOException {
        int length = WritableUtils.readVInt(in);
        boolean[] arr = new boolean[length];
        byte b = 0;
        for (int i = 0; i < length; i++) {
            if (i % 8 == 0) {
                b = in.readByte();
            }
            arr[i] = (b & 0x01) > 0;
            b >>= 1;
        }
        return arr;
    }

    /**
     * Writes a boolean[] to the output.
     *
     * @param out encoder instance which wraps the output stream where data is written.
     * @param boolArray boolean array.
     * @throws IOException when failed writing the data to the stream.
     */
    public static void writeBoolArray(Encoder out, boolean[] boolArray) throws IOException {
        out.writeInt(boolArray.length);
        int byteArrLength = (int) Math.ceil(boolArray.length / 8.0);
        byte b = 0;
        byte[] arr = new byte[byteArrLength];
        int i;
        int arrIndex = 0;
        for (i = 0; i < boolArray.length; i++) {
            if (i % 8 == 0 && i != 0) {
                arr[arrIndex++] = b;
                b = 0;
            }
            b >>= 1;
            if (boolArray[i])
                b |= 0x80;
            else
                b &= 0x7F;
        }
        if (i % 8 != 0) {
            for (int j = 0; j < 8 - (i % 8); j++) {
                // shift for the remaining byte
                b >>= 1;
                b &= 0x7F;
            }
        }
        arr[arrIndex++] = b;
        out.writeFixed(arr);
    }

    /**
     * Reads a boolean[] from input.
     *
     * @param in decoder instance which wraps the input stream where data is read.
     * @return boolean array.
     * @throws IOException when failed reading the data from stream.
     */
    public static boolean[] readBoolArray(Decoder in) throws IOException {
        int length = in.readInt();
        boolean[] boolArr = new boolean[length];
        int byteArrLength = (int) Math.ceil(length / 8.0);
        byte[] byteArr = new byte[byteArrLength];
        in.readFixed(byteArr);
        int arrIndex = 0;
        byte b = 0;
        for (int i = 0; i < length; i++) {
            if (i % 8 == 0) {
                b = byteArr[arrIndex++];
            }
            boolArr[i] = (b & 0x01) > 0;
            b >>= 1;
        }
        return boolArr;
    }

    /**
     * Writes the String array to the given DataOutput.
     *
     * @param out the data output to write to.
     * @param arr the array to write. @see #readStringArray(DataInput).
     * @throws IOException when failed writing the data to output stream.
     */
    public static void writeStringArray(DataOutput out, String[] arr) throws IOException {
        WritableUtils.writeVInt(out, arr.length);
        for (String str : arr) {
            Text.writeString(out, str);
        }
    }

    /**
     * Reads and returns a String array that is written by
     * {@link #writeStringArray(DataOutput, String[])}.
     *
     * @param in the data input to read from.
     * @return read String[].
     * @throws IOException when failed reading the data from stream.
     */
    public static String[] readStringArray(DataInput in) throws IOException {
        int len = WritableUtils.readVInt(in);
        String[] arr = new String[len];
        for (int i = 0; i < len; i++) {
            arr[i] = Text.readString(in);
        }
        return arr;
    }

    /**
     * Stores the given object in the configuration under the given dataKey.
     *
     * @param obj the object to store.
     * @param conf the configuration to store the object into.
     * @param dataKey the key to store the data.
     * @param <T> the given object clreplaced type.
     * @throws IOException when failed storing the given data in Hadoop conf.
     */
    public static <T> void storeToConf(T obj, Configuration conf, String dataKey) throws IOException {
        String clreplacedKey = dataKey + "._clreplaced";
        conf.set(clreplacedKey, obj.getClreplaced().getName());
        DefaultStringifier.store(conf, obj, dataKey);
    }

    /**
     * Loads the object stored by {@link #storeToConf(Object, Configuration, String)}
     * method from the configuration under the given dataKey.
     *
     * @param conf the configuration to read from.
     * @param dataKey the key to get the data from.
     * @param <T> the given object clreplaced type.
     * @return the store object.
     * @throws IOException when failed retrieving the data given key from Hadoop conf.
     */
    @SuppressWarnings("unchecked")
    public static <T> T loadFromConf(Configuration conf, String dataKey) throws IOException {
        String clreplacedKey = dataKey + "._clreplaced";
        String clreplacedName = conf.get(clreplacedKey);
        try {
            return (T) DefaultStringifier.load(conf, dataKey, ClreplacedLoadingUtils.loadClreplaced(clreplacedName));
        } catch (IOException | ClreplacedNotFoundException ex) {
            throw new IOException(ex);
        }
    }

    /**
     * Copies the contents of the buffers into a single byte[].
     *
     * @param buffers input buffers to be merged.
     * @return merged byte array.
     */
    // TODO: not tested
    public static byte[] getAsBytes(List<ByteBuffer> buffers) {
        // find total size
        int size = 0;
        for (ByteBuffer buffer : buffers) {
            size += buffer.remaining();
        }
        byte[] arr = new byte[size];
        int offset = 0;
        for (ByteBuffer buffer : buffers) {
            int len = buffer.remaining();
            buffer.get(arr, offset, len);
            offset += len;
        }
        return arr;
    }

    /**
     * Reads until the end of the input stream, and returns the contents as a byte[].
     *
     * @param in input stream where the data is read.
     * @return byte array of read data from the stream.
     * @throws IOException when failed reading the data from input stream.
     */
    public static byte[] readFully(InputStream in) throws IOException {
        List<ByteBuffer> buffers = new ArrayList<>(4);
        while (true) {
            ByteBuffer buffer = ByteBuffer.allocate(BUFFER_SIZE);
            int count = in.read(buffer.array(), 0, BUFFER_SIZE);
            if (count > 0) {
                buffer.limit(count);
                buffers.add(buffer);
            }
            if (count < BUFFER_SIZE)
                break;
        }
        return getAsBytes(buffers);
    }
}

18 View Complete Implementation : TestAvroRelConversion.java
Copyright Apache License 2.0
Author : apache
private static <T> T genericRecordFromBytes(byte[] bytes, Schema schema) throws IOException {
    BinaryDecoder binDecoder = DecoderFactory.defaultFactory().createBinaryDecoder(bytes, null);
    GenericDatumReader<T> reader = new GenericDatumReader<>(schema);
    return reader.read(null, binDecoder);
}

18 View Complete Implementation : DocumentSerializer.java
Copyright Apache License 2.0
Author : ArchitectingHBase
public clreplaced DoreplacedentSerializer implements HbaseEventSerializer {

    public static final Log LOG = LogFactory.getLog(DoreplacedentSerializer.clreplaced);

    private byte[] payload;

    private byte[] cf;

    private BinaryDecoder decoder = null;

    private Doreplacedent doreplacedent = null;

    private Map<String, String> headers = null;

    private SpecificDatumReader<Doreplacedent> reader = null;

    @Override
    public void configure(Context context) {
        LOG.info("Performing configuration from Context.");
    }

    @Override
    public void configure(ComponentConfiguration conf) {
        LOG.info("Performing configuration from ComponentConfiguration.");
    }

    // tag::SERIALIZE1[]
    @Override
    public void initialize(Event event, byte[] cf) {
        if (LOG.isDebugEnabled())
            LOG.debug("Performing initialization for a " + event.getBody().length + " bytes event");
        else
            LOG.info("Performing initialization for an event");
        // <1>
        this.payload = event.getBody();
        this.cf = cf;
        this.headers = event.getHeaders();
        reader = new SpecificDatumReader<Doreplacedent>(Doreplacedent.clreplaced);
    }

    @Override
    public List<Row> getActions() throws FlumeException {
        if (LOG.isInfoEnabled())
            LOG.info("Retrieving actions.");
        List<Row> actions = new LinkedList<Row>();
        try {
            decoder = DecoderFactory.get().binaryDecoder(payload, decoder);
            // <2>
            doreplacedent = reader.read(doreplacedent, decoder);
            byte[] rowKeyBytes = Bytes.add(DigestUtils.md5("" + doreplacedent.getSin()), Bytes.toBytes(doreplacedent.getSin().intValue()));
            LOG.info("SIN = " + doreplacedent.getSin());
            LOG.info("rowKey = " + Bytes.toStringBinary(rowKeyBytes));
            Put put = new Put(rowKeyBytes);
            // <3>
            put.addColumn(cf, Bytes.toBytes(System.currentTimeMillis()), payload);
            actions.add(put);
            String firstCellColumn;
            if ((firstCellColumn = headers.get(COLUMN)) != null) {
                String payload = headers.get(PAYLOAD);
                // <4>
                put.addColumn(cf, Base64.decode(firstCellColumn), Base64.decode(payload));
                LOG.info("Updating first cell " + Bytes.toStringBinary(Base64.decode(firstCellColumn)));
            }
        } catch (Exception e) {
            LOG.error("Unable to serialize flume event to HBase action!", e);
            throw new FlumeException("Unable to serialize flume event to HBase action!", e);
        }
        return actions;
    }

    // end::SERIALIZE1[]
    @Override
    public List<Increment> getIncrements() {
        return new LinkedList<Increment>();
    }

    @Override
    public void close() {
    }
}

18 View Complete Implementation : GenericSchemaDecoder.java
Copyright Apache License 2.0
Author : BriData
/**
 * 解析被generic schema封装的实际数据
 *
 * @param schema  schema对象
 * @param payload 实际数据
 * @return List<GenericRecord>
 * @throws Exception
 */
public List<GenericRecord> decode(Schema schema, byte[] payload, int start, int len) throws Exception {
    logger.trace("Schema:" + schema.toString() + " schema payload:" + new String(payload, "utf-8"));
    List<GenericRecord> list = new LinkedList<>();
    DatumReader<GenericRecord> reader = new GenericDatumReader<>(schema);
    BinaryDecoder decoder = DecoderFactory.get().binaryDecoder(payload, start, len, null);
    while (!decoder.isEnd()) {
        list.add(reader.read(null, decoder));
    }
    return list;
}

18 View Complete Implementation : GenericSchemaDecoder.java
Copyright Apache License 2.0
Author : BriData
/**
 * 解析被generic schema封装的实际数据
 *
 * @param schema  schema对象
 * @param payload 实际数据
 * @return List<GenericRecord>
 * @throws Exception
 */
public List<GenericRecord> decode(Schema schema, byte[] payload) throws Exception {
    logger.trace("Schema:" + schema.toString() + " schema payload:" + new String(payload, "utf-8"));
    List<GenericRecord> list = new LinkedList<>();
    DatumReader<GenericRecord> reader = new GenericDatumReader<>(schema);
    BinaryDecoder decoder = getBinaryDecoder(payload);
    while (!decoder.isEnd()) {
        list.add(reader.read(null, decoder));
    }
    return list;
}

18 View Complete Implementation : AvroGenericRecordWritable.java
Copyright Apache License 2.0
Author : bunnyg
/**
 * Wrapper around an Avro GenericRecord.  Necessary because Hive's deserializer
 * will happily deserialize any object - as long as it's a writable.
 */
public clreplaced AvroGenericRecordWritable implements Writable {

    GenericRecord record;

    private BinaryDecoder binaryDecoder;

    // There are two areas of exploration for optimization here.
    // 1.  We're serializing the schema with every object.  If we replacedume the schema
    // provided by the table is always correct, we don't need to do this and
    // and can just send the serialized bytes.
    // 2.  We serialize/deserialize to/from bytes immediately.  We may save some
    // time but doing this lazily, but until there's evidence this is useful,
    // it's not worth adding the extra state.
    public GenericRecord getRecord() {
        return record;
    }

    public void setRecord(GenericRecord record) {
        this.record = record;
    }

    public AvroGenericRecordWritable() {
    }

    public AvroGenericRecordWritable(GenericRecord record) {
        this.record = record;
    }

    @Override
    public void write(DataOutput out) throws IOException {
        // Write schema since we need it to pull the data out. (see point #1 above)
        String schemaString = record.getSchema().toString(false);
        out.writeUTF(schemaString);
        // Write record to byte buffer
        GenericDatumWriter<GenericRecord> gdw = new GenericDatumWriter<GenericRecord>();
        BinaryEncoder be = EncoderFactory.get().directBinaryEncoder((DataOutputStream) out, null);
        gdw.setSchema(record.getSchema());
        gdw.write(record, be);
    }

    @Override
    public void readFields(DataInput in) throws IOException {
        Schema schema = Schema.parse(in.readUTF());
        record = new GenericData.Record(schema);
        binaryDecoder = DecoderFactory.defaultFactory().createBinaryDecoder((InputStream) in, binaryDecoder);
        GenericDatumReader<GenericRecord> gdr = new GenericDatumReader<GenericRecord>(schema);
        record = gdr.read(record, binaryDecoder);
    }
}

18 View Complete Implementation : DivolteKafkaConsumer.java
Copyright Apache License 2.0
Author : divolte
/**
 * Starts the consumer threads. Each consumer thread will request a
 * {@link EventHandler} from the provided {@link Supplier}.
 */
public void startConsumer() {
    ImmutableMap<String, Integer> threadsPerTopicMap = ImmutableMap.of(Objects.requireNonNull(topic), numThreads);
    for (final KafkaStream<byte[], byte[]> stream : consumer.createMessageStreams(threadsPerTopicMap).get(topic)) {
        final BinaryDecoder decoder = DecoderFactory.get().binaryDecoder(new byte[0], null);
        final SpecificDatumReader<T> reader = new SpecificDatumReader<>(schema);
        scheduleReader(stream, decoder, reader);
    }
}

18 View Complete Implementation : UnwrapInterceptor.java
Copyright Apache License 2.0
Author : ggear
/**
 * Use this {@link Interceptor} to unwrap a Flume encoded event from a direct
 * {@link Source source} {@link Channel channel} (ie Kafka).
 */
public clreplaced UnwrapInterceptor implements Interceptor {

    private final static Logger LOG = LoggerFactory.getLogger(UnwrapInterceptor.clreplaced);

    private BinaryDecoder decoder = null;

    private SpecificDatumReader<AvroFlumeEvent> reader = null;

    private static Map<String, String> toStringMap(Map<CharSequence, CharSequence> charSequenceMap, Map<String, String> mergeMap) {
        Map<String, String> stringMap = new HashMap<>();
        for (Map.Entry<CharSequence, CharSequence> entry : charSequenceMap.entrySet()) {
            stringMap.put(entry.getKey().toString(), entry.getValue().toString());
        }
        stringMap.putAll(mergeMap);
        return stringMap;
    }

    @Override
    public void initialize() {
        decoder = DecoderFactory.get().directBinaryDecoder(new ByteArrayInputStream(new byte[0]), decoder);
        reader = new SpecificDatumReader<>(AvroFlumeEvent.clreplaced);
        if (LOG.isInfoEnabled()) {
            LOG.info("Event Unwrap Interceptor initialised");
        }
    }

    @Override
    public Event intercept(Event event) {
        return unwrap(event);
    }

    @Override
    public List<Event> intercept(List<Event> events) {
        List<Event> eventsUnwrapped = new ArrayList<>();
        for (Event event : events) {
            eventsUnwrapped.add(unwrap(event));
        }
        return eventsUnwrapped;
    }

    @Override
    public void close() {
    }

    public Event unwrap(Event event) {
        if (LOG.isDebugEnabled()) {
            LOG.debug("Attempting to unwrap event, body [" + event.getBody().length + "] bytes");
        }
        Event eventUnwrapped = event;
        InputStream eventWrappedStream = new ByteArrayInputStream(event.getBody());
        try {
            decoder = DecoderFactory.get().directBinaryDecoder(eventWrappedStream, decoder);
            AvroFlumeEvent eventUnwrappedAvro = reader.read(null, decoder);
            eventUnwrapped = EventBuilder.withBody(eventUnwrappedAvro.getBody().array(), toStringMap(eventUnwrappedAvro.getHeaders(), event.getHeaders()));
            if (LOG.isDebugEnabled()) {
                LOG.debug("Event successfully unwrapped, header [" + eventUnwrappedAvro.getHeaders().size() + "] fields, body [" + eventUnwrapped.getBody().length + "] bytes");
            }
        } catch (Exception exception) {
            if (LOG.isWarnEnabled()) {
                LOG.warn("Failed to unwrap event, " + "perhaps this source is not connected to a sinkless connector?", exception);
            }
        } finally {
            IOUtils.closeQuietly(eventWrappedStream);
        }
        return eventUnwrapped;
    }

    public static clreplaced Builder implements Interceptor.Builder {

        @Override
        public void configure(Context context) {
        }

        @Override
        public Interceptor build() {
            return new UnwrapInterceptor();
        }
    }
}

18 View Complete Implementation : AvroDeserializationSchema.java
Copyright MIT License
Author : jMetal
public clreplaced AvroDeserializationSchema<T> implements DeserializationSchema<T> {

    private static final long serialVersionUID = 1L;

    private final Clreplaced<T> avroType;

    private transient DatumReader<T> reader;

    private transient BinaryDecoder decoder;

    private transient String path;

    public AvroDeserializationSchema(Clreplaced<T> avroType, String path) {
        this.avroType = avroType;
        this.path = path;
    }

    @Override
    public T deserialize(byte[] message) {
        ensureInitialized();
        try {
            // decoder = DecoderFactory.get().binaryDecoder(message, decoder);
            File file = new File(path);
            Schema schema = new Schema.Parser().parse(file);
            SpecificDatumReader<T> reader = new SpecificDatumReader<>(schema);
            decoder = DecoderFactory.get().binaryDecoder(message, null);
            return reader.read(null, decoder);
        } catch (Exception ex) {
            throw new RuntimeException(ex);
        }
    }

    private void ensureInitialized() {
        if (reader == null) {
            if (org.apache.avro.specific.SpecificRecordBase.clreplaced.isreplacedignableFrom(avroType)) {
                reader = new SpecificDatumReader<T>(avroType);
            } else {
                reader = new ReflectDatumReader<T>(avroType);
            }
        }
    }

    @Override
    public boolean isEndOfStream(T nextElement) {
        return false;
    }

    @Override
    public TypeInformation<T> getProducedType() {
        return TypeExtractor.getForClreplaced(avroType);
    }
}

18 View Complete Implementation : ColumnDecoder.java
Copyright Apache License 2.0
Author : kite-sdk
/**
 * An Avro Decoder implementation used for decoding Avro instances from HBase
 * columns. This is basically an Avro BinaryDecoder with custom encoding of int,
 * long, and String types.
 *
 * int and long are serialized in standard 4 and 8 byte format (instead of
 * Avro's ZigZag encoding) so that we can use HBase's atomic increment
 * functionality on columns.
 *
 * Strings are encoded as UTF-8 bytes. This is consistent
 * with HBase, and will allow appends in the future.
 */
public clreplaced ColumnDecoder extends Decoder {

    private final BinaryDecoder wrappedDecoder;

    private final InputStream in;

    private final DataInputStream dataIn;

    public ColumnDecoder(InputStream in) {
        this.in = in;
        this.wrappedDecoder = new DecoderFactory().binaryDecoder(in, null);
        this.dataIn = new DataInputStream(in);
    }

    @Override
    public void readNull() throws IOException {
        wrappedDecoder.readNull();
    }

    @Override
    public boolean readBoolean() throws IOException {
        return wrappedDecoder.readBoolean();
    }

    @Override
    public int readInt() throws IOException {
        byte[] b = new byte[4];
        dataIn.readFully(b);
        return Bytes.toInt(b);
    }

    @Override
    public long readLong() throws IOException {
        byte[] b = new byte[8];
        dataIn.readFully(b);
        return Bytes.toLong(b);
    }

    @Override
    public float readFloat() throws IOException {
        return wrappedDecoder.readFloat();
    }

    @Override
    public double readDouble() throws IOException {
        return wrappedDecoder.readDouble();
    }

    @Override
    public Utf8 readString(Utf8 old) throws IOException {
        // replacedumes 'in' is ByteArrayInputStream so knows length
        int bytesAvailable = in.available();
        byte[] bytes = new byte[bytesAvailable];
        in.read(bytes);
        return new Utf8(bytes);
    }

    @Override
    public String readString() throws IOException {
        return readString(null).toString();
    }

    @Override
    public void skipString() throws IOException {
        // replacedumes 'in' is ByteArrayInputStream so knows length
        int bytesAvailable = in.available();
        in.skip(bytesAvailable);
    }

    @Override
    public ByteBuffer readBytes(ByteBuffer old) throws IOException {
        return wrappedDecoder.readBytes(old);
    }

    @Override
    public void skipBytes() throws IOException {
        wrappedDecoder.skipBytes();
    }

    @Override
    public void readFixed(byte[] bytes, int start, int length) throws IOException {
        wrappedDecoder.readFixed(bytes, start, length);
    }

    @Override
    public void skipFixed(int length) throws IOException {
        wrappedDecoder.skipFixed(length);
    }

    @Override
    public int readEnum() throws IOException {
        return wrappedDecoder.readEnum();
    }

    @Override
    public long readArrayStart() throws IOException {
        return wrappedDecoder.readArrayStart();
    }

    @Override
    public long arrayNext() throws IOException {
        return wrappedDecoder.arrayNext();
    }

    @Override
    public long skipArray() throws IOException {
        return wrappedDecoder.skipArray();
    }

    @Override
    public long readMapStart() throws IOException {
        return wrappedDecoder.readMapStart();
    }

    @Override
    public long mapNext() throws IOException {
        return wrappedDecoder.mapNext();
    }

    @Override
    public long skipMap() throws IOException {
        return wrappedDecoder.skipMap();
    }

    @Override
    public int readIndex() throws IOException {
        return wrappedDecoder.readIndex();
    }
}

18 View Complete Implementation : AvroGenericDeserializer.java
Copyright BSD 2-Clause "Simplified" License
Author : linkedin
public V deserialize(V reuseRecord, BinaryDecoder decoder) throws Exception {
    try {
        return datumReader.read(reuseRecord, decoder);
    } catch (Exception e) {
        throw new RuntimeException("Could not deserialize bytes back into Avro object", e);
    }
}

18 View Complete Implementation : AvroUtils.java
Copyright BSD 2-Clause "Simplified" License
Author : linkedin
/**
 * Decode and deserialize the byte array into an instance of an Avro record
 * @param schema schema describing the expected information of the bytes, valid for type T
 * @param bytes bytes to decode
 * @param reuse existing instance of T that may be used to populate with the decoded result. There is no guarantee it
 *              will actually be used.
 * @param <T> type of the instance to decode
 * @return decoded instance of T
 */
public static <T> T decodeAvroGenericRecord(Schema schema, byte[] bytes, T reuse) throws IOException {
    BinaryDecoder binDecoder = DecoderFactory.defaultFactory().createBinaryDecoder(bytes, null);
    GenericDatumReader<T> reader = new GenericDatumReader<>(schema);
    return reader.read(reuse, binDecoder);
}

18 View Complete Implementation : AvroDatumEnvelopeSerDe.java
Copyright Apache License 2.0
Author : outbrain
/**
 * @param buffer data
 * @param writer writer schema for the SpecificDatumReader, useful when producers may use different schema versions
 * @return deserialized object
 */
public DatumEnvelope deserializeDatumEnvelope(final ByteBuffer buffer, final Schema writer) {
    try (final InputStream byteBufferInputStream = new ByteBufferInputStream(Collections.singletonList(buffer))) {
        // hack alert: using old envelope to reconcile version diffs
        final DatumReader<DatumEnvelope> datumReader = new SpecificDatumReader<>(writer, DatumEnvelope.getClreplacedSchema());
        final BinaryDecoder decoder = DecoderFactory.get().directBinaryDecoder(byteBufferInputStream, null);
        return datumReader.read(null, decoder);
    } catch (final Exception e) {
        throw new RuntimeException("Could not deserialize datum envelope", e);
    }
}

18 View Complete Implementation : Person.java
Copyright Apache License 2.0
Author : Talend
public static Person desFromAvroBytes(byte[] record) throws IOException {
    DatumReader<GenericRecord> datumReader = new GenericDatumReader<GenericRecord>(schema);
    BinaryDecoder decoder = null;
    decoder = DecoderFactory.get().binaryDecoder(record, decoder);
    GenericRecord avroValue = datumReader.read(null, decoder);
    return fromAvroRecord(avroValue);
}

18 View Complete Implementation : EventsConnection.java
Copyright Apache License 2.0
Author : trvedata
/**
 * Called by Jetty when a binary frame is received from the WebSocket client.
 */
@Override
public void onWebSocketBinary(byte[] payload, int offset, int len) {
    Session session = getSession();
    if (session == null || !session.isOpen())
        return;
    BinaryDecoder decoder = DecoderFactory.get().binaryDecoder(payload, offset, len, null);
    Object request;
    try {
        request = readFromClient.read(null, decoder).getMessage();
    } catch (Exception e) {
        log.warn("Failed to decode message from client", e);
        session.close(StatusCode.BAD_PAYLOAD, "Could not parse frame");
        return;
    }
    try {
        if (request instanceof SendMessage) {
            sendMessageRequest((SendMessage) request);
        } else if (request instanceof SubscribeToChannel) {
            subscribeRequest((SubscribeToChannel) request);
        } else {
            throw new IllegalStateException("Unknown request type: " + request.getClreplaced().getName());
        }
    } catch (Exception e) {
        log.info("Failed to publish message", e);
        session.close(StatusCode.SERVER_ERROR, "Internal server error");
    }
}

18 View Complete Implementation : AvroJobSpecDeserializer.java
Copyright Apache License 2.0
Author : apache
@Slf4j
public clreplaced AvroJobSpecDeserializer implements Deserializer<AvroJobSpec> {

    private BinaryDecoder _decoder;

    private SpecificDatumReader<AvroJobSpec> _reader;

    private SchemaVersionWriter<?> _versionWriter;

    @Override
    public void configure(Map<String, ?> configs, boolean isKey) {
        InputStream dummyInputStream = new ByteArrayInputStream(new byte[0]);
        _decoder = DecoderFactory.get().binaryDecoder(dummyInputStream, null);
        _reader = new SpecificDatumReader<AvroJobSpec>(AvroJobSpec.SCHEMA$);
        _versionWriter = new FixedSchemaVersionWriter();
    }

    @Override
    public AvroJobSpec deserialize(String topic, byte[] data) {
        try (InputStream is = new ByteArrayInputStream(data)) {
            _versionWriter.readSchemaVersioningInformation(new DataInputStream(is));
            Decoder decoder = DecoderFactory.get().binaryDecoder(is, _decoder);
            return _reader.read(null, decoder);
        } catch (IOException e) {
            throw new RuntimeException("Could not decode message");
        }
    }

    @Override
    public void close() {
    }
}

17 View Complete Implementation : AvroUtils.java
Copyright BSD 2-Clause "Simplified" License
Author : linkedin
/**
 * Decode and deserialize the byte array into an instance of an Avro record
 * @param schema schema describing the expected information of the bytes, valid for type T
 * @param bytes bytes to decode
 * @param reuse existing instance of T that may be used to populate with the decoded result. There is no guarantee it
 *              will actually be used.
 * @param <T> type of the instance to decode
 * @return decoded instance of T
 */
public static <T extends SpecificRecord> T decodeAvroSpecificRecord(Schema schema, byte[] bytes, T reuse) throws IOException {
    BinaryDecoder binDecoder = DecoderFactory.defaultFactory().createBinaryDecoder(bytes, null);
    SpecificDatumReader<T> reader = new SpecificDatumReader<>(schema);
    return reader.read(reuse, binDecoder);
}

17 View Complete Implementation : AvroWritable.java
Copyright Apache License 2.0
Author : opencb
public abstract clreplaced AvroWritable<T extends GenericRecord> implements Writable {

    protected T value;

    private final SpecificDatumWriter<T> writer;

    private final SpecificDatumReader<T> reader;

    private final BinaryEncoder binaryEncoder;

    private final BinaryDecoder binaryDecoder;

    private final DataInputAsInputStream is;

    private final DataOutputAsOutputStream os;

    protected AvroWritable(Clreplaced<T> sampleVariantStatsClreplaced) {
        writer = new SpecificDatumWriter<>(sampleVariantStatsClreplaced);
        reader = new SpecificDatumReader<>(sampleVariantStatsClreplaced);
        os = new DataOutputAsOutputStream(null);
        is = new DataInputAsInputStream(null);
        binaryEncoder = EncoderFactory.get().directBinaryEncoder(os, null);
        binaryDecoder = DecoderFactory.get().directBinaryDecoder(is, null);
    }

    @Override
    public void write(DataOutput dataOutput) throws IOException {
        writeAvro(dataOutput);
    }

    @Override
    public void readFields(DataInput dataInput) throws IOException {
        readAvro(dataInput);
    }

    protected final void writeAvro(DataOutput out) throws IOException {
        // Replace the DataOutput used by the encoder
        os.setOut(out);
        writer.write(value, binaryEncoder);
        binaryEncoder.flush();
    }

    protected final void readAvro(DataInput in) throws IOException {
        // Replace the DataInput used by the decoder
        is.setIn(in);
        value = reader.read(value, binaryDecoder);
    }

    public T getValue() {
        return value;
    }

    public AvroWritable<T> setValue(T value) {
        this.value = value;
        return this;
    }

    private static clreplaced DataOutputAsOutputStream extends OutputStream {

        private DataOutput out;

        DataOutputAsOutputStream(DataOutput out) {
            this.out = out;
        }

        public DataOutputAsOutputStream setOut(DataOutput out) {
            this.out = out;
            return this;
        }

        @Override
        public void write(int b) throws IOException {
            out.write(b);
        }

        @Override
        public void write(byte[] b) throws IOException {
            out.write(b);
        }

        @Override
        public void write(byte[] b, int off, int len) throws IOException {
            out.write(b, off, len);
        }
    }

    private static clreplaced DataInputAsInputStream extends InputStream {

        private DataInput in;

        DataInputAsInputStream(DataInput in) {
            this.in = in;
        }

        public DataInputAsInputStream setIn(DataInput in) {
            this.in = in;
            return this;
        }

        @Override
        public int read() throws IOException {
            return Byte.toUnsignedInt(in.readByte());
        }

        @Override
        public int read(byte[] b) throws IOException {
            in.readFully(b);
            return b.length;
        }

        @Override
        public int read(byte[] b, int off, int len) throws IOException {
            in.readFully(b, off, len);
            return len;
        }
    }
}

17 View Complete Implementation : VariantToAvroBinaryConverter.java
Copyright Apache License 2.0
Author : opencb
@Override
public Variant convertToDataModelType(Binary object) {
    BinaryDecoder decoder = null;
    try {
        byte[] data = object.getData();
        data = CompressionUtils.decompress(data);
        InputStream is = new ByteArrayInputStream(data);
        // is = new GZIPInputStream(is);
        decoder = DecoderFactory.get().directBinaryDecoder(is, decoder);
        return new Variant(reader.read(null, decoder));
    } catch (IOException e) {
        throw new UncheckedIOException(e);
    } catch (DataFormatException e) {
        throw new RuntimeException(e);
    }
}

17 View Complete Implementation : JavaToAvroSerializer.java
Copyright Apache License 2.0
Author : Stratio
@Override
public StratioStreamingMessage deserialize(byte[] object) {
    StratioStreamingMessage result = null;
    BinaryDecoder bd = DecoderFactory.get().binaryDecoder(object, null);
    if (datumReader == null) {
        datumReader = new SpecificDatumReader(InsertMessage.getClreplacedSchema());
    }
    try {
        InsertMessage insertMessage = (InsertMessage) datumReader.read(null, bd);
        result = convertMessage(insertMessage);
    } catch (IOException e) {
        e.printStackTrace();
    }
    return result;
}

17 View Complete Implementation : SpecificAvroSerializer.java
Copyright Apache License 2.0
Author : viniciusccarvalho
/**
 * @author Vinicius Carvalho
 */
public clreplaced SpecificAvroSerializer implements Serializer<Sensor> {

    private SpecificDatumWriter<Sensor> writer = new SpecificDatumWriter<>(Sensor.clreplaced);

    private SpecificDatumReader<Sensor> reader = new SpecificDatumReader<>(Sensor.SCHEMA$);

    private BinaryEncoder encoder;

    private BinaryDecoder decoder;

    @Override
    public Sensor read(byte[] bytes) throws Exception {
        decoder = DecoderFactory.get().binaryDecoder(bytes, this.decoder);
        return reader.read(null, decoder);
    }

    @Override
    public byte[] write(Sensor type) throws Exception {
        ByteArrayOutputStream baos = new ByteArrayOutputStream();
        encoder = EncoderFactory.get().binaryEncoder(baos, this.encoder);
        writer.write(type, encoder);
        encoder.flush();
        return baos.toByteArray();
    }
}

17 View Complete Implementation : AvroReader.java
Copyright Apache License 2.0
Author : apache
@Override
public T read(byte[] bytes, int offset, int length) {
    try {
        BinaryDecoder decoderFromCache = decoders.get();
        BinaryDecoder decoder = DecoderFactory.get().binaryDecoder(bytes, offset, length, decoderFromCache);
        if (decoderFromCache == null) {
            decoders.set(decoder);
        }
        return reader.read(null, DecoderFactory.get().binaryDecoder(bytes, offset, length, decoder));
    } catch (IOException e) {
        throw new SchemaSerializationException(e);
    }
}

17 View Complete Implementation : AvroGenericDeserializer.java
Copyright BSD 2-Clause "Simplified" License
Author : linkedin
public Iterable<V> deserializeObjects(BinaryDecoder decoder) throws Exception {
    List<V> objects = new ArrayList();
    try {
        while (!decoder.isEnd()) {
            objects.add(datumReader.read(null, decoder));
        }
    } catch (Exception e) {
        throw new RuntimeException("Could not deserialize bytes back into Avro objects", e);
    }
    return objects;
}

16 View Complete Implementation : AvroCoder.java
Copyright Apache License 2.0
Author : apache
@Override
public T decode(InputStream inStream) throws IOException {
    // Get a BinaryDecoder instance from the ThreadLocal cache and attempt to reuse it.
    BinaryDecoder decoderInstance = DECODER_FACTORY.directBinaryDecoder(inStream, decoder.get());
    // Save the potentially-new instance for later.
    decoder.set(decoderInstance);
    return reader.get().read(null, decoderInstance);
}

16 View Complete Implementation : TestAvroEventDeserializer.java
Copyright Apache License 2.0
Author : apache
@Test
public void resetTest() throws IOException {
    File tempFile = newTestFile(true);
    String target = tempFile.getAbsolutePath();
    logger.info("Target: {}", target);
    TransientPositionTracker tracker = new TransientPositionTracker(target);
    AvroEventDeserializer.Builder desBuilder = new AvroEventDeserializer.Builder();
    EventDeserializer deserializer = desBuilder.build(new Context(), new ResettableFileInputStream(tempFile, tracker));
    BinaryDecoder decoder = null;
    DatumReader<GenericRecord> reader = new GenericDatumReader<GenericRecord>(schema);
    decoder = DecoderFactory.get().binaryDecoder(deserializer.readEvent().getBody(), decoder);
    replacedertEquals("bar", reader.read(null, decoder).get("foo").toString());
    deserializer.reset();
    decoder = DecoderFactory.get().binaryDecoder(deserializer.readEvent().getBody(), decoder);
    replacedertEquals("bar", reader.read(null, decoder).get("foo").toString());
    deserializer.mark();
    decoder = DecoderFactory.get().binaryDecoder(deserializer.readEvent().getBody(), decoder);
    replacedertEquals("baz", reader.read(null, decoder).get("foo").toString());
    deserializer.reset();
    decoder = DecoderFactory.get().binaryDecoder(deserializer.readEvent().getBody(), decoder);
    replacedertEquals("baz", reader.read(null, decoder).get("foo").toString());
    replacedertNull(deserializer.readEvent());
}

16 View Complete Implementation : AvroParser.java
Copyright Apache License 2.0
Author : apache
/**
 * An {@link EnreplacedyParser} that parses Avro serialized bytes from an event.
 *
 * The Avro schema used to serialize the data should be set as either a URL
 * or literal in the flume.avro.schema.url or flume.avro.schema.literal event
 * headers respectively.
 */
public clreplaced AvroParser implements EnreplacedyParser<GenericRecord> {

    static Configuration conf = new Configuration();

    /**
     * A cache of literal schemas to avoid re-parsing the schema.
     */
    private static final LoadingCache<String, Schema> schemasFromLiteral = CacheBuilder.newBuilder().build(new CacheLoader<String, Schema>() {

        @Override
        public Schema load(String literal) {
            Preconditions.checkNotNull(literal, "Schema literal cannot be null without a Schema URL");
            return new Schema.Parser().parse(literal);
        }
    });

    /**
     * A cache of schemas retrieved by URL to avoid re-parsing the schema.
     */
    private static final LoadingCache<String, Schema> schemasFromURL = CacheBuilder.newBuilder().build(new CacheLoader<String, Schema>() {

        @Override
        public Schema load(String url) throws IOException {
            Schema.Parser parser = new Schema.Parser();
            InputStream is = null;
            try {
                FileSystem fs = FileSystem.get(URI.create(url), conf);
                if (url.toLowerCase(Locale.ENGLISH).startsWith("hdfs:/")) {
                    is = fs.open(new Path(url));
                } else {
                    is = new URL(url).openStream();
                }
                return parser.parse(is);
            } finally {
                if (is != null) {
                    is.close();
                }
            }
        }
    });

    /**
     * The schema of the destination dataset.
     *
     * Used as the reader schema during parsing.
     */
    private final Schema datasetSchema;

    /**
     * A cache of DatumReaders per schema.
     */
    private final LoadingCache<Schema, DatumReader<GenericRecord>> readers = CacheBuilder.newBuilder().build(new CacheLoader<Schema, DatumReader<GenericRecord>>() {

        @Override
        public DatumReader<GenericRecord> load(Schema schema) {
            // must use the target dataset's schema for reading to ensure the
            // records are able to be stored using it
            return new GenericDatumReader<GenericRecord>(schema, datasetSchema);
        }
    });

    /**
     * The binary decoder to reuse for event parsing.
     */
    private BinaryDecoder decoder = null;

    /**
     * Create a new AvroParser given the schema of the destination dataset.
     *
     * @param datasetSchema The schema of the destination dataset.
     */
    private AvroParser(Schema datasetSchema) {
        this.datasetSchema = datasetSchema;
    }

    /**
     * Parse the enreplacedy from the body of the given event.
     *
     * @param event The event to parse.
     * @param reuse If non-null, this may be reused and returned from this method.
     * @return The parsed enreplacedy as a GenericRecord.
     * @throws EventDeliveryException A recoverable error such as an error
     *                                downloading the schema from the URL has
     *                                occurred.
     * @throws NonRecoverableEventException A non-recoverable error such as an
     *                                      unparsable schema or enreplacedy has
     *                                      occurred.
     */
    @Override
    public GenericRecord parse(Event event, GenericRecord reuse) throws EventDeliveryException, NonRecoverableEventException {
        decoder = DecoderFactory.get().binaryDecoder(event.getBody(), decoder);
        try {
            DatumReader<GenericRecord> reader = readers.getUnchecked(schema(event));
            return reader.read(reuse, decoder);
        } catch (IOException ex) {
            throw new NonRecoverableEventException("Cannot deserialize event", ex);
        } catch (RuntimeException ex) {
            throw new NonRecoverableEventException("Cannot deserialize event", ex);
        }
    }

    /**
     * Get the schema from the event headers.
     *
     * @param event The Flume event
     * @return The schema for the event
     * @throws EventDeliveryException A recoverable error such as an error
     *                                downloading the schema from the URL has
     *                                occurred.
     * @throws NonRecoverableEventException A non-recoverable error such as an
     *                                      unparsable schema has occurred.
     */
    private static Schema schema(Event event) throws EventDeliveryException, NonRecoverableEventException {
        Map<String, String> headers = event.getHeaders();
        String schemaURL = headers.get(AVRO_SCHEMA_URL_HEADER);
        try {
            if (schemaURL != null) {
                return schemasFromURL.get(schemaURL);
            } else {
                String schemaLiteral = headers.get(AVRO_SCHEMA_LITERAL_HEADER);
                if (schemaLiteral == null) {
                    throw new NonRecoverableEventException("No schema in event headers." + " Headers must include either " + AVRO_SCHEMA_URL_HEADER + " or " + AVRO_SCHEMA_LITERAL_HEADER);
                }
                return schemasFromLiteral.get(schemaLiteral);
            }
        } catch (ExecutionException ex) {
            throw new EventDeliveryException("Cannot get schema", ex.getCause());
        } catch (UncheckedExecutionException ex) {
            throw new NonRecoverableEventException("Cannot parse schema", ex.getCause());
        }
    }

    public static clreplaced Builder implements EnreplacedyParser.Builder<GenericRecord> {

        @Override
        public EnreplacedyParser<GenericRecord> build(Schema datasetSchema, Context config) {
            return new AvroParser(datasetSchema);
        }
    }
}

16 View Complete Implementation : AvroKuduOperationsProducer.java
Copyright Apache License 2.0
Author : apache
/**
 * An Avro serializer that generates one operation per event by deserializing the event
 * body as an Avro record and mapping its fields to columns in a Kudu table.
 *
 * <p><strong>Avro Kudu Operations Producer configuration parameters</strong>
 * <table cellpadding=3 cellspacing=0 border=1
 *        summary="Avro Kudu Operations Producer configuration parameters">
 * <tr><th>Property Name</th>
 *   <th>Default</th>
 *   <th>Required?</th>
 *   <th>Description</th></tr>
 * <tr>
 *   <td>producer.operation</td>
 *   <td>upsert</td>
 *   <td>No</td>
 *   <td>The operation used to write events to Kudu.
 *   Supported operations are 'insert' and 'upsert'</td>
 * </tr>
 * <tr>
 *   <td>producer.schemaPath</td>
 *   <td></td>
 *   <td>No</td>
 *   <td>The location of the Avro schema file used to deserialize the Avro-encoded event bodies.
 *   It's used whenever an event does not include its own schema. If not specified, the
 *   schema must be specified on a per-event basis, either by url or as a literal.
 *   Schemas must be record type.</td>
 * </tr>
 * </table>
 */
public clreplaced AvroKuduOperationsProducer implements KuduOperationsProducer {

    public static final String OPERATION_PROP = "operation";

    public static final String SCHEMA_PROP = "schemaPath";

    public static final String DEFAULT_OPERATION = "upsert";

    public static final String SCHEMA_URL_HEADER = "flume.avro.schema.url";

    public static final String SCHEMA_LITERAL_HEADER = "flume.avro.schema.literal";

    private String operation = "";

    private GenericRecord reuse;

    private KuduTable table;

    private String defaultSchemaUrl;

    /**
     * The binary decoder to reuse for event parsing.
     */
    private BinaryDecoder decoder = null;

    /**
     * A cache of schemas retrieved by URL to avoid re-parsing the schema.
     */
    private static final LoadingCache<String, Schema> schemasFromURL = CacheBuilder.newBuilder().build(new CacheLoader<String, Schema>() {

        @Override
        public Schema load(String url) throws IOException {
            Schema.Parser parser = new Schema.Parser();
            InputStream is = null;
            try {
                FileSystem fs = FileSystem.get(URI.create(url), conf);
                if (url.toLowerCase(Locale.ENGLISH).startsWith("hdfs:/")) {
                    is = fs.open(new Path(url));
                } else {
                    is = new URL(url).openStream();
                }
                return parser.parse(is);
            } finally {
                if (is != null) {
                    is.close();
                }
            }
        }
    });

    /**
     * A cache of literal schemas to avoid re-parsing the schema.
     */
    private static final LoadingCache<String, Schema> schemasFromLiteral = CacheBuilder.newBuilder().build(new CacheLoader<String, Schema>() {

        @Override
        public Schema load(String literal) {
            Preconditions.checkNotNull(literal, "Schema literal cannot be null without a Schema URL");
            return new Schema.Parser().parse(literal);
        }
    });

    /**
     * A cache of DatumReaders per schema.
     */
    private static final LoadingCache<Schema, DatumReader<GenericRecord>> readers = CacheBuilder.newBuilder().build(new CacheLoader<Schema, DatumReader<GenericRecord>>() {

        @Override
        public DatumReader<GenericRecord> load(Schema schema) {
            return new GenericDatumReader<>(schema);
        }
    });

    private static final Configuration conf = new Configuration();

    public AvroKuduOperationsProducer() {
    }

    @Override
    public void configure(Context context) {
        this.operation = context.getString(OPERATION_PROP, DEFAULT_OPERATION);
        String schemaPath = context.getString(SCHEMA_PROP);
        if (schemaPath != null) {
            defaultSchemaUrl = schemaPath;
        }
    }

    @Override
    public void initialize(KuduTable table) {
        this.table = table;
    }

    @Override
    public List<Operation> getOperations(Event event) throws FlumeException {
        Schema schema = getSchema(event);
        DatumReader<GenericRecord> reader = readers.getUnchecked(schema);
        decoder = DecoderFactory.get().binaryDecoder(event.getBody(), decoder);
        try {
            reuse = reader.read(reuse, decoder);
        } catch (IOException e) {
            throw new FlumeException("Cannot deserialize event", e);
        }
        Operation op;
        switch(operation.toLowerCase(Locale.ENGLISH)) {
            case "upsert":
                op = table.newUpsert();
                break;
            case "insert":
                op = table.newInsert();
                break;
            default:
                throw new FlumeException(String.format("Unexpected operation %s", operation));
        }
        setupOp(op, reuse);
        return Collections.singletonList(op);
    }

    private void setupOp(Operation op, GenericRecord record) {
        PartialRow row = op.getRow();
        for (ColumnSchema col : table.getSchema().getColumns()) {
            String name = col.getName();
            Object value = record.get(name);
            if (value == null) {
                // Set null if nullable, otherwise leave unset for possible Kudu default.
                if (col.isNullable()) {
                    row.setNull(name);
                }
            } else {
                // Avro doesn't support 8- or 16-bit integer types, but we'll allow them to be preplaceded as
                // a larger type.
                try {
                    switch(col.getType()) {
                        case BOOL:
                            row.addBoolean(name, (boolean) value);
                            break;
                        case INT8:
                            row.addByte(name, (byte) value);
                            break;
                        case INT16:
                            row.addShort(name, (short) value);
                            break;
                        case INT32:
                            row.addInt(name, (int) value);
                            break;
                        // Fall through
                        case INT64:
                        case UNIXTIME_MICROS:
                            row.addLong(name, (long) value);
                            break;
                        case FLOAT:
                            row.addFloat(name, (float) value);
                            break;
                        case DOUBLE:
                            row.addDouble(name, (double) value);
                            break;
                        case STRING:
                            row.addString(name, value.toString());
                            break;
                        case BINARY:
                            row.addBinary(name, (byte[]) value);
                            break;
                        default:
                            throw new FlumeException(String.format("Unrecognized type %s for column %s", col.getType().toString(), name));
                    }
                } catch (ClreplacedCastException e) {
                    throw new FlumeException(String.format("Failed to coerce value for column '%s' to type %s", col.getName(), col.getType()), e);
                }
            }
        }
    }

    private Schema getSchema(Event event) throws FlumeException {
        Map<String, String> headers = event.getHeaders();
        String schemaUrl = headers.get(SCHEMA_URL_HEADER);
        String schemaLiteral = headers.get(SCHEMA_LITERAL_HEADER);
        try {
            if (schemaUrl != null) {
                return schemasFromURL.get(schemaUrl);
            } else if (schemaLiteral != null) {
                return schemasFromLiteral.get(schemaLiteral);
            } else if (defaultSchemaUrl != null) {
                return schemasFromURL.get(defaultSchemaUrl);
            } else {
                throw new FlumeException(String.format("No schema for event. " + "Specify configuration property '%s' or event header '%s'", SCHEMA_PROP, SCHEMA_URL_HEADER));
            }
        } catch (ExecutionException e) {
            throw new FlumeException("Cannot get schema", e);
        } catch (RuntimeException e) {
            throw new FlumeException("Cannot parse schema", e);
        }
    }

    @Override
    public void close() {
    }
}

16 View Complete Implementation : HoodieAvroUtils.java
Copyright Apache License 2.0
Author : apache
/**
 * Convert serialized bytes back into avro record.
 */
public static GenericRecord bytesToAvro(byte[] bytes, Schema schema) throws IOException {
    BinaryDecoder decoder = DecoderFactory.get().binaryDecoder(bytes, reuseDecoder.get());
    reuseDecoder.set(decoder);
    GenericDatumReader<GenericRecord> reader = new GenericDatumReader<GenericRecord>(schema);
    return reader.read(null, decoder);
}

16 View Complete Implementation : MapOutputValue.java
Copyright Apache License 2.0
Author : apache
public clreplaced MapOutputValue {

    private static BinaryDecoder binaryDecoder;

    private String schemaName;

    private GenericRecord record;

    private GenericDatumWriter<GenericRecord> WRITER;

    private EncoderFactory factory = EncoderFactory.get();

    private BinaryEncoder binaryEncoder;

    public MapOutputValue(String schemaName, GenericRecord record) {
        this.schemaName = schemaName;
        this.record = record;
    }

    public String getSchemaName() {
        return schemaName;
    }

    public GenericRecord getRecord() {
        return record;
    }

    public byte[] toBytes() throws IOException {
        ByteArrayOutputStream dataStream = new ByteArrayOutputStream();
        Schema schema = record.getSchema();
        if (WRITER == null) {
            WRITER = new GenericDatumWriter<GenericRecord>(schema);
        }
        binaryEncoder = factory.directBinaryEncoder(dataStream, binaryEncoder);
        WRITER.write(record, binaryEncoder);
        // serialize to bytes, we also need to know the schema name when we
        // process this record on the reducer since reducer gets the record from
        // multiple mappers. So we first write the schema/source name and then
        // write the serialized bytes
        ByteArrayOutputStream out = new ByteArrayOutputStream();
        DataOutputStream dos = new DataOutputStream(out);
        dos.writeInt(schema.getName().getBytes().length);
        dos.write(schema.getName().getBytes());
        byte[] dataBytes = dataStream.toByteArray();
        dos.writeInt(dataBytes.length);
        dos.write(dataBytes);
        return out.toByteArray();
    }

    public static MapOutputValue fromBytes(byte[] bytes, Map<String, Schema> schemaMap) throws IOException {
        DataInputStream dataInputStream = new DataInputStream(new ByteArrayInputStream(bytes));
        int length = dataInputStream.readInt();
        byte[] sourceNameBytes = new byte[length];
        dataInputStream.read(sourceNameBytes);
        String schemaName = new String(sourceNameBytes);
        int recordDataLength = dataInputStream.readInt();
        byte[] recordBytes = new byte[recordDataLength];
        dataInputStream.read(recordBytes);
        Schema schema = schemaMap.get(schemaName);
        GenericRecord record = new GenericData.Record(schema);
        binaryDecoder = DecoderFactory.get().binaryDecoder(recordBytes, binaryDecoder);
        GenericDatumReader<GenericRecord> gdr = new GenericDatumReader<GenericRecord>(schema);
        gdr.read(record, binaryDecoder);
        return new MapOutputValue(schemaName, record);
    }
}

16 View Complete Implementation : AvroReaderWithExplicitSchema.java
Copyright Apache License 2.0
Author : apache
public clreplaced AvroReaderWithExplicitSchema extends AvroRecordReader {

    private final InputStream in;

    private final RecordSchema recordSchema;

    private final DatumReader<GenericRecord> datumReader;

    private final BinaryDecoder decoder;

    private GenericRecord genericRecord;

    public AvroReaderWithExplicitSchema(final InputStream in, final RecordSchema recordSchema, final Schema avroSchema) {
        this.in = in;
        this.recordSchema = recordSchema;
        datumReader = new NonCachingDatumReader<>(avroSchema);
        decoder = DecoderFactory.get().binaryDecoder(in, null);
    }

    @Override
    public void close() throws IOException {
        in.close();
    }

    @Override
    protected GenericRecord nextAvroRecord() throws IOException {
        if (decoder.isEnd()) {
            return null;
        }
        try {
            genericRecord = datumReader.read(genericRecord, decoder);
        } catch (final EOFException eof) {
            return null;
        }
        return genericRecord;
    }

    @Override
    public RecordSchema getSchema() {
        return recordSchema;
    }
}

16 View Complete Implementation : AvroReader.java
Copyright Apache License 2.0
Author : apache
@Override
public T read(InputStream inputStream) {
    try {
        BinaryDecoder decoderFromCache = decoders.get();
        BinaryDecoder decoder = DecoderFactory.get().binaryDecoder(inputStream, decoderFromCache);
        if (decoderFromCache == null) {
            decoders.set(decoder);
        }
        return reader.read(null, DecoderFactory.get().binaryDecoder(inputStream, decoder));
    } catch (Exception e) {
        throw new SchemaSerializationException(e);
    } finally {
        try {
            inputStream.close();
        } catch (IOException e) {
            log.error("AvroReader close inputStream close error", e.getMessage());
        }
    }
}

16 View Complete Implementation : MessageDeserializerImpl.java
Copyright Apache License 2.0
Author : apache
/**
 * Deserialize messages of type TMessage from input decoder.
 * @param decoder An Avro BinaryDecoder instance that is reading the input stream containing the message.
 * @param observer An instance of the MultiObserver clreplaced that will process the message.
 * @param sequence A long value which contains the sequence number of the message in the input stream.
 * @throws IOException Read of input stream in decoder failed.
 * @throws IllegalAccessException Target method in observer is not accessible.
 * @throws InvocationTargetException Subclreplaced threw and exception.
 */
public void deserialize(final BinaryDecoder decoder, final MultiObserver observer, final long sequence) throws IOException, IllegalAccessException, InvocationTargetException {
    final TMessage message = messageReader.read(null, decoder);
    if (message != null) {
        observer.onNext(sequence, message);
    } else {
        throw new RuntimeException("Failed to deserialize message [" + msgMetaClreplaced.getSimpleName() + "]");
    }
}

16 View Complete Implementation : GenericSchemaDecoder.java
Copyright Apache License 2.0
Author : BriData
/**
 * unwrap the osource generic schema
 *
 * @param input data
 * @return List<GenericData>
 * @throws Exception
 */
public List<GenericData> unwrap(byte[] input) throws Exception {
    BinaryDecoder decoder = getBinaryDecoder(input);
    List<GenericData> list = new LinkedList<>();
    while (!decoder.isEnd()) {
        try {
            GenericRecord record = datumReader.read(null, decoder);
            GenericData schemaBean = new GenericData();
            Utf8 utf8 = (Utf8) record.get(GenericData.TABLE_NAME);
            schemaBean.setTableName(utf8.toString());
            schemaBean.setSchemaHash((Integer) record.get(GenericData.SCHEMA_HASH));
            ByteBuffer buffer = (ByteBuffer) record.get(GenericData.PAYLOAD);
            schemaBean.setPayload(buffer.array());
            list.add(schemaBean);
        } catch (Exception e) {
            throw e;
        }
    }
    logger.trace("message count:" + list.size());
    return list;
}

16 View Complete Implementation : AvroEventParser.java
Copyright Apache License 2.0
Author : flumebase
public clreplaced AvroEventParser extends EventParser {

    private static final Logger LOG = LoggerFactory.getLogger(AvroEventParser.clreplaced.getName());

    public static final String SCHEMA_PARAM = "schema";

    /**
     * Configuration parameters.
     */
    private Map<String, String> mParams;

    /**
     * Current event we're parsing.
     */
    private Event mEvent;

    /**
     * Schema for input events
     */
    private Schema mSchema;

    /**
     * Current event deserialized into a generic data record
     */
    private GenericData.Record mRecord;

    // true if the mEvent is deserialized into mRecord.
    private boolean mIsDecoded;

    // Avro parsing utility objects below.
    private DecoderFactory mDecoderFactory;

    private BinaryDecoder mDecoder;

    private GenericDatumReader<GenericData.Record> mDatumReader;

    /**
     * Creates a new AvroEventParser, with a string--string parameter map specified
     * by the user who created the stream we are parsing.
     */
    public AvroEventParser(Map<String, String> params) {
        mParams = params;
        // Initialize avro decoder.
        String schemaStr = mParams.get(SCHEMA_PARAM);
        if (null != schemaStr) {
            // If schemaStr is null, validate() will fail, so we won't
            // need these things that we can't initialize.
            try {
                mSchema = Schema.parse(schemaStr);
                mDecoderFactory = new DecoderFactory();
                mRecord = new GenericData.Record(mSchema);
                mDatumReader = new GenericDatumReader<GenericData.Record>(mSchema);
            } catch (RuntimeException re) {
            // Couldn't parse schema. Ok, we'll get this in the validate() method.
            }
        }
    }

    /**
     * {@inheritDoc}
     */
    @Override
    public void reset(Event e) {
        mEvent = e;
        mIsDecoded = false;
    }

    /**
     * {@inheritDoc}
     */
    @Override
    public Object getColumn(int colIdx, Type expectedType) throws IOException {
        if (!mIsDecoded) {
            // Now that we actually want a record value, decode the input bytes.
            mDecoder = mDecoderFactory.createBinaryDecoder(mEvent.getBody(), mDecoder);
            mRecord = mDatumReader.read(mRecord, mDecoder);
            mIsDecoded = true;
        }
        return avroToNative(mRecord.get(colIdx), expectedType);
    }

    @Override
    public boolean validate(StreamSymbol streamSym) {
        // Check that we have an incoming schema.
        String schemaStr = mParams.get(SCHEMA_PARAM);
        if (null == schemaStr) {
            LOG.error("The EventParser for this stream requires the '" + SCHEMA_PARAM + "' property to be set. Try recreating the stream as: " + "CREATE STREAM .. EVENT FORMAT 'avro' PROPERTIES ('" + SCHEMA_PARAM + "' = ...)");
            return false;
        } else {
            try {
                Schema.parse(schemaStr);
            } catch (RuntimeException re) {
                LOG.error("Couldn't parse specified schema for the stream: " + re);
                return false;
            }
            // Given a schema -- does it match the expected column types?
            // TODO -- can we induce the field defs from the schema?
            // we should be able to say something like: CREATE STREAM foo (auto) FROM SCHEMA '....'
            // (Currently no, since PRECISE values are held as strings in avro.)
            List<Schema.Field> schemaFields = null;
            try {
                schemaFields = mSchema.getFields();
            } catch (AvroRuntimeException are) {
                // This wasn't a record schema, it was a single field or something.
                LOG.error("Schemas for events must be of record type. Each column must " + "represent a field of the same name.");
                return false;
            }
            List<TypedField> columnFields = streamSym.getFields();
            for (int i = 0; i < columnFields.size(); i++) {
                TypedField col = columnFields.get(i);
                Type colType = col.getType();
                // Get the schema field that matches this column name.
                Schema.Field schemaField = null;
                for (Schema.Field testSchemaField : schemaFields) {
                    if (testSchemaField.name().equals(col.getUserAlias())) {
                        schemaField = testSchemaField;
                        break;
                    }
                }
                if (null == schemaField) {
                    // Can't find a field in the schema to match the current column.
                    LOG.error("The Avro schema does not contain a field with the same name " + "as column '" + col.getUserAlias() + "'.");
                    return false;
                }
                // Are the schemas compatible?
                if (!schemaField.schema().equals(colType.getAvroSchema())) {
                    boolean warned = false;
                    if (colType.isNullable() && colType.isPrimitive()) {
                        // A common error is that the rtsql type is nullable and the schema
                        // type isn't. Give a specific suggestion in this case.
                        Type nonNullVersion = Type.getPrimitive(colType.getPrimitiveTypeName());
                        if (schemaField.schema().equals(nonNullVersion.getAvroSchema())) {
                            LOG.error("Column " + col.getUserAlias() + " has type " + colType + ", but requires type " + nonNullVersion + " to match the Avro schema.");
                            warned = true;
                        }
                    }
                    if (!warned) {
                        // Give a generic error message.
                        LOG.error("Column " + col.getUserAlias() + " has type " + colType + " with avro schema " + colType.getAvroSchema() + " but the stream schema field " + schemaField.name() + " has " + " an incompatible Avro schema: " + schemaField.schema());
                    }
                    return false;
                }
            }
        }
        // Looks good!
        return true;
    }

    /**
     * @return a Java object in the native Java type that FlumeBase expects to
     * work with.
     * @param in the object returned from the Avro generic record.
     * @param expectedType the expected FlumeBase type of the returned object.
     * @see AvroOutputElementImpl.nativeToAvro()
     */
    private Object avroToNative(Object in, Type expectedType) {
        if (null == in) {
            return null;
        } else if (expectedType.getPrimitiveTypeName().equals(Type.TypeName.PRECISE)) {
            // Convert string input to precise type.
            PreciseType preciseType = PreciseType.toPreciseType(expectedType);
            return preciseType.parseStringInput((String) in);
        } else {
            // All other types map to themselves.
            return in;
        }
    }
}

16 View Complete Implementation : AvroResolver.java
Copyright Apache License 2.0
Author : greenplum-db
/**
 * Clreplaced AvroResolver handles deserialization of records that were serialized
 * using the AVRO serialization framework.
 */
public clreplaced AvroResolver extends BasePlugin implements Resolver {

    private static final String MAPKEY_DELIM = ":";

    private static final String RECORDKEY_DELIM = ":";

    private static final String COLLECTION_DELIM = ",";

    private GenericRecord avroRecord = null;

    private DatumReader<GenericRecord> reader = null;

    // member kept to enable reuse, and thus avoid repeated allocation
    private BinaryDecoder decoder = null;

    private List<Schema.Field> fields = null;

    private RecordkeyAdapter recordkeyAdapter = new RecordkeyAdapter();

    private String collectionDelim;

    private String mapkeyDelim;

    private String recordkeyDelim;

    private HcfsType hcfsType;

    private AvroUtilities avroUtilities;

    /**
     * Constructs a new instance of the AvroFileAccessor
     */
    public AvroResolver() {
        super();
        avroUtilities = AvroUtilities.getInstance();
    }

    /*
     * Initializes an AvroResolver. Initializes Avro data structure: the Avro
     * record - fields information and the Avro record reader. All Avro data is
     * build from the Avro schema, which is based on the *.avsc file that was
     * preplaceded by the user
     *
     * throws RuntimeException if Avro schema could not be retrieved or parsed
     */
    @Override
    public void initialize(RequestContext requestContext) {
        super.initialize(requestContext);
        hcfsType = HcfsType.getHcfsType(configuration, context);
        Schema schema = avroUtilities.obtainSchema(context, configuration, hcfsType);
        reader = new GenericDatumReader<>(schema);
        fields = schema.getFields();
        collectionDelim = context.getOption("COLLECTION_DELIM") == null ? COLLECTION_DELIM : context.getOption("COLLECTION_DELIM");
        mapkeyDelim = context.getOption("MAPKEY_DELIM") == null ? MAPKEY_DELIM : context.getOption("MAPKEY_DELIM");
        recordkeyDelim = context.getOption("RECORDKEY_DELIM") == null ? RECORDKEY_DELIM : context.getOption("RECORDKEY_DELIM");
    }

    /**
     * Returns a list of the fields of one record. Each record field is
     * represented by a OneField item. OneField item contains two fields: an
     * integer representing the field type and a Java Object representing the
     * field value.
     */
    @Override
    public List<OneField> getFields(OneRow row) throws Exception {
        avroRecord = makeAvroRecord(row.getData(), avroRecord);
        List<OneField> record = new LinkedList<>();
        int recordkeyIndex = (context.getRecordkeyColumn() == null) ? -1 : context.getRecordkeyColumn().columnIndex();
        int currentIndex = 0;
        for (Schema.Field field : fields) {
            /*
             * Add the record key if exists
             */
            if (currentIndex == recordkeyIndex) {
                currentIndex += recordkeyAdapter.appendRecordkeyField(record, context, row);
            }
            currentIndex += populateRecord(record, avroRecord.get(field.name()), field.schema());
        }
        return record;
    }

    /**
     * Constructs and sets the fields of a {@link OneRow}.
     *
     * @param record list of {@link OneField}
     * @return the constructed {@link OneRow}
     */
    @Override
    public OneRow setFields(List<OneField> record) {
        GenericRecord genericRecord = new GenericData.Record((Schema) context.getMetadata());
        int cnt = 0;
        for (OneField field : record) {
            // Avro does not seem to understand regular byte arrays
            if (field.val instanceof byte[]) {
                field.val = ByteBuffer.wrap((byte[]) field.val);
            }
            genericRecord.put(cnt++, field.val);
        }
        return new OneRow(null, genericRecord);
    }

    /**
     * The record can arrive from one out of two different sources: a sequence
     * file or an AVRO file. If it comes from an AVRO file, then it was already
     * obtained as a {@link GenericRecord} when it was fetched from the
     * file so in this case a cast is enough.
     * On the other hand, if the source is a sequence file, then the input
     * parameter obj hides a bytes [] buffer which is in fact one Avro record
     * serialized. Here, we build the Avro record from the flat buffer, using
     * the AVRO API. Then (for both cases) in the remaining functions we build a
     * {@code List<OneField>} record from the Avro record.
     *
     * @param obj         object holding an Avro record
     * @param reuseRecord Avro record to be reused to create new record from obj
     * @return Avro record
     * @throws IOException if creating the Avro record from byte array failed
     */
    GenericRecord makeAvroRecord(Object obj, GenericRecord reuseRecord) throws IOException {
        if (obj instanceof GenericRecord) {
            return (GenericRecord) obj;
        } else {
            byte[] bytes = ((BytesWritable) obj).getBytes();
            decoder = DecoderFactory.get().binaryDecoder(bytes, decoder);
            return reader.read(reuseRecord, decoder);
        }
    }

    /**
     * For a given field in the Avro record we extract its value and insert it
     * into the output {@code List<OneField>} record. An Avro field can be a
     * primitive type or an array type.
     *
     * @param record      list of fields to be populated
     * @param fieldValue  field value
     * @param fieldSchema field schema
     * @return the number of populated fields
     */
    int populateRecord(List<OneField> record, Object fieldValue, Schema fieldSchema) {
        Schema.Type fieldType = fieldSchema.getType();
        int ret = 0;
        switch(fieldType) {
            case ARRAY:
                if (fieldValue == null) {
                    return addOneFieldToRecord(record, DataType.TEXT, null);
                }
                List<OneField> listRecord = new LinkedList<>();
                ret = setArrayField(listRecord, fieldValue, fieldSchema);
                addOneFieldToRecord(record, DataType.TEXT, String.format("[%s]", HdfsUtilities.toString(listRecord, collectionDelim)));
                break;
            case MAP:
                if (fieldValue == null) {
                    return addOneFieldToRecord(record, DataType.TEXT, null);
                }
                List<OneField> mapRecord = new LinkedList<>();
                ret = setMapField(mapRecord, fieldValue, fieldSchema);
                addOneFieldToRecord(record, DataType.TEXT, String.format("{%s}", HdfsUtilities.toString(mapRecord, collectionDelim)));
                break;
            case RECORD:
                if (fieldValue == null) {
                    return addOneFieldToRecord(record, DataType.TEXT, null);
                }
                List<OneField> recRecord = new LinkedList<>();
                ret = setRecordField(recRecord, fieldValue, fieldSchema);
                addOneFieldToRecord(record, DataType.TEXT, String.format("{%s}", HdfsUtilities.toString(recRecord, collectionDelim)));
                break;
            case UNION:
                /*
                 * When an Avro field is actually a union, we resolve the type
                 * of the union element, and delegate the record update via
                 * recursion
                 */
                int unionIndex = GenericData.get().resolveUnion(fieldSchema, fieldValue);
                /*
                 * Retrieve index of the non null data type from the type array
                 * if value is null
                 */
                if (fieldValue == null) {
                    // exclusive or replacedignment
                    unionIndex ^= 1;
                }
                ret = populateRecord(record, fieldValue, fieldSchema.getTypes().get(unionIndex));
                break;
            case ENUM:
                ret = addOneFieldToRecord(record, DataType.TEXT, fieldValue);
                break;
            case INT:
                ret = addOneFieldToRecord(record, DataType.INTEGER, fieldValue);
                break;
            case DOUBLE:
                ret = addOneFieldToRecord(record, DataType.FLOAT8, fieldValue);
                break;
            case STRING:
                fieldValue = (fieldValue != null) ? fieldValue.toString() : null;
                ret = addOneFieldToRecord(record, DataType.TEXT, fieldValue);
                break;
            case FLOAT:
                ret = addOneFieldToRecord(record, DataType.REAL, fieldValue);
                break;
            case LONG:
                ret = addOneFieldToRecord(record, DataType.BIGINT, fieldValue);
                break;
            case BYTES:
            case FIXED:
                ret = addOneFieldToRecord(record, DataType.BYTEA, fieldValue);
                break;
            case BOOLEAN:
                ret = addOneFieldToRecord(record, DataType.BOOLEAN, fieldValue);
                break;
            default:
                break;
        }
        return ret;
    }

    /**
     * When an Avro field is actually a record, we iterate through each field
     * for each entry, the field name and value are added to a local record
     * {@code List<OneField>} complexRecord with the necessary delimiter we
     * create an object of type OneField and insert it into the output
     * {@code List<OneField>} record.
     *
     * @param record    list of fields to be populated
     * @param value     field value
     * @param recSchema record schema
     * @return number of populated fields
     */
    int setRecordField(List<OneField> record, Object value, Schema recSchema) {
        GenericRecord rec = ((GenericData.Record) value);
        Schema fieldKeySchema = Schema.create(Schema.Type.STRING);
        int currentIndex = 0;
        for (Schema.Field field : recSchema.getFields()) {
            Schema fieldSchema = field.schema();
            Object fieldValue = rec.get(field.name());
            List<OneField> complexRecord = new LinkedList<>();
            populateRecord(complexRecord, field.name(), fieldKeySchema);
            populateRecord(complexRecord, fieldValue, fieldSchema);
            addOneFieldToRecord(record, DataType.TEXT, HdfsUtilities.toString(complexRecord, recordkeyDelim));
            currentIndex++;
        }
        return currentIndex;
    }

    /**
     * When an Avro field is actually a map, we resolve the type of the map
     * value For each entry, the field name and value are added to a local
     * record we create an object of type OneField and insert it into the output
     * {@code List<OneField>} record.
     * <p>
     * Unchecked warning is suppressed to enable us to cast fieldValue to a Map.
     * (since the value schema has been identified to me of type map)
     *
     * @param record     list of fields to be populated
     * @param fieldValue field value
     * @param mapSchema  map schema
     * @return number of populated fields
     */
    @SuppressWarnings("unchecked")
    int setMapField(List<OneField> record, Object fieldValue, Schema mapSchema) {
        Schema keySchema = Schema.create(Schema.Type.STRING);
        Schema valueSchema = mapSchema.getValueType();
        Map<String, ?> avroMap = ((Map<String, ?>) fieldValue);
        for (Map.Entry<String, ?> entry : avroMap.entrySet()) {
            List<OneField> complexRecord = new LinkedList<>();
            populateRecord(complexRecord, entry.getKey(), keySchema);
            populateRecord(complexRecord, entry.getValue(), valueSchema);
            addOneFieldToRecord(record, DataType.TEXT, HdfsUtilities.toString(complexRecord, mapkeyDelim));
        }
        return avroMap.size();
    }

    /**
     * When an Avro field is actually an array, we resolve the type of the array
     * element, and for each element in the Avro array, we recursively invoke
     * the population of {@code List<OneField>} record.
     *
     * @param record      list of fields to be populated
     * @param fieldValue  field value
     * @param arraySchema array schema
     * @return number of populated fields
     */
    int setArrayField(List<OneField> record, Object fieldValue, Schema arraySchema) {
        Schema typeSchema = arraySchema.getElementType();
        GenericData.Array<?> array = (GenericData.Array<?>) fieldValue;
        int length = array.size();
        for (int i = 0; i < length; i++) {
            populateRecord(record, array.get(i), typeSchema);
        }
        return length;
    }

    /**
     * Creates the {@link OneField} object and adds it to the output {@code List<OneField>}
     * record. Strings and byte arrays are held inside special types in the Avro
     * record so we transfer them to standard types in order to enable their
     * insertion in the GPDBWritable instance.
     *
     * @param record           list of fields to be populated
     * @param gpdbWritableType field type
     * @param val              field value
     * @return 1 (number of populated fields)
     */
    int addOneFieldToRecord(List<OneField> record, DataType gpdbWritableType, Object val) {
        OneField oneField = new OneField();
        oneField.type = gpdbWritableType.getOID();
        switch(gpdbWritableType) {
            case BYTEA:
                if (val == null) {
                    oneField.val = null;
                } else if (val instanceof ByteBuffer) {
                    oneField.val = ((ByteBuffer) val).array();
                } else {
                    /**
                     * Entry point when the underlying bytearray is from a Fixed
                     * data
                     */
                    oneField.val = ((GenericData.Fixed) val).bytes();
                }
                break;
            default:
                oneField.val = val;
                break;
        }
        record.add(oneField);
        return 1;
    }
}

16 View Complete Implementation : MapOutputValue.java
Copyright Apache License 2.0
Author : Hanmourang
public clreplaced MapOutputValue {

    private static BinaryDecoder binaryDecoder;

    private String schemaName;

    private GenericRecord record;

    private GenericDatumWriter<GenericRecord> WRITER;

    private EncoderFactory factory = EncoderFactory.get();

    private BinaryEncoder binaryEncoder;

    public MapOutputValue(String schemaName, GenericRecord record) {
        this.schemaName = schemaName;
        this.record = record;
    }

    public String getSchemaName() {
        return schemaName;
    }

    public GenericRecord getRecord() {
        return record;
    }

    public byte[] toBytes() throws IOException {
        ByteArrayOutputStream dataStream = new ByteArrayOutputStream();
        Schema schema = record.getSchema();
        if (WRITER == null) {
            WRITER = new GenericDatumWriter<GenericRecord>(schema);
        }
        binaryEncoder = factory.directBinaryEncoder(dataStream, binaryEncoder);
        WRITER.write(record, binaryEncoder);
        // serialize to bytes, we also need to know the schema name when we
        // process this record on the reducer since reducer gets the record from
        // multiple mappers. So we first write the schema/source name and then
        // write the serialized bytes
        ByteArrayOutputStream out = new ByteArrayOutputStream();
        DataOutputStream dos = new DataOutputStream(out);
        System.out.println("schema name:" + schemaName + "  length:" + schemaName.getBytes().length);
        dos.writeInt(schema.getName().getBytes().length);
        dos.write(schema.getName().getBytes());
        byte[] dataBytes = dataStream.toByteArray();
        System.out.println("Data Buffer length:" + dataBytes.length);
        dos.writeInt(dataBytes.length);
        dos.write(dataBytes);
        return out.toByteArray();
    }

    public static MapOutputValue fromBytes(byte[] bytes, Map<String, Schema> schemaMap) throws IOException {
        DataInputStream dataInputStream = new DataInputStream(new ByteArrayInputStream(bytes));
        int length = dataInputStream.readInt();
        byte[] sourceNameBytes = new byte[length];
        dataInputStream.read(sourceNameBytes);
        String schemaName = new String(sourceNameBytes);
        int recordDataLength = dataInputStream.readInt();
        byte[] recordBytes = new byte[recordDataLength];
        dataInputStream.read(recordBytes);
        Schema schema = schemaMap.get(schemaName);
        GenericRecord record = new GenericData.Record(schema);
        binaryDecoder = DecoderFactory.get().binaryDecoder(recordBytes, binaryDecoder);
        GenericDatumReader<GenericRecord> gdr = new GenericDatumReader<GenericRecord>(schema);
        gdr.read(record, binaryDecoder);
        return new MapOutputValue(schemaName, record);
    }
}

16 View Complete Implementation : AvroCompatibilityHelperTest.java
Copyright BSD 2-Clause "Simplified" License
Author : linkedin
@Test
public void testBinaryDecoder() {
    ByteArrayInputStream is = new ByteArrayInputStream(new byte[0]);
    BinaryDecoder binaryDecoder = AvroCompatibilityHelper.newBinaryDecoder(is);
    replacedert.replacedertNotNull(binaryDecoder);
}