org.apache.avro.Schema.getName() - java examples

Here are the examples of the java api org.apache.avro.Schema.getName() taken from open source projects. By voting up you can indicate which examples are most useful and appropriate.

112 Examples 7

18 View Complete Implementation : ConfluentKafkaSchemaRegistry.java
Copyright Apache License 2.0
Author : apache
@Override
public Integer register(Schema schema) throws SchemaRegistryException {
    return register(schema, schema.getName());
}

18 View Complete Implementation : AvroStorageUtils.java
Copyright Apache License 2.0
Author : linkedin
/**
 * check whether a schema is a space holder (using field name)
 */
public static boolean isUDPartialRecordSchema(Schema s) {
    return s.getName().equals(NONAME);
}

18 View Complete Implementation : AvroTypeUtil.java
Copyright Apache License 2.0
Author : apache
/**
 * Convert a raw value to an Avro object to serialize in Avro type system, using the provided character set when necessary.
 * The counter-part method which reads an Avro object back to a raw value is {@link #normalizeValue(Object, Schema, String)}.
 */
public static Object convertToAvroObject(final Object rawValue, final Schema fieldSchema, final Charset charset) {
    return convertToAvroObject(rawValue, fieldSchema, fieldSchema.getName(), charset);
}

18 View Complete Implementation : AvroTypeUtil.java
Copyright Apache License 2.0
Author : apache
public static RecordSchema createSchema(final Schema avroSchema, final boolean includeText) {
    if (avroSchema == null) {
        throw new IllegalArgumentException("Avro Schema cannot be null");
    }
    SchemaIdentifier identifier = new StandardSchemaIdentifier.Builder().name(avroSchema.getName()).build();
    return createSchema(avroSchema, includeText ? avroSchema.toString() : null, identifier);
}

18 View Complete Implementation : SwaggerAvroModelConverter.java
Copyright Apache License 2.0
Author : cerner
protected String getName(Schema schema) {
    return schema.getName();
}

17 View Complete Implementation : ProducerGroup.java
Copyright Apache License 2.0
Author : otaviof
/**
 * Serialize a array of bytes into a generic Avro object, based on clreplaced defined Schema.
 *
 * @param payload array of bytes with submitted payload;
 * @param schema Avro schema;
 * @return GenericRecord representation;
 */
private GenericRecord convertToAvro(byte[] payload, Schema schema) throws ProducerGroupAvroConversionException {
    var input = new ByteArrayInputStream(payload);
    var output = new ByteArrayOutputStream();
    var enc = EncoderFactory.get().binaryEncoder(output, null);
    var reader = new GenericDatumReader<GenericRecord>(schema);
    var writer = new GenericDatumWriter<GenericRecord>(schema);
    log.info("Parsing request body against Schema '{}'", schema.getName());
    log.debug("Message body informed is: '{}'", new String(payload));
    try {
        var dec = DecoderFactory.get().jsonDecoder(schema, input);
        var record = reader.read(null, dec);
        writer.write(record, enc);
        enc.flush();
        return record;
    } catch (IOException | AvroTypeException e) {
        log.error("Error on parsing message body: '{}', caused by '{}'", e.getMessage(), e.getCause());
        throw new ProducerGroupAvroConversionException(e.getMessage());
    }
}

17 View Complete Implementation : RootSchemaUtils.java
Copyright Apache License 2.0
Author : Talend
/**
 * Checks whether given schema is Root schema
 * It checks names of this schema and its 2 child fields
 * Also it checks schema type
 * This schema name is supposed to be "Root"
 * Field name are supposed to be "Main" and "OutOfBand"
 * However this method doesn't check field schemas types
 *
 * @param avro record schema
 * @return true is given schema is Root; false otherwise
 */
public static boolean isRootSchema(final Schema schema) {
    if (schema == null) {
        return false;
    }
    String schemaName = schema.getName();
    if (!"Root".equals(schemaName)) {
        return false;
    }
    if (Type.RECORD != schema.getType()) {
        return false;
    }
    Field main = schema.getField(MAIN_FIELD_NAME);
    Field outOfBand = schema.getField(OUTOFBAND_FIELD_NAME);
    if (main == null || outOfBand == null) {
        return false;
    }
    return true;
}

17 View Complete Implementation : TestMapOutputValue.java
Copyright Apache License 2.0
Author : Hanmourang
@Test
public void testSerDeser() throws Exception {
    Schema schema1 = AvroTestUtil.createSchemaFor(input1SourceName, input1Dimensions, input1Metrics);
    Schema schema2 = AvroTestUtil.createSchemaFor(input2SourceName, input2Dimensions, input2Metrics);
    GenericRecord record1 = AvroTestUtil.generateDummyRecord(schema1, input1Dimensions, input1Metrics);
    // construct map output value and serialize
    MapOutputValue valueOrig = new MapOutputValue(schema1.getName(), record1);
    byte[] bytes = valueOrig.toBytes();
    // deserialize
    // create schema map
    Map<String, Schema> schemaMap = new HashMap<String, Schema>();
    schemaMap.put(schema1.getName(), schema1);
    schemaMap.put(schema2.getName(), schema2);
    MapOutputValue valueNew = MapOutputValue.fromBytes(bytes, schemaMap);
    replacedert.replacedertEquals(valueOrig.getSchemaName(), valueNew.getSchemaName());
    replacedert.replacedertEquals(valueOrig.getRecord(), valueNew.getRecord());
}

17 View Complete Implementation : CodeGenerator.java
Copyright BSD 2-Clause "Simplified" License
Author : linkedin
private void validateParsedSchemas(Map<String, SchemaDetails> parsedSchemas, Map<File, File> fileToParent) {
    for (Map.Entry<String, SchemaDetails> entry : parsedSchemas.entrySet()) {
        String fqcn = entry.getKey();
        SchemaDetails schemaDetails = entry.getValue();
        if (!schemaDetails.isTopLevel()) {
            continue;
        }
        Schema schema = schemaDetails.getSchema();
        File file = schemaDetails.getLocation();
        File root = fileToParent.get(file);
        if (validateSchemaNamespaceVsFilePath) {
            String namespace = schema.getNamespace();
            String relativePath;
            if (root == file) {
                relativePath = "";
            } else {
                relativePath = root.toPath().relativize(file.toPath().getParent()).toString().replaceAll(Pattern.quote(File.pathSeparator), ".");
            }
            if (namespace == null) {
                if (!relativePath.equals("")) {
                    throw new IllegalArgumentException("schema " + fqcn + " has no namespace yet is defined in " + file + " who's relative path to root is " + relativePath);
                }
            } else {
                if (!relativePath.equals(namespace)) {
                    throw new IllegalArgumentException("schema " + fqcn + " belongs to namespace " + namespace + " yet is defined in " + file + " who's relative path to root is " + relativePath);
                }
            }
        }
        if (validateSchemaNameVsFileName) {
            String name = schema.getName();
            String fileName = FilenameUtils.removeExtension(file.getName());
            if (!fileName.equals(name)) {
                throw new IllegalArgumentException("schema " + fqcn + " has name " + name + " yet is defined in a file called " + file.getName());
            }
        }
    }
}

16 View Complete Implementation : AvroSchemaUtil.java
Copyright Apache License 2.0
Author : Netflix
static Schema copyRecord(Schema record, List<Schema.Field> newFields, String newName) {
    Schema copy;
    if (newName != null) {
        copy = Schema.createRecord(newName, record.getDoc(), null, record.isError(), newFields);
        // the namespace is defaulted to the record's namespace if it is null, which causes renames
        // without the namespace to fail. using "" instead of null changes this behavior to match the
        // original schema.
        copy.addAlias(record.getName(), record.getNamespace() == null ? "" : record.getNamespace());
    } else {
        copy = Schema.createRecord(record.getName(), record.getDoc(), record.getNamespace(), record.isError(), newFields);
    }
    for (Map.Entry<String, Object> prop : record.getObjectProps().entrySet()) {
        copy.addProp(prop.getKey(), prop.getValue());
    }
    return copy;
}

16 View Complete Implementation : AvroUtils.java
Copyright Apache License 2.0
Author : apache
/**
 * Copies the input {@link org.apache.avro.Schema} but changes the schema name.
 * @param schema {@link org.apache.avro.Schema} to copy.
 * @param newName name for the copied {@link org.apache.avro.Schema}.
 * @return A {@link org.apache.avro.Schema} that is a copy of schema, but has the name newName.
 */
public static Schema switchName(Schema schema, String newName) {
    if (schema.getName().equals(newName)) {
        return schema;
    }
    Schema newSchema = Schema.createRecord(newName, schema.getDoc(), schema.getNamespace(), schema.isError());
    List<Field> fields = schema.getFields();
    Iterable<Field> fieldsNew = Iterables.transform(fields, new Function<Field, Field>() {

        @Override
        public Schema.Field apply(Field input) {
            // this should never happen but the API has marked input as Nullable
            if (null == input) {
                return null;
            }
            Field field = new Field(input.name(), input.schema(), input.doc(), input.defaultValue(), input.order());
            return field;
        }
    });
    newSchema.setFields(Lists.newArrayList(fieldsNew));
    return newSchema;
}

16 View Complete Implementation : QueryCassandraTest.java
Copyright Apache License 2.0
Author : apache
@Test
public void testCreateSchemaOneColumn() throws Exception {
    ResultSet rs = CreplacedandraQueryTestUtil.createMockResultSetOneColumn();
    Schema schema = QueryCreplacedandra.createSchema(rs);
    replacedertNotNull(schema);
    replacedertEquals(schema.getName(), "users");
}

16 View Complete Implementation : AvroUtilitiesTest.java
Copyright Apache License 2.0
Author : greenplum-db
/**
 * Helper method for testing schema
 *
 * @param schema
 */
private static void verifySchema(Schema schema, String name) {
    replacedertNotNull(schema);
    replacedertEquals(Schema.Type.RECORD, schema.getType());
    replacedertEquals(name, schema.getName());
    Map<String, String> fieldToType = new HashMap<String, String>() {

        {
            put("id", "long");
            put("username", "string");
            put("followers", "array");
        }
    };
    for (String key : fieldToType.keySet()) {
        replacedertEquals(fieldToType.get(key), schema.getField(key).schema().getType().getName());
    }
}

16 View Complete Implementation : TestJdbcCommon.java
Copyright Apache License 2.0
Author : apache
@Test
public void testCreateSchemaNoColumns() throws ClreplacedNotFoundException, SQLException {
    final ResultSet resultSet = mock(ResultSet.clreplaced);
    final ResultSetMetaData resultSetMetaData = mock(ResultSetMetaData.clreplaced);
    when(resultSet.getMetaData()).thenReturn(resultSetMetaData);
    when(resultSetMetaData.getColumnCount()).thenReturn(0);
    when(resultSetMetaData.getTableName(1)).thenThrow(SQLException.clreplaced);
    final Schema schema = JdbcCommon.createSchema(resultSet);
    replacedertNotNull(schema);
    // records name, should be result set first column table name
    // Notice! sql select may join data from different tables, other columns
    // may have different table names
    replacedertEquals("NiFi_ExecuteSQL_Record", schema.getName());
    replacedertNull(schema.getField("ID"));
}

16 View Complete Implementation : TestCSVSchemaInference.java
Copyright Apache License 2.0
Author : kite-sdk
@Test
public void testSchemaNamespace() throws Exception {
    InputStream stream = new ByteArrayInputStream(csvLines.getBytes("utf8"));
    Schema schema = CSVUtil.inferNullableSchema("com.example.TestRecord", stream, new CSVProperties.Builder().hasHeader().build());
    replacedert.replacedertEquals("Should use name", "TestRecord", schema.getName());
    replacedert.replacedertEquals("Should set namespace", "com.example", schema.getNamespace());
}

16 View Complete Implementation : SpecificAvroDao.java
Copyright Apache License 2.0
Author : kite-sdk
private static String getSchemaName(Schema schema) {
    if (schema.getType() == Schema.Type.UNION) {
        List<Schema> types = schema.getTypes();
        if (types.size() == 2) {
            if (types.get(0).getType() == Schema.Type.NULL) {
                return types.get(1).getName();
            } else if (types.get(1).getType() == Schema.Type.NULL) {
                return types.get(0).getName();
            }
        }
        throw new IllegalArgumentException("Unsupported union schema: " + schema);
    }
    return schema.getName();
}

15 View Complete Implementation : ConverterTest.java
Copyright Apache License 2.0
Author : elodina
@Test
public void rootRecord() {
    String xsd = "<xs:schema xmlns:xs='http://www.w3.org/2001/XMLSchema'>" + "   <xs:element name='root'>" + "     <xs:complexType>" + "       <xs:sequence>" + "         <xs:element name='i' type='xs:int'/>" + "         <xs:element name='s' type='xs:string'/>" + "         <xs:element name='d' type='xs:double'/>" + "       </xs:sequence>" + "     </xs:complexType>" + "   </xs:element>" + "</xs:schema>";
    Schema schema = Converter.createSchema(xsd);
    replacedertEquals(Schema.Type.RECORD, schema.getType());
    replacedertEquals("AnonType_root", schema.getName());
    replacedertEquals(3, schema.getFields().size());
    replacedertEquals(Schema.Type.INT, schema.getField("i").schema().getType());
    replacedertEquals(Schema.Type.STRING, schema.getField("s").schema().getType());
    replacedertEquals(Schema.Type.DOUBLE, schema.getField("d").schema().getType());
    String xml = "<root>" + "  <i>1</i>" + "  <s>s</s>" + "  <d>1.0</d>" + "</root>";
    GenericData.Record record = Converter.createDatum(schema, xml);
    replacedertEquals(1, record.get("i"));
    replacedertEquals("s", record.get("s"));
    replacedertEquals(1.0, record.get("d"));
}

15 View Complete Implementation : TestSchemaManager.java
Copyright Apache License 2.0
Author : kite-sdk
@Test
public void testUpdateSchema() throws IOException {
    SchemaManager manager = SchemaManager.create(getConfiguration(), testDirectory);
    manager.writeSchema(DatasetTestUtilities.USER_SCHEMA);
    Schema schema = manager.getNewestSchema();
    replacedert.replacedertEquals(DatasetTestUtilities.USER_SCHEMA, schema);
    // Create an updated schema and ensure it can be written.
    Schema updatedSchema = SchemaBuilder.record(schema.getName()).fields().requiredString("username").requiredString("email").optionalBoolean("extra_field").endRecord();
    manager.writeSchema(updatedSchema);
    replacedert.replacedertEquals(updatedSchema, manager.getNewestSchema());
}

15 View Complete Implementation : TestCopyCommandClusterNewField.java
Copyright Apache License 2.0
Author : kite-sdk
@Override
public Schema getEvolvedSchema(Schema original) {
    List<Schema.Field> fields = Lists.newArrayList();
    fields.add(new Schema.Field("new", Schema.createUnion(ImmutableList.of(Schema.create(Schema.Type.NULL), Schema.create(Schema.Type.STRING))), "New field", NullNode.getInstance()));
    for (Schema.Field field : original.getFields()) {
        fields.add(new Schema.Field(field.name(), field.schema(), field.doc(), field.defaultValue()));
    }
    Schema evolved = Schema.createRecord(original.getName(), original.getDoc(), original.getNamespace(), false);
    evolved.setFields(fields);
    return evolved;
}

15 View Complete Implementation : FastDeserializerGenerator.java
Copyright BSD 2-Clause "Simplified" License
Author : linkedin
private void processComplexType(JVar fieldSchemaVar, String name, Schema schema, Schema readerFieldSchema, JBlock methodBody, FieldAction action, BiConsumer<JBlock, JExpression> putExpressionIntoParent, Supplier<JExpression> reuseSupplier) {
    switch(schema.getType()) {
        case RECORD:
            processRecord(fieldSchemaVar, schema.getName(), schema, readerFieldSchema, methodBody, action, putExpressionIntoParent, reuseSupplier);
            break;
        case ARRAY:
            processArray(fieldSchemaVar, name, schema, readerFieldSchema, methodBody, action, putExpressionIntoParent, reuseSupplier);
            break;
        case MAP:
            processMap(fieldSchemaVar, name, schema, readerFieldSchema, methodBody, action, putExpressionIntoParent, reuseSupplier);
            break;
        case UNION:
            processUnion(fieldSchemaVar, name, schema, readerFieldSchema, methodBody, action, putExpressionIntoParent, reuseSupplier);
            break;
        default:
            throw new FastDeserializerGeneratorException("Incorrect complex type: " + action.getType());
    }
}

15 View Complete Implementation : PruneColumns.java
Copyright Apache License 2.0
Author : Netflix
private static Schema copyRecord(Schema record, List<Schema.Field> newFields) {
    Schema copy = Schema.createRecord(record.getName(), record.getDoc(), record.getNamespace(), record.isError(), newFields);
    for (Map.Entry<String, Object> prop : record.getObjectProps().entrySet()) {
        copy.addProp(prop.getKey(), prop.getValue());
    }
    return copy;
}

15 View Complete Implementation : FastDeserializerGenerator.java
Copyright Apache License 2.0
Author : RTBHOUSE
private void processComplexType(JVar fieldSchemaVar, String name, Schema schema, Schema readerFieldSchema, JBlock methodBody, FieldAction action, BiConsumer<JBlock, JExpression> putExpressionIntoParent) {
    switch(schema.getType()) {
        case RECORD:
            processRecord(fieldSchemaVar, schema.getName(), schema, readerFieldSchema, methodBody, action, putExpressionIntoParent);
            break;
        case ARRAY:
            processArray(fieldSchemaVar, name, schema, readerFieldSchema, methodBody, action, putExpressionIntoParent);
            break;
        case MAP:
            processMap(fieldSchemaVar, name, schema, readerFieldSchema, methodBody, action, putExpressionIntoParent);
            break;
        case UNION:
            processUnion(fieldSchemaVar, name, schema, readerFieldSchema, methodBody, action, putExpressionIntoParent);
            break;
        default:
            throw new FastDeserializerGeneratorException("Incorrect complex type: " + action.getType());
    }
}

15 View Complete Implementation : TestJdbcCommon.java
Copyright Apache License 2.0
Author : apache
@Test
public void testCreateSchema() throws ClreplacedNotFoundException, SQLException {
    final Statement st = con.createStatement();
    st.executeUpdate("insert into restaurants values (1, 'Irifunes', 'San Mateo')");
    st.executeUpdate("insert into restaurants values (2, 'Estradas', 'Daly City')");
    st.executeUpdate("insert into restaurants values (3, 'Prime Rib House', 'San Francisco')");
    final ResultSet resultSet = st.executeQuery("select * from restaurants");
    final Schema schema = JdbcCommon.createSchema(resultSet);
    replacedertNotNull(schema);
    // records name, should be result set first column table name
    // Notice! sql select may join data from different tables, other columns
    // may have different table names
    replacedertEquals("RESTAURANTS", schema.getName());
    replacedertNotNull(schema.getField("ID"));
    replacedertNotNull(schema.getField("NAME"));
    replacedertNotNull(schema.getField("CITY"));
    st.close();
// con.close();
}

14 View Complete Implementation : TestJdbcCommon.java
Copyright Apache License 2.0
Author : apache
@Test
public void testCreateSchemaNoTableName() throws ClreplacedNotFoundException, SQLException {
    final ResultSet resultSet = mock(ResultSet.clreplaced);
    final ResultSetMetaData resultSetMetaData = mock(ResultSetMetaData.clreplaced);
    when(resultSet.getMetaData()).thenReturn(resultSetMetaData);
    when(resultSetMetaData.getColumnCount()).thenReturn(1);
    when(resultSetMetaData.getTableName(1)).thenReturn("");
    when(resultSetMetaData.getColumnType(1)).thenReturn(Types.INTEGER);
    when(resultSetMetaData.getColumnName(1)).thenReturn("ID");
    final Schema schema = JdbcCommon.createSchema(resultSet);
    replacedertNotNull(schema);
    // records name, should be result set first column table name
    replacedertEquals("NiFi_ExecuteSQL_Record", schema.getName());
}

14 View Complete Implementation : TestCSVSchemaInference.java
Copyright Apache License 2.0
Author : kite-sdk
@Test
public void testSchemaInference() throws Exception {
    InputStream stream = new ByteArrayInputStream(csvLines.getBytes("utf8"));
    Schema schema = CSVUtil.inferSchema("TestRecord", stream, new CSVProperties.Builder().hasHeader().build());
    replacedert.replacedertEquals("Should use name", "TestRecord", schema.getName());
    replacedert.replacedertNull("Should not have namespace", schema.getNamespace());
    replacedert.replacedertNotNull(schema.getField("long"));
    replacedert.replacedertNotNull(schema.getField("float"));
    replacedert.replacedertNotNull(schema.getField("double"));
    replacedert.replacedertNotNull(schema.getField("double2"));
    replacedert.replacedertNotNull(schema.getField("string"));
    replacedert.replacedertNotNull(schema.getField("nullable_long"));
    replacedert.replacedertNotNull(schema.getField("nullable_string"));
    replacedert.replacedertEquals("Should infer a long", schema(Schema.Type.LONG), schema.getField("long").schema());
    replacedert.replacedertEquals("Should infer a float (ends in f)", schema(Schema.Type.FLOAT), schema.getField("float").schema());
    replacedert.replacedertEquals("Should infer a double (ends in d)", nullable(Schema.Type.DOUBLE), schema.getField("double").schema());
    replacedert.replacedertEquals("Should infer a double (decimal defaults to double)", nullable(Schema.Type.DOUBLE), schema.getField("double2").schema());
    replacedert.replacedertEquals("Should infer a non-null string (not numeric)", schema(Schema.Type.STRING), schema.getField("string").schema());
    replacedert.replacedertEquals("Should infer a nullable long (second line is a long)", nullable(Schema.Type.LONG), schema.getField("nullable_long").schema());
    replacedert.replacedertEquals("Should infer a nullable string (second is missing)", nullable(Schema.Type.STRING), schema.getField("nullable_string").schema());
}

14 View Complete Implementation : LoggingConsumer.java
Copyright Apache License 2.0
Author : linkedin
private void endSourceStats(Schema sourceSchema) {
    Formatter fmt = new Formatter(_perSourceMsgBuilder);
    fmt.format("%s=%d (%.3f ms) ", sourceSchema.getName(), _curSourceEvents, (System.nanoTime() - _curSourceStartTs) / NANOS_PER_MS);
    fmt.flush();
    fmt.close();
}

14 View Complete Implementation : SchemaGeneratorUtils.java
Copyright Apache License 2.0
Author : Talend
/**
 * Generate the Avro schema from a tree representation of the schema.
 *
 * @param tree Hashmap representing a tree generated by the method generateTree()
 * @param elementToGenerate the current part of the tree that will be generated.
 * @return
 */
public static Schema convertTreeToAvroSchema(Map<String, Set<Object>> tree, String elementToGenerate, Schema inputSchema) {
    List<Schema.Field> fieldList = new ArrayList<>();
    if (tree.containsKey(elementToGenerate)) {
        for (Object treeElement : tree.get(elementToGenerate)) {
            if (treeElement instanceof String) {
                // path element, generate the schema of the subtree then add it as a field.
                Schema subElementSchema = convertTreeToAvroSchema(tree, (String) treeElement, inputSchema);
                String elementName = (String) treeElement;
                if (elementName.contains(".")) {
                    elementName = StringUtils.substringAfterLast(elementName, ".");
                }
                fieldList.add(new Field(elementName, subElementSchema, "", ""));
            } else if (treeElement instanceof Field) {
                // field element, adding it to the field list.
                fieldList.add((Field) treeElement);
            } else {
                TalendRuntimeException.build(CommonErrorCodes.UNEXPECTED_ARGUMENT).setAndThrow("Should be only String or Field", treeElement.getClreplaced().toString());
            }
        }
    } else {
        if (!TREE_ROOT_DEFAULT_VALUE.equals(elementToGenerate)) {
            TalendRuntimeException.build(CommonErrorCodes.UNEXPECTED_ARGUMENT).setAndThrow(tree.keySet().toString(), elementToGenerate);
        }
    }
    try {
        if (inputSchema == null) {
            return Schema.createRecord(fieldList);
        } else if ("$".equals(elementToGenerate)) {
            return Schema.createRecord(inputSchema.getName(), inputSchema.getDoc(), inputSchema.getNamespace(), inputSchema.isError(), fieldList);
        } else if (retrieveFieldFromJsonPath(inputSchema, elementToGenerate) != null) {
            // If the field exist in the inputSchema, copy its schema
            Schema currentSchema = retrieveFieldFromJsonPath(inputSchema, elementToGenerate).schema();
            return Schema.createRecord(currentSchema.getName(), currentSchema.getDoc(), currentSchema.getNamespace(), currentSchema.isError(), fieldList);
        } else {
            return Schema.createRecord(fieldList);
        }
    } catch (AvroRuntimeException e) {
        // this will be throw if we are trying to get the name of an anonymous type
        return Schema.createRecord(fieldList);
    }
}

13 View Complete Implementation : AvroUtilsTest.java
Copyright Apache License 2.0
Author : apache
@Test
public void testSwitchName() {
    String originalName = "originalName";
    String newName = "newName";
    Schema schema = SchemaBuilder.record(originalName).fields().requiredDouble("double").optionalFloat("float").endRecord();
    Schema newSchema = AvroUtils.switchName(schema, newName);
    replacedert.replacedertEquals(newSchema.getName(), newName);
    for (Schema.Field field : newSchema.getFields()) {
        replacedert.replacedertEquals(field, schema.getField(field.name()));
    }
    replacedert.replacedertEquals(newName, AvroUtils.switchName(schema, newName).getName());
    replacedert.replacedertEquals(schema, AvroUtils.switchName(AvroUtils.switchName(schema, newName), schema.getName()));
}

13 View Complete Implementation : AbstractRealtimeRecordReader.java
Copyright Apache License 2.0
Author : apache
/**
 * Generate a reader schema off the provided writeSchema, to just project out the provided columns.
 */
public static Schema generateProjectionSchema(Schema writeSchema, Map<String, Field> schemaFieldsMap, List<String> fieldNames) {
    /**
     * Avro & Presto field names seems to be case sensitive (support fields differing only in case) whereas
     * Hive/Impala/SparkSQL(default) are case-insensitive. Spark allows this to be configurable using
     * spark.sql.caseSensitive=true
     *
     * For a RT table setup with no delta-files (for a latest file-slice) -> we translate parquet schema to Avro Here
     * the field-name case is dependent on parquet schema. Hive (1.x/2.x/CDH) translate column projections to
     * lower-cases
     */
    List<Schema.Field> projectedFields = new ArrayList<>();
    for (String fn : fieldNames) {
        Schema.Field field = schemaFieldsMap.get(fn.toLowerCase());
        if (field == null) {
            throw new HoodieException("Field " + fn + " not found in log schema. Query cannot proceed! " + "Derived Schema Fields: " + new ArrayList<>(schemaFieldsMap.keySet()));
        } else {
            projectedFields.add(new Schema.Field(field.name(), field.schema(), field.doc(), field.defaultValue()));
        }
    }
    Schema projectedSchema = Schema.createRecord(writeSchema.getName(), writeSchema.getDoc(), writeSchema.getNamespace(), writeSchema.isError());
    projectedSchema.setFields(projectedFields);
    return projectedSchema;
}

13 View Complete Implementation : AvroLoader.java
Copyright Apache License 2.0
Author : apache
/**
 * Builds the Meta Data of from the Avro Schema
 *
 * @return
 */
protected InstanceInformation getHeader() {
    String relation = schema.getName();
    attributes = new ArrayList<Attribute>();
    /**
     * By Definition, the returned list is in the order of their positions. *
     */
    List<Schema.Field> fields = schema.getFields();
    for (Field field : fields) {
        Schema attributeSchema = field.schema();
        /**
         * Currently SAMOA supports only NOMINAL & Numeric Types. *
         */
        if (attributeSchema.getType() == Schema.Type.ENUM) {
            List<String> attributeLabels = attributeSchema.getEnumSymbols();
            attributes.add(new Attribute(field.name(), attributeLabels));
        } else if (isNumeric(field))
            attributes.add(new Attribute(field.name()));
    }
    return new InstanceInformation(relation, attributes);
}

13 View Complete Implementation : TestJdbcCommon.java
Copyright Apache License 2.0
Author : apache
@Test
public void testCreateSchemaOnlyColumnLabel() throws ClreplacedNotFoundException, SQLException {
    final ResultSet resultSet = mock(ResultSet.clreplaced);
    final ResultSetMetaData resultSetMetaData = mock(ResultSetMetaData.clreplaced);
    when(resultSet.getMetaData()).thenReturn(resultSetMetaData);
    when(resultSetMetaData.getColumnCount()).thenReturn(2);
    when(resultSetMetaData.getTableName(1)).thenReturn("TEST");
    when(resultSetMetaData.getColumnType(1)).thenReturn(Types.INTEGER);
    when(resultSetMetaData.getColumnName(1)).thenReturn("");
    when(resultSetMetaData.getColumnLabel(1)).thenReturn("ID");
    when(resultSetMetaData.getColumnType(2)).thenReturn(Types.VARCHAR);
    when(resultSetMetaData.getColumnName(2)).thenReturn("VCHARC");
    when(resultSetMetaData.getColumnLabel(2)).thenReturn("NOT_VCHARC");
    final Schema schema = JdbcCommon.createSchema(resultSet);
    replacedertNotNull(schema);
    replacedertNotNull(schema.getField("ID"));
    replacedertNotNull(schema.getField("NOT_VCHARC"));
    // records name, should be result set first column table name
    replacedertEquals("TEST", schema.getName());
}

13 View Complete Implementation : ParquetReaderTest.java
Copyright Apache License 2.0
Author : apache
@Test
public void testSchema() throws IOException, InjectionException {
    final JavaConfigurationBuilder builder = Tang.Factory.getTang().newConfigurationBuilder();
    builder.bindNamedParameter(PathString.clreplaced, file.getAbsolutePath());
    final Configuration conf = builder.build();
    final Injector injector = Tang.Factory.getTang().newInjector(conf);
    final ParquetReader reader = injector.getInstance(ParquetReader.clreplaced);
    final Schema schema = reader.createAvroSchema();
    replacedert.replacedertEquals("User", schema.getName());
    replacedert.replacedertEquals(Schema.Type.RECORD, schema.getType());
}

13 View Complete Implementation : TestCSVSchemaInference.java
Copyright Apache License 2.0
Author : kite-sdk
@Test
public void testNullableSchemaInference() throws Exception {
    InputStream stream = new ByteArrayInputStream(csvLines.getBytes("utf8"));
    Schema schema = CSVUtil.inferNullableSchema("TestRecord", stream, new CSVProperties.Builder().hasHeader().build(), ImmutableSet.of("float"));
    replacedert.replacedertEquals("Should use name", "TestRecord", schema.getName());
    replacedert.replacedertNull("Should not have namespace", schema.getNamespace());
    replacedert.replacedertNotNull(schema.getField("long"));
    replacedert.replacedertNotNull(schema.getField("float"));
    replacedert.replacedertNotNull(schema.getField("double"));
    replacedert.replacedertNotNull(schema.getField("double2"));
    replacedert.replacedertNotNull(schema.getField("string"));
    replacedert.replacedertNotNull(schema.getField("nullable_long"));
    replacedert.replacedertNotNull(schema.getField("nullable_string"));
    replacedert.replacedertEquals("Should infer a long", nullable(Schema.Type.LONG), schema.getField("long").schema());
    replacedert.replacedertEquals("Should infer a non-null float (required, ends in f)", schema(Schema.Type.FLOAT), schema.getField("float").schema());
    replacedert.replacedertEquals("Should infer a double (ends in d)", nullable(Schema.Type.DOUBLE), schema.getField("double").schema());
    replacedert.replacedertEquals("Should infer a double (decimal defaults to double)", nullable(Schema.Type.DOUBLE), schema.getField("double2").schema());
    replacedert.replacedertEquals("Should infer a string (not numeric)", nullable(Schema.Type.STRING), schema.getField("string").schema());
    replacedert.replacedertEquals("Should infer a long (second line is a long)", nullable(Schema.Type.LONG), schema.getField("nullable_long").schema());
    replacedert.replacedertEquals("Should infer a nullable string (second is missing)", nullable(Schema.Type.STRING), schema.getField("nullable_string").schema());
}

13 View Complete Implementation : CassandraQueryFactory.java
Copyright Apache License 2.0
Author : apache
private static void populateFieldsToQuery(Schema schema, StringBuilder builder) throws Exception {
    switch(schema.getType()) {
        case INT:
            builder.append("int");
            break;
        case MAP:
            builder.append("map<text,");
            populateFieldsToQuery(schema.getValueType(), builder);
            builder.append(">");
            break;
        case ARRAY:
            builder.append("list<");
            populateFieldsToQuery(schema.getElementType(), builder);
            builder.append(">");
            break;
        case LONG:
            builder.append("bigint");
            break;
        case FLOAT:
            builder.append("float");
            break;
        case DOUBLE:
            builder.append("double");
            break;
        case BOOLEAN:
            builder.append("boolean");
            break;
        case BYTES:
            builder.append("blob");
            break;
        case RECORD:
            builder.append("frozen<").append(schema.getName()).append(">");
            break;
        case STRING:
        case FIXED:
        case ENUM:
            builder.append("text");
            break;
        case UNION:
            for (Schema unionElementSchema : schema.getTypes()) {
                if (unionElementSchema.getType().equals(Schema.Type.RECORD)) {
                    String recordName = unionElementSchema.getName();
                    if (!builder.toString().contains(recordName)) {
                        builder.append("frozen<").append(recordName).append(">");
                    } else {
                        LOG.warn("Same Field Type can't be mapped recursively. This is not supported with Creplacedandra UDT types, Please use byte dataType for recursive mapping.");
                        throw new Exception("Same Field Type has mapped recursively");
                    }
                    break;
                } else if (!unionElementSchema.getType().equals(Schema.Type.NULL)) {
                    populateFieldsToQuery(unionElementSchema, builder);
                    break;
                }
            }
            break;
    }
}

13 View Complete Implementation : MRCompactorAvroKeyDedupJobRunner.java
Copyright Apache License 2.0
Author : apache
/**
 * Returns a schema composed of all fields in topicSchema whose doc match "(?i).*primarykey".
 * If there's no such field, topicSchema itself will be returned.
 */
public static Schema getKeySchema(Schema topicSchema) {
    Preconditions.checkArgument(topicSchema.getType() == Schema.Type.RECORD);
    Optional<Schema> newSchema = getKeySchemaFromRecord(topicSchema);
    if (newSchema.isPresent()) {
        return newSchema.get();
    } else {
        LOG.warn(String.format("No field in the schema of %s is annotated as primarykey. Using all fields for deduping", topicSchema.getName()));
        return topicSchema;
    }
}

13 View Complete Implementation : MultiConverterTest.java
Copyright Apache License 2.0
Author : apache
private void checkConvertedAvroData(Schema schema, GenericRecord record) {
    replacedert.replacedertEquals(schema.getNamespace(), "example.avro");
    replacedert.replacedertEquals(schema.getType(), Schema.Type.RECORD);
    replacedert.replacedertEquals(schema.getName(), "User");
    replacedert.replacedertEquals(schema.getFields().size(), 3);
    Schema.Field nameField = schema.getField("name");
    replacedert.replacedertEquals(nameField.name(), "name");
    replacedert.replacedertEquals(nameField.schema().getType(), Schema.Type.STRING);
    Schema.Field favNumberField = schema.getField("favorite_number");
    replacedert.replacedertEquals(favNumberField.name(), "favorite_number");
    replacedert.replacedertEquals(favNumberField.schema().getType(), Schema.Type.INT);
    Schema.Field favColorField = schema.getField("favorite_color");
    replacedert.replacedertEquals(favColorField.name(), "favorite_color");
    replacedert.replacedertEquals(favColorField.schema().getType(), Schema.Type.STRING);
    replacedert.replacedertEquals(record.get("name"), "Alyssa");
    replacedert.replacedertEquals(record.get("favorite_number"), 256d);
    replacedert.replacedertEquals(record.get("favorite_color"), "yellow");
}

13 View Complete Implementation : AvroUtils.java
Copyright Apache License 2.0
Author : apache
private static Optional<Schema> removeUncomparableFieldsFromRecord(Schema record, Map<Schema, Optional<Schema>> processed) {
    Preconditions.checkArgument(record.getType() == Schema.Type.RECORD);
    Optional<Schema> result = processed.get(record);
    if (null != result) {
        return result;
    }
    List<Field> fields = Lists.newArrayList();
    for (Field field : record.getFields()) {
        Optional<Schema> newFieldSchema = removeUncomparableFields(field.schema(), processed);
        if (newFieldSchema.isPresent()) {
            fields.add(new Field(field.name(), newFieldSchema.get(), field.doc(), field.defaultValue()));
        }
    }
    Schema newSchema = Schema.createRecord(record.getName(), record.getDoc(), record.getNamespace(), false);
    newSchema.setFields(fields);
    result = Optional.of(newSchema);
    processed.put(record, result);
    return result;
}

12 View Complete Implementation : GobblinTrackingEventFlattenFilterConverter.java
Copyright Apache License 2.0
Author : apache
@Override
public Schema convertSchema(Schema inputSchema, WorkUnitState workUnit) throws SchemaConversionException {
    Preconditions.checkArgument(AvroUtils.checkReaderWriterCompatibility(gobblinTrackingEventSchema, inputSchema, true));
    Schema outputSchema = Schema.createRecord(ConfigUtils.getString(config, NEW_SCHEMA_NAME, inputSchema.getName()), inputSchema.getDoc(), inputSchema.getNamespace(), inputSchema.isError());
    outputSchema.setFields(newFields);
    return outputSchema;
}

12 View Complete Implementation : AvroUtils.java
Copyright Apache License 2.0
Author : apache
/**
 * Inner recursive method called by {@link #dropRecursiveFields(Schema)}
 * @param schemaEntry
 * @param parents
 * @param fieldsWithRecursion
 * @return the transformed Schema, null if schema is recursive w.r.t parent schema traversed so far
 */
private static Schema dropRecursive(SchemaEntry schemaEntry, List<SchemaEntry> parents, List<SchemaEntry> fieldsWithRecursion) {
    Schema schema = schemaEntry.schema;
    // ignore primitive fields
    switch(schema.getType()) {
        case UNION:
            {
                List<Schema> unionTypes = schema.getTypes();
                List<Schema> copiedUnionTypes = new ArrayList<Schema>();
                for (Schema unionSchema : unionTypes) {
                    SchemaEntry unionSchemaEntry = new SchemaEntry(schemaEntry.fieldName, unionSchema);
                    copiedUnionTypes.add(dropRecursive(unionSchemaEntry, parents, fieldsWithRecursion));
                }
                if (copiedUnionTypes.stream().anyMatch(x -> x == null)) {
                    // one or more types in the union are referring to a parent type (directly recursive),
                    // entire union must be dropped
                    return null;
                } else {
                    Schema copySchema = Schema.createUnion(copiedUnionTypes);
                    copyProperties(schema, copySchema);
                    return copySchema;
                }
            }
        case RECORD:
            {
                // check if the type of this schema matches any in the parents list
                if (parents.stream().anyMatch(parent -> parent.fullyQualifiedType().equals(schemaEntry.fullyQualifiedType()))) {
                    fieldsWithRecursion.add(schemaEntry);
                    return null;
                }
                List<SchemaEntry> newParents = new ArrayList<>(parents);
                newParents.add(schemaEntry);
                List<Schema.Field> copiedSchemaFields = new ArrayList<>();
                for (Schema.Field field : schema.getFields()) {
                    String fieldName = schemaEntry.fieldName != null ? schemaEntry.fieldName + "." + field.name() : field.name();
                    SchemaEntry fieldSchemaEntry = new SchemaEntry(fieldName, field.schema());
                    Schema copiedFieldSchema = dropRecursive(fieldSchemaEntry, newParents, fieldsWithRecursion);
                    if (copiedFieldSchema == null) {
                    } else {
                        Schema.Field copiedField = new Schema.Field(field.name(), copiedFieldSchema, field.doc(), field.defaultValue(), field.order());
                        copyFieldProperties(field, copiedField);
                        copiedSchemaFields.add(copiedField);
                    }
                }
                if (copiedSchemaFields.size() > 0) {
                    Schema copiedRecord = Schema.createRecord(schema.getName(), schema.getDoc(), schema.getNamespace(), schema.isError());
                    copiedRecord.setFields(copiedSchemaFields);
                    copyProperties(schema, copiedRecord);
                    return copiedRecord;
                } else {
                    return null;
                }
            }
        case ARRAY:
            {
                Schema itemSchema = schema.getElementType();
                SchemaEntry itemSchemaEntry = new SchemaEntry(schemaEntry.fieldName, itemSchema);
                Schema copiedItemSchema = dropRecursive(itemSchemaEntry, parents, fieldsWithRecursion);
                if (copiedItemSchema == null) {
                    return null;
                } else {
                    Schema copiedArraySchema = Schema.createArray(copiedItemSchema);
                    copyProperties(schema, copiedArraySchema);
                    return copiedArraySchema;
                }
            }
        case MAP:
            {
                Schema valueSchema = schema.getValueType();
                SchemaEntry valueSchemaEntry = new SchemaEntry(schemaEntry.fieldName, valueSchema);
                Schema copiedValueSchema = dropRecursive(valueSchemaEntry, parents, fieldsWithRecursion);
                if (copiedValueSchema == null) {
                    return null;
                } else {
                    Schema copiedMapSchema = Schema.createMap(copiedValueSchema);
                    copyProperties(schema, copiedMapSchema);
                    return copiedMapSchema;
                }
            }
        default:
            {
                return schema;
            }
    }
}

12 View Complete Implementation : AvroUtils.java
Copyright Apache License 2.0
Author : apache
/**
 * Decorate the {@link Schema} for a record with additional {@link Field}s.
 * @param inputSchema: must be a {@link Record} schema.
 * @return the decorated Schema. Fields are appended to the inputSchema.
 */
public static Schema decorateRecordSchema(Schema inputSchema, @Nonnull List<Field> fieldList) {
    Preconditions.checkState(inputSchema.getType().equals(Type.RECORD));
    List<Field> outputFields = deepCopySchemaFields(inputSchema);
    List<Field> newOutputFields = Stream.concat(outputFields.stream(), fieldList.stream()).collect(Collectors.toList());
    Schema outputSchema = Schema.createRecord(inputSchema.getName(), inputSchema.getDoc(), inputSchema.getNamespace(), inputSchema.isError());
    outputSchema.setFields(newOutputFields);
    copyProperties(inputSchema, outputSchema);
    return outputSchema;
}

12 View Complete Implementation : AvroUtilsTest.java
Copyright Apache License 2.0
Author : apache
@Test
public void overrideSchemaNameAndNamespaceTest() {
    String inputName = "input_name";
    String inputNamespace = "input_namespace";
    String outputName = "output_name";
    String outputNamespace = "output_namespace";
    Schema inputSchema = SchemaBuilder.record(inputName).namespace(inputNamespace).fields().name("integer1").type().intBuilder().endInt().noDefault().endRecord();
    Map<String, String> namespaceOverrideMap = new HashMap<>();
    namespaceOverrideMap.put(inputNamespace, outputNamespace);
    Schema newSchema = AvroUtils.overrideNameAndNamespace(inputSchema, outputName, Optional.of(namespaceOverrideMap));
    replacedert.replacedertEquals(newSchema.getName(), outputName);
    replacedert.replacedertEquals(newSchema.getNamespace(), outputNamespace);
}

12 View Complete Implementation : HoodieAvroUtils.java
Copyright Apache License 2.0
Author : apache
/**
 * Adds the Hoodie metadata fields to the given schema.
 */
public static Schema addMetadataFields(Schema schema) {
    List<Schema.Field> parentFields = new ArrayList<>();
    Schema.Field commitTimeField = new Schema.Field(HoodieRecord.COMMIT_TIME_METADATA_FIELD, METADATA_FIELD_SCHEMA, "", NullNode.getInstance());
    Schema.Field commitSeqnoField = new Schema.Field(HoodieRecord.COMMIT_SEQNO_METADATA_FIELD, METADATA_FIELD_SCHEMA, "", NullNode.getInstance());
    Schema.Field recordKeyField = new Schema.Field(HoodieRecord.RECORD_KEY_METADATA_FIELD, METADATA_FIELD_SCHEMA, "", NullNode.getInstance());
    Schema.Field parreplacedionPathField = new Schema.Field(HoodieRecord.PARreplacedION_PATH_METADATA_FIELD, METADATA_FIELD_SCHEMA, "", NullNode.getInstance());
    Schema.Field fileNameField = new Schema.Field(HoodieRecord.FILENAME_METADATA_FIELD, METADATA_FIELD_SCHEMA, "", NullNode.getInstance());
    parentFields.add(commitTimeField);
    parentFields.add(commitSeqnoField);
    parentFields.add(recordKeyField);
    parentFields.add(parreplacedionPathField);
    parentFields.add(fileNameField);
    for (Schema.Field field : schema.getFields()) {
        if (!isMetadataField(field.name())) {
            Schema.Field newField = new Schema.Field(field.name(), field.schema(), field.doc(), field.defaultValue());
            for (Map.Entry<String, JsonNode> prop : field.getJsonProps().entrySet()) {
                newField.addProp(prop.getKey(), prop.getValue());
            }
            parentFields.add(newField);
        }
    }
    Schema mergedSchema = Schema.createRecord(schema.getName(), schema.getDoc(), schema.getNamespace(), false);
    mergedSchema.setFields(parentFields);
    return mergedSchema;
}

12 View Complete Implementation : MapOutputValue.java
Copyright Apache License 2.0
Author : apache
public byte[] toBytes() throws IOException {
    ByteArrayOutputStream dataStream = new ByteArrayOutputStream();
    Schema schema = record.getSchema();
    if (WRITER == null) {
        WRITER = new GenericDatumWriter<GenericRecord>(schema);
    }
    binaryEncoder = factory.directBinaryEncoder(dataStream, binaryEncoder);
    WRITER.write(record, binaryEncoder);
    // serialize to bytes, we also need to know the schema name when we
    // process this record on the reducer since reducer gets the record from
    // multiple mappers. So we first write the schema/source name and then
    // write the serialized bytes
    ByteArrayOutputStream out = new ByteArrayOutputStream();
    DataOutputStream dos = new DataOutputStream(out);
    dos.writeInt(schema.getName().getBytes().length);
    dos.write(schema.getName().getBytes());
    byte[] dataBytes = dataStream.toByteArray();
    dos.writeInt(dataBytes.length);
    dos.write(dataBytes);
    return out.toByteArray();
}

12 View Complete Implementation : MapOutputValue.java
Copyright Apache License 2.0
Author : Hanmourang
public byte[] toBytes() throws IOException {
    ByteArrayOutputStream dataStream = new ByteArrayOutputStream();
    Schema schema = record.getSchema();
    if (WRITER == null) {
        WRITER = new GenericDatumWriter<GenericRecord>(schema);
    }
    binaryEncoder = factory.directBinaryEncoder(dataStream, binaryEncoder);
    WRITER.write(record, binaryEncoder);
    // serialize to bytes, we also need to know the schema name when we
    // process this record on the reducer since reducer gets the record from
    // multiple mappers. So we first write the schema/source name and then
    // write the serialized bytes
    ByteArrayOutputStream out = new ByteArrayOutputStream();
    DataOutputStream dos = new DataOutputStream(out);
    System.out.println("schema name:" + schemaName + "  length:" + schemaName.getBytes().length);
    dos.writeInt(schema.getName().getBytes().length);
    dos.write(schema.getName().getBytes());
    byte[] dataBytes = dataStream.toByteArray();
    System.out.println("Data Buffer length:" + dataBytes.length);
    dos.writeInt(dataBytes.length);
    dos.write(dataBytes);
    return out.toByteArray();
}

12 View Complete Implementation : NormalizeUtils.java
Copyright Apache License 2.0
Author : Talend
/**
 * Transform input schema to a new schema.
 *
 * The schema of the array field `pathToNormalize` will be modified to the schema of its fields.
 */
public static Schema transformSchema(Schema inputSchema, String[] pathToNormalize, int pathIterator) {
    List<Schema.Field> fieldList = new ArrayList<>();
    for (Schema.Field field : inputSchema.getFields()) {
        Schema unwrappedSchema = getUnwrappedSchema(field);
        if ((pathIterator < pathToNormalize.length) && (field.name().equals(pathToNormalize[pathIterator])) && (unwrappedSchema.getType().equals(Schema.Type.ARRAY))) {
            fieldList.add(new Schema.Field(field.name(), unwrappedSchema.getElementType(), field.doc(), field.defaultVal()));
        } else if (unwrappedSchema.getType().equals(Schema.Type.RECORD)) {
            if ((pathIterator < pathToNormalize.length) && (field.name().equals(pathToNormalize[pathIterator]))) {
                Schema subElementSchema = transformSchema(unwrappedSchema, pathToNormalize, ++pathIterator);
                fieldList.add(new Schema.Field(field.name(), subElementSchema, null, null));
            } else {
                // if we are outside of the pathToNormalize, set the pathIterator at something that cannot be used
                // again
                Schema subElementSchema = transformSchema(unwrappedSchema, pathToNormalize, pathToNormalize.length);
                fieldList.add(new Schema.Field(field.name(), subElementSchema, null, null));
            }
        } else {
            // element add it directly
            fieldList.add(new Schema.Field(field.name(), field.schema(), field.doc(), field.defaultVal()));
        }
    }
    return Schema.createRecord(inputSchema.getName(), inputSchema.getDoc(), inputSchema.getNamespace(), inputSchema.isError(), fieldList);
}

12 View Complete Implementation : SchemaGeneratorUtils.java
Copyright Apache License 2.0
Author : Talend
/**
 * Merge a KV-Schema into a single schema.
 *
 * For each level, the schema will contains the elements present in the keySchema first, then the ones present in
 * the valueSchema.
 *
 * @param keySchema an avro Schema
 * @param valueSchema an avro Schema
 * @return an avro Schema merging the two previous schema
 */
public static Schema mergeKeyValues(Schema keySchema, Schema valueSchema) {
    List<Schema.Field> fieldList = new ArrayList<>();
    for (Field field : keySchema.getFields()) {
        if (valueSchema.getField(field.name()) != null) {
            // element in both key and value => create sub element
            fieldList.add(new Field(field.name(), mergeKeyValues(field.schema(), valueSchema.getField(field.name()).schema()), "", ""));
        } else {
            // Element only present in the key
            fieldList.add(new Field(field.name(), field.schema(), field.doc(), field.defaultVal()));
        }
    }
    for (Field field : valueSchema.getFields()) {
        if (keySchema.getField(field.name()) == null) {
            // Element only present in the value
            fieldList.add(new Field(field.name(), field.schema(), field.doc(), field.defaultVal()));
        }
    }
    if (fieldList.size() > 0) {
        try {
            return Schema.createRecord(keySchema.getName(), keySchema.getDoc(), keySchema.getNamespace(), keySchema.isError(), fieldList);
        } catch (AvroRuntimeException e) {
            // this will be throw if we are trying to get the name of an anonymous type
            return Schema.createRecord(fieldList);
        }
    } else {
        return AvroUtils.createEmptySchema();
    }
}

11 View Complete Implementation : AvroHttpJoinConverter.java
Copyright Apache License 2.0
Author : apache
@Override
public Schema convertSchemaImpl(Schema inputSchema, WorkUnitState workUnitState) throws SchemaConversionException {
    if (inputSchema == null) {
        throw new SchemaConversionException("input schema is empty");
    }
    List<Schema.Field> fields = AvroUtils.deepCopySchemaFields(inputSchema);
    Schema.Field requestResponseField = new Schema.Field(HTTP_REQUEST_RESPONSE_FIELD, HttpRequestResponseRecord.getClreplacedSchema(), "http output schema contains request url and return result", null);
    fields.add(requestResponseField);
    Schema combinedSchema = Schema.createRecord(inputSchema.getName(), inputSchema.getDoc() + " (Http request and response are contained)", inputSchema.getNamespace(), false);
    combinedSchema.setFields(fields);
    return combinedSchema;
}

11 View Complete Implementation : EnvelopePayloadConverter.java
Copyright Apache License 2.0
Author : apache
@Override
public Schema convertSchema(Schema inputSchema, WorkUnitState workUnit) throws SchemaConversionException {
    List<Field> outputSchemaFields = new ArrayList<>();
    for (Field field : inputSchema.getFields()) {
        outputSchemaFields.add(convertFieldSchema(inputSchema, field, workUnit));
    }
    Schema outputSchema = Schema.createRecord(inputSchema.getName(), inputSchema.getDoc(), inputSchema.getNamespace(), inputSchema.isError());
    outputSchema.setFields(outputSchemaFields);
    return outputSchema;
}

11 View Complete Implementation : AvroUtils.java
Copyright Apache License 2.0
Author : apache
/**
 * Merge oldSchema and newSchame. Set a field default value to null, if this field exists in the old schema but not in the new schema.
 * @param oldSchema
 * @param newSchema
 * @return schema that contains all the fields in both old and new schema.
 */
public static Schema nullifyFieldsForSchemaMerge(Schema oldSchema, Schema newSchema) {
    if (oldSchema == null) {
        LOG.warn("No previous schema available, use the new schema instead.");
        return newSchema;
    }
    if (!(oldSchema.getType().equals(Type.RECORD) && newSchema.getType().equals(Type.RECORD))) {
        LOG.warn("Both previous schema and new schema need to be record type. Quit merging schema.");
        return newSchema;
    }
    List<Field> combinedFields = Lists.newArrayList();
    for (Field newFld : newSchema.getFields()) {
        combinedFields.add(new Field(newFld.name(), newFld.schema(), newFld.doc(), newFld.defaultValue()));
    }
    for (Field oldFld : oldSchema.getFields()) {
        if (newSchema.getField(oldFld.name()) == null) {
            List<Schema> union = Lists.newArrayList();
            Schema oldFldSchema = oldFld.schema();
            if (oldFldSchema.getType().equals(Type.UNION)) {
                union.add(Schema.create(Type.NULL));
                for (Schema itemInUion : oldFldSchema.getTypes()) {
                    if (!itemInUion.getType().equals(Type.NULL)) {
                        union.add(itemInUion);
                    }
                }
                Schema newFldSchema = Schema.createUnion(union);
                combinedFields.add(new Field(oldFld.name(), newFldSchema, oldFld.doc(), oldFld.defaultValue()));
            } else {
                union.add(Schema.create(Type.NULL));
                union.add(oldFldSchema);
                Schema newFldSchema = Schema.createUnion(union);
                combinedFields.add(new Field(oldFld.name(), newFldSchema, oldFld.doc(), oldFld.defaultValue()));
            }
        }
    }
    Schema mergedSchema = Schema.createRecord(newSchema.getName(), newSchema.getDoc(), newSchema.getNamespace(), newSchema.isError());
    mergedSchema.setFields(combinedFields);
    return mergedSchema;
}

11 View Complete Implementation : AvroUtilsTest.java
Copyright Apache License 2.0
Author : apache
@Test
public void testSwitchNamespace() {
    String originalNamespace = "originalNamespace";
    String originalName = "originalName";
    String newNamespace = "newNamespace";
    Schema schema = SchemaBuilder.builder(originalNamespace).record(originalName).fields().requiredDouble("double").optionalFloat("float").endRecord();
    Map<String, String> map = Maps.newHashMap();
    map.put(originalNamespace, newNamespace);
    Schema newSchema = AvroUtils.switchNamespace(schema, map);
    replacedert.replacedertEquals(newSchema.getNamespace(), newNamespace);
    replacedert.replacedertEquals(newSchema.getName(), originalName);
    for (Schema.Field field : newSchema.getFields()) {
        replacedert.replacedertEquals(field, schema.getField(field.name()));
    }
}