org.apache.avro.Schema.getFields() - java examples

Here are the examples of the java api org.apache.avro.Schema.getFields() taken from open source projects. By voting up you can indicate which examples are most useful and appropriate.

155 Examples 7

19 View Complete Implementation : FileDelimitedWriter.java
Copyright Apache License 2.0
Author : Talend
private String getRowString(IndexedRecord record) {
    StringBuilder sb = new StringBuilder();
    for (Schema.Field field : recordSchema.getFields()) {
        Object value = record.get(field.pos());
        if (value != null) {
            sb.append(value);
        }
        if (field.pos() != (recordSchema.getFields().size() - 1)) {
            sb.append(outputRuntime.fieldSeparator);
        }
    }
    sb.append(outputRuntime.rowSeparator);
    return sb.toString();
}

19 View Complete Implementation : NetSuiteOutputWriter.java
Copyright Apache License 2.0
Author : Talend
/**
 * Create record for outgoing {@code reject} flow.
 *
 * @param response write response
 * @param record indexed record which was submitted
 * @return result record
 */
private IndexedRecord createRejectRecord(NsWriteResponse<RefT> response, IndexedRecord record) {
    GenericData.Record targetRecord = new GenericData.Record(rejectSchema);
    for (Schema.Field field : schema.getFields()) {
        Schema.Field targetField = rejectSchema.getField(field.name());
        if (targetField != null) {
            Object value = record.get(field.pos());
            targetRecord.put(targetField.name(), value);
        }
    }
    String errorCode;
    String errorMessage;
    NsStatus status = response.getStatus();
    if (!status.getDetails().isEmpty()) {
        errorCode = status.getDetails().get(0).getCode();
        errorMessage = status.getDetails().get(0).getMessage();
    } else {
        errorCode = "GENERAL_ERROR";
        errorMessage = "Operation failed";
    }
    Schema.Field errorCodeField = NetSuiteDatasetRuntimeImpl.getNsFieldByName(rejectSchema, "errorCode");
    if (errorCodeField != null) {
        targetRecord.put(errorCodeField.pos(), errorCode);
    }
    Schema.Field errorMessageField = NetSuiteDatasetRuntimeImpl.getNsFieldByName(rejectSchema, "errorMessage");
    if (errorMessageField != null) {
        targetRecord.put(errorMessageField.pos(), errorMessage);
    }
    return targetRecord;
}

18 View Complete Implementation : CommonUtils.java
Copyright Apache License 2.0
Author : Talend
public static Schema.Field getField(Schema schema, String fieldName) {
    if (schema == null) {
        return null;
    }
    for (Schema.Field outField : schema.getFields()) {
        if (outField.name().equals(fieldName)) {
            return outField;
        }
    }
    return null;
}

18 View Complete Implementation : TSnowflakeRowProperties.java
Copyright Apache License 2.0
Author : Talend
private String generateSqlQuery(String tableName, Schema schema) {
    StringBuilder sql = new StringBuilder();
    sql.append("SELECT ");
    List<Schema.Field> fields = schema.getFields();
    boolean firstOne = true;
    for (Schema.Field field : fields) {
        if (firstOne) {
            firstOne = false;
        } else {
            sql.append(", ");
        }
        sql.append(tableName).append(".").append(field.name());
    }
    sql.append(" FROM ").append(tableName);
    return sql.toString();
}

18 View Complete Implementation : SalesforceSchemaTest.java
Copyright Apache License 2.0
Author : Talend
private void replacedertBulk(TSalesforceBulkExecProperties modelProps) throws Throwable {
    modelProps.module.main.schema.setValue(schema);
    getComponentService().afterProperty("schema", modelProps.module.main);
    Schema output = modelProps.schemaFlow.schema.getValue();
    replacedertThat(output.getType(), is(Schema.Type.RECORD));
    replacedertThat(output.getFields(), hreplacedize(5));
    replacedertThat(output.getFields().get(0).name(), is("col_1"));
    replacedertThat(output.getFields().get(0).schema().getType(), is(Schema.Type.STRING));
    replacedertThat(output.getFields().get(1).name(), is("col_2"));
    replacedertThat(output.getFields().get(1).schema().getType(), is(Schema.Type.INT));
    replacedertThat(output.getFields().get(2).name(), is("col_3"));
    replacedertThat(output.getFields().get(2).schema().getType(), is(Schema.Type.LONG));
    replacedertThat(output.getFields().get(2).getProp(SchemaConstants.TALEND_COLUMN_PATTERN), is("yyyy-MM-dd'T'HH:mm:ss'.000Z'"));
    replacedertThat(output.getFields().get(3).name(), is("salesforce_id"));
    replacedertThat(output.getFields().get(3).schema().getType(), is(Schema.Type.STRING));
    replacedertThat(output.getFields().get(4).name(), is("salesforce_created"));
    replacedertThat(output.getFields().get(4).schema().getType(), is(Schema.Type.STRING));
    Schema reject = modelProps.schemaReject.schema.getValue();
    replacedertThat(reject.getType(), is(Schema.Type.RECORD));
    replacedertThat(reject.getFields(), hreplacedize(4));
    replacedertThat(reject.getFields().get(0).name(), is("col_1"));
    replacedertThat(reject.getFields().get(0).schema().getType(), is(Schema.Type.STRING));
    replacedertThat(reject.getFields().get(1).name(), is("col_2"));
    replacedertThat(reject.getFields().get(1).schema().getType(), is(Schema.Type.INT));
    replacedertThat(reject.getFields().get(2).name(), is("col_3"));
    replacedertThat(reject.getFields().get(2).schema().getType(), is(Schema.Type.LONG));
    replacedertThat(reject.getFields().get(2).getProp(SchemaConstants.TALEND_COLUMN_PATTERN), is("yyyy-MM-dd'T'HH:mm:ss'.000Z'"));
    replacedertThat(reject.getFields().get(3).name(), is("error"));
    replacedertThat(reject.getFields().get(3).schema().getType(), is(Schema.Type.STRING));
}

18 View Complete Implementation : NetSuiteDatasetRuntimeImpl.java
Copyright Apache License 2.0
Author : Talend
/**
 * Find and return <code>schema field</code> by it's name.
 *
 * @param schema schema
 * @param fieldName name of field to be found
 * @return schema field or <code>null</code> if field was not found
 */
public static Schema.Field getNsFieldByName(Schema schema, String fieldName) {
    for (Schema.Field field : schema.getFields()) {
        String nsFieldName = getNsFieldName(field);
        if (fieldName.equals(nsFieldName)) {
            return field;
        }
    }
    return null;
}

18 View Complete Implementation : FileOutputDelimitedRuntime.java
Copyright Apache License 2.0
Author : Talend
// This schema should be schema of IndexRecord
public void writeHeader(Writer writer, Schema schema) throws IOException {
    if (props.includeHeader.getValue()) {
        // TODO support PARALLEL ? need recheck with code of javajet
        // If the target is stream, would write header directly
        if (props.targetIsStream.getValue() || (file != null && file.length() == 0) || (zipFile != null && zipFile.length() == 0)) {
            for (Schema.Field field : schema.getFields()) {
                writer.write(field.name());
                if (field.pos() != (schema.getFields().size() - 1)) {
                    writer.write(fieldSeparator);
                }
            }
            writer.write(rowSeparator);
        }
    }
}

18 View Complete Implementation : SalesforceBulkQueryReader.java
Copyright Apache License 2.0
Author : Talend
private String getQueryString() throws IOException {
    if (dataset.sourceType.getValue() == SourceType.MODULE_SELECTION) {
        final Schema schema = getSchema();
        if (schema == null) {
            throw new IllegalStateException("The schema must not be null");
        }
        StringBuilder sb = new StringBuilder();
        sb.append("select ");
        int count = 0;
        for (Schema.Field se : schema.getFields()) {
            if (count++ > 0) {
                sb.append(", ");
            }
            sb.append(se.name());
        }
        sb.append(" from ");
        sb.append(dataset.moduleName.getValue());
        if (!StringUtils.isEmpty(dataset.condition.getValue())) {
            sb.append(" where ");
            sb.append(dataset.condition.getValue());
        }
        if (limit > 0) {
            sb.append(" limit " + limit);
        }
        return sb.toString();
    } else {
        return dataset.query.getValue();
    }
}

18 View Complete Implementation : DefaultQueryGenerator.java
Copyright Apache License 2.0
Author : Talend
@Override
public String generateQuery() {
    Schema schema = setting.getSchema();
    List<Field> fields = schema.getFields();
    if (fields != null && !fields.isEmpty()) {
        final String tableNameWithDBAndSchema = getTableNameWithDBAndSchema(this.getDBName(), this.getDBSchemaName(), getDBTableName());
        String columnField = null;
        if (needFullName4Column()) {
            columnField = generateColumnFields(tableNameWithDBAndSchema);
        } else {
            columnField = generateColumnFields(getDBTableName());
        }
        StringBuffer sql = new StringBuffer(100);
        sql.append(JAVA_TEXT_FENCE);
        sql.append(SQL_SELECT);
        sql.append(SPACE);
        sql.append(columnField);
        sql.append(ENTER);
        sql.append(SQL_FROM);
        sql.append(SPACE);
        sql.append(tableNameWithDBAndSchema);
        sql.append(JAVA_TEXT_FENCE);
        return processResultSQL(sql.toString());
    }
    return EMPTY;
}

18 View Complete Implementation : MarketoUtils.java
Copyright Apache License 2.0
Author : Talend
public static List<String> getSchemaFields(Schema schema) {
    List<String> fieldNames = new ArrayList<>();
    for (Schema.Field f : schema.getFields()) {
        fieldNames.add(f.name());
    }
    return fieldNames;
}

17 View Complete Implementation : KafkaDatasetOtherDelimTestIT.java
Copyright Apache License 2.0
Author : Talend
@Test
public void getSchemaTest() {
    KafkaDatasetRuntime runtime = new KafkaDatasetRuntime();
    runtime.initialize(null, createDatasetCSV(createDatastore(), TOPIC_IN, KafkaDatasetProperties.FieldDelimiterType.OTHER, fieldDelimiter));
    Schema schema = runtime.getSchema();
    replacedertNotNull(schema);
    replacedertNotEquals(0, schema.getFields().size());
    runtime.initialize(null, createDatasetCSV(createDatastore(), "fake", KafkaDatasetProperties.FieldDelimiterType.OTHER, fieldDelimiter));
    schema = runtime.getSchema();
    replacedertNotNull(schema);
    replacedertEquals(0, schema.getFields().size());
}

17 View Complete Implementation : JDBCSQLBuilder.java
Copyright Apache License 2.0
Author : Talend
public String generateSQL4Insert(String tablename, Schema schema) {
    StringBuilder sb = new StringBuilder();
    sb.append("INSERT INTO ").append(getProtectedChar()).append(tablename).append(getProtectedChar()).append(" ");
    sb.append("(");
    List<Schema.Field> fields = schema.getFields();
    boolean firstOne = true;
    for (Schema.Field field : fields) {
        if (firstOne) {
            firstOne = false;
        } else {
            sb.append(",");
        }
        String dbColumnName = field.getProp(SchemaConstants.TALEND_COLUMN_DB_COLUMN_NAME);
        sb.append(dbColumnName);
    }
    sb.append(")");
    sb.append(" VALUES ");
    sb.append("(");
    firstOne = true;
    for (@SuppressWarnings("unused") Schema.Field field : fields) {
        if (firstOne) {
            firstOne = false;
        } else {
            sb.append(",");
        }
        sb.append("?");
    }
    sb.append(")");
    return sb.toString();
}

17 View Complete Implementation : MarketoColumnMappingsTable.java
Copyright Apache License 2.0
Author : Talend
public List<String> getMarketoColumns(Schema schema) {
    List<String> result = new ArrayList<>();
    Map<String, String> mappings = getInputedNameMappingsForMarketo();
    String marketoCol = null;
    String schemaCol = null;
    for (Field f : schema.getFields()) {
        marketoCol = mappings.get(f.name());
        if (StringUtils.isEmpty(marketoCol)) {
            schemaCol = f.getProp(SchemaConstants.TALEND_COLUMN_DB_COLUMN_NAME);
            if (!StringUtils.isEmpty(schemaCol)) {
                marketoCol = schemaCol;
            } else {
                marketoCol = f.name();
            }
        }
        result.add(marketoCol);
    }
    return result;
}

17 View Complete Implementation : TJDBCRowPropertiesTest.java
Copyright Apache License 2.0
Author : Talend
/**
 * Run the void updateOutputSchemas() method test.
 *
 * @throws Exception
 *
 * @generatedBy CodePro at 17-6-20 PM3:13
 */
@Test
public void testUpdateOutputSchemas() throws Exception {
    TJDBCRowProperties fixture = new TJDBCRowProperties("row");
    fixture.init();
    Schema main = SchemaBuilder.builder().record("schema").fields().endRecord();
    fixture.main.schema.setValue(main);
    fixture.updateOutputSchemas();
    Schema flow = fixture.schemaFlow.schema.getValue();
    Schema reject = fixture.schemaReject.schema.getValue();
    replacedertEquals(main, flow);
    replacedertEquals(2, reject.getFields().size());
}

17 View Complete Implementation : DBTestUtils.java
Copyright Apache License 2.0
Author : Talend
private static IndexedRecord copyValueFrom(IndexedRecord record) {
    Schema schema = record.getSchema();
    IndexedRecord result = new GenericData.Record(schema);
    List<Schema.Field> fields = schema.getFields();
    for (int i = 0; i < fields.size(); i++) {
        result.put(i, record.get(i));
    }
    return result;
}

17 View Complete Implementation : TFileInputDelimitedProperties.java
Copyright Apache License 2.0
Author : Talend
protected List<String> getFieldNames(Property schema) {
    Schema s = (Schema) schema.getValue();
    List<String> fieldNames = new ArrayList<>();
    for (Schema.Field f : s.getFields()) {
        fieldNames.add(f.name());
    }
    return fieldNames;
}

17 View Complete Implementation : TSalesforceOutputBulkProperties.java
Copyright Apache License 2.0
Author : Talend
protected List<String> getFieldNames(Property schema) {
    String sJson = schema.getStringValue();
    Schema s = new Schema.Parser().parse(sJson);
    List<String> fieldNames = new ArrayList<>();
    for (Schema.Field f : s.getFields()) {
        fieldNames.add(f.name());
    }
    return fieldNames;
}

17 View Complete Implementation : TFilterRowProperties.java
Copyright Apache License 2.0
Author : Talend
protected List<String> getFieldNames(Property<Schema> schema) {
    Schema s = schema.getValue();
    List<String> fieldNames = new ArrayList<>();
    for (Schema.Field f : s.getFields()) {
        fieldNames.add(f.name());
    }
    return fieldNames;
}

17 View Complete Implementation : NetSuiteDatasetRuntimeImpl.java
Copyright Apache License 2.0
Author : Talend
/**
 * Extend a schema with additional fields.
 *
 * @param sourceSchema source schema
 * @param newSchemaName name of new schema
 * @param fieldsToAdd fields to be added
 * @return new schema
 */
public static Schema extendSchema(Schema sourceSchema, String newSchemaName, List<Schema.Field> fieldsToAdd) {
    Schema newSchema = Schema.createRecord(newSchemaName, sourceSchema.getDoc(), sourceSchema.getNamespace(), sourceSchema.isError());
    List<Schema.Field> copyFieldList = new ArrayList<>();
    for (Schema.Field se : sourceSchema.getFields()) {
        Schema.Field field = copyField(se);
        copyFieldList.add(field);
    }
    copyFieldList.addAll(fieldsToAdd);
    newSchema.setFields(copyFieldList);
    for (Map.Entry<String, Object> entry : sourceSchema.getObjectProps().entrySet()) {
        newSchema.addProp(entry.getKey(), entry.getValue());
    }
    return newSchema;
}

17 View Complete Implementation : TMarketoListOperationPropertiesTest.java
Copyright Apache License 2.0
Author : Talend
@Test
public void testGetSOAPSchemaMain() throws Exception {
    Schema s = MarketoConstants.getListOperationSOAPSchema();
    replacedertEquals(4, s.getFields().size());
}

17 View Complete Implementation : TSnowflakeOutputProperties.java
Copyright Apache License 2.0
Author : Talend
protected List<String> getFieldNames(Property<?> schema) {
    Schema s = (Schema) schema.getValue();
    List<String> fieldNames = new ArrayList<>();
    for (Schema.Field f : s.getFields()) {
        fieldNames.add(f.name());
    }
    return fieldNames;
}

17 View Complete Implementation : TSalesforceOutputProperties.java
Copyright Apache License 2.0
Author : Talend
private List<Schema.Field> cloneFields(Schema metadataSchema) {
    List<Schema.Field> copyFieldList = new ArrayList<>();
    for (Schema.Field se : metadataSchema.getFields()) {
        Schema.Field field = new Schema.Field(se.name(), se.schema(), se.doc(), se.defaultVal(), se.order());
        field.getObjectProps().putAll(se.getObjectProps());
        for (Map.Entry<String, Object> entry : se.getObjectProps().entrySet()) {
            field.addProp(entry.getKey(), entry.getValue());
        }
        copyFieldList.add(field);
    }
    return copyFieldList;
}

17 View Complete Implementation : KafkaDatasetTestIT.java
Copyright Apache License 2.0
Author : Talend
@Test
public void getSchemaTest() {
    KafkaDatasetRuntime runtime = new KafkaDatasetRuntime();
    runtime.initialize(null, createDatasetCSV(createDatastore(), TOPIC_IN, KafkaDatasetProperties.FieldDelimiterType.SEMICOLON, ";"));
    Schema schema = runtime.getSchema();
    replacedertNotNull(schema);
    replacedertNotEquals(0, schema.getFields().size());
    runtime.initialize(null, createDatasetCSV(createDatastore(), "fake", KafkaDatasetProperties.FieldDelimiterType.SEMICOLON, ";"));
    schema = runtime.getSchema();
    replacedertNotNull(schema);
    replacedertEquals(0, schema.getFields().size());
}

17 View Complete Implementation : TJDBCOutputPropertiesTest.java
Copyright Apache License 2.0
Author : Talend
/**
 * Run the void updateOutputSchemas() method test.
 *
 * @throws Exception
 *
 * @generatedBy CodePro at 17-6-20 PM3:13
 */
@Test
public void testUpdateOutputSchemas() throws Exception {
    TJDBCOutputProperties fixture = new TJDBCOutputProperties("output");
    fixture.init();
    Schema main = SchemaBuilder.builder().record("schema").fields().endRecord();
    fixture.main.schema.setValue(main);
    fixture.updateOutputSchemas();
    Schema flow = fixture.schemaFlow.schema.getValue();
    Schema reject = fixture.schemaReject.schema.getValue();
    replacedertEquals(main, flow);
    replacedertEquals(2, reject.getFields().size());
}

17 View Complete Implementation : NormalizeUtils.java
Copyright Apache License 2.0
Author : Talend
/**
 * Transform input schema to a new schema.
 *
 * The schema of the array field `pathToNormalize` will be modified to the schema of its fields.
 */
public static Schema transformSchema(Schema inputSchema, String[] pathToNormalize, int pathIterator) {
    List<Schema.Field> fieldList = new ArrayList<>();
    for (Schema.Field field : inputSchema.getFields()) {
        Schema unwrappedSchema = getUnwrappedSchema(field);
        if ((pathIterator < pathToNormalize.length) && (field.name().equals(pathToNormalize[pathIterator])) && (unwrappedSchema.getType().equals(Schema.Type.ARRAY))) {
            fieldList.add(new Schema.Field(field.name(), unwrappedSchema.getElementType(), field.doc(), field.defaultVal()));
        } else if (unwrappedSchema.getType().equals(Schema.Type.RECORD)) {
            if ((pathIterator < pathToNormalize.length) && (field.name().equals(pathToNormalize[pathIterator]))) {
                Schema subElementSchema = transformSchema(unwrappedSchema, pathToNormalize, ++pathIterator);
                fieldList.add(new Schema.Field(field.name(), subElementSchema, null, null));
            } else {
                // if we are outside of the pathToNormalize, set the pathIterator at something that cannot be used
                // again
                Schema subElementSchema = transformSchema(unwrappedSchema, pathToNormalize, pathToNormalize.length);
                fieldList.add(new Schema.Field(field.name(), subElementSchema, null, null));
            }
        } else {
            // element add it directly
            fieldList.add(new Schema.Field(field.name(), field.schema(), field.doc(), field.defaultVal()));
        }
    }
    return Schema.createRecord(inputSchema.getName(), inputSchema.getDoc(), inputSchema.getNamespace(), inputSchema.isError(), fieldList);
}

17 View Complete Implementation : BigQueryBaseIndexedRecordConverter.java
Copyright Apache License 2.0
Author : Talend
/**
 * Construct all field's converter for read or write Ensure to do that before read and write operation
 */
protected void initFieldConverters() {
    List<Schema.Field> fields = schema.getFields();
    fieldConverters = new HashMap<>();
    for (Schema.Field field : fields) {
        fieldConverters.put(field.name(), getConverter(field.schema()));
    }
}

17 View Complete Implementation : JDBCSQLBuilder.java
Copyright Apache License 2.0
Author : Talend
public String generateSQL4SelectTable(String tablename, Schema schema) {
    StringBuilder sql = new StringBuilder();
    sql.append("SELECT ");
    List<Schema.Field> fields = schema.getFields();
    boolean firstOne = true;
    for (Schema.Field field : fields) {
        if (firstOne) {
            firstOne = false;
        } else {
            sql.append(", ");
        }
        String dbColumnName = field.getProp(SchemaConstants.TALEND_COLUMN_DB_COLUMN_NAME);
        sql.append(tablename).append(".").append(dbColumnName);
    }
    sql.append(" FROM ").append(getProtectedChar()).append(tablename).append(getProtectedChar());
    return sql.toString();
}

17 View Complete Implementation : BulkFileWriter.java
Copyright Apache License 2.0
Author : Talend
public String[] getHeaders(Schema schema) {
    List<String> headers = new ArrayList<String>();
    for (Schema.Field f : schema.getFields()) {
        headers.add(f.name());
    }
    return headers.toArray(new String[headers.size()]);
}

17 View Complete Implementation : NetSuiteOutputWriter.java
Copyright Apache License 2.0
Author : Talend
/**
 * Create record for outgoing {@code success} flow.
 *
 * @param response write response
 * @param record indexed record which was written
 * @return result record
 */
private IndexedRecord createSuccessRecord(NsWriteResponse<RefT> response, IndexedRecord record) {
    NsRef ref = NsRef.fromNativeRef(response.getRef());
    GenericData.Record targetRecord = new GenericData.Record(flowSchema);
    for (Schema.Field field : schema.getFields()) {
        Schema.Field targetField = flowSchema.getField(field.name());
        if (targetField != null) {
            Object value = record.get(field.pos());
            targetRecord.put(targetField.name(), value);
        }
    }
    Schema.Field internalIdField = NetSuiteDatasetRuntimeImpl.getNsFieldByName(flowSchema, "internalId");
    if (internalIdField != null && targetRecord.get(internalIdField.pos()) == null) {
        targetRecord.put(internalIdField.pos(), ref.getInternalId());
    }
    Schema.Field externalIdField = NetSuiteDatasetRuntimeImpl.getNsFieldByName(flowSchema, "externalId");
    if (externalIdField != null && targetRecord.get(externalIdField.pos()) == null) {
        targetRecord.put(externalIdField.pos(), ref.getExternalId());
    }
    if (ref.getRefType() == RefType.CUSTOMIZATION_REF) {
        Schema.Field scriptIdField = NetSuiteDatasetRuntimeImpl.getNsFieldByName(flowSchema, "scriptId");
        if (scriptIdField != null && targetRecord.get(scriptIdField.pos()) == null) {
            targetRecord.put(scriptIdField.pos(), ref.getScriptId());
        }
    }
    return targetRecord;
}

17 View Complete Implementation : MarketoInputWriter.java
Copyright Apache License 2.0
Author : Talend
public void adaptSchemaToDynamic() throws IOException {
    Schema design = this.properties.schemaInput.schema.getValue();
    if (!isDynamic) {
        return;
    }
    try {
        Schema runtimeSchema;
        runtimeSchema = sink.getDynamicSchema("", design);
        // preserve mappings to re-apply them after
        Map<String, String> mappings = properties.mappingInput.getNameMappingsForMarketo();
        List<String> columnNames = new ArrayList<>();
        List<String> mktoNames = new ArrayList<>();
        for (Field f : runtimeSchema.getFields()) {
            columnNames.add(f.name());
            if (mappings.get(f.name()) != null) {
                mktoNames.add(mappings.get(f.name()));
            } else {
                mktoNames.add("");
            }
        }
        properties.mappingInput.columnName.setValue(columnNames);
        properties.mappingInput.marketoColumnName.setValue(mktoNames);
        properties.schemaInput.schema.setValue(runtimeSchema);
    } catch (IOException e) {
        LOG.error(e.getMessage());
        throw e;
    }
}

17 View Complete Implementation : OsgiSalesforceEsbTestIT.java
Copyright Apache License 2.0
Author : Talend
@Test
public void testStaticGetSchema() throws IOException {
    SalesforceConnectionProperties scp = setupProps(null, !ADD_QUOTES);
    Schema schema = SalesforceSourceOrSink.getSchema(null, scp, EXISTING_MODULE_NAME);
    replacedertNotNull(schema);
    replacedertThat(schema.getFields(), hreplacedize(greaterThan(10)));
// replacedertTrue(schema.getRoot().getChildren().size() > 10);
}

17 View Complete Implementation : NormalizeUtils.java
Copyright Apache License 2.0
Author : Talend
/**
 * Generate a new Index Record which is the duplicated of the input record.
 *
 * @return the new record
 */
public static GenericRecord duplicateRecord(IndexedRecord inputRecord, Schema inputSchema, Schema outputSchema) {
    GenericRecordBuilder outputRecord = new GenericRecordBuilder(outputSchema);
    for (Schema.Field field : inputSchema.getFields()) {
        // The column was existing on the input record, we forward it to the output record.
        Object inputValue = inputRecord.get(inputSchema.getField(field.name()).pos());
        // The current column can be a Record (an hierarchical sub-object), a list or directly a value.
        // If we are on a record, we need to recursively duplicate the element
        // If we are on a list, we need to duplicate each element of the list
        // if we are on a object, we duplicate it to the output.
        if (inputValue instanceof GenericData.Record) {
            // The sub-schema at this level is a union of "empty" and a record,
            // so we need to get the true sub-schema
            Schema inputChildSchema = getChildSchemaAsRecord(inputSchema, field);
            Schema outputChildSchema = getChildSchemaAsRecord(outputSchema, field);
            Object childRecord = duplicateRecord((IndexedRecord) inputValue, inputChildSchema, outputChildSchema);
            outputRecord.set(field.name(), childRecord);
        } else if (inputValue instanceof List) {
            // We are on a list, duplicate each sub element
            List<Object> outputElements = new ArrayList<>();
            for (Object element : (List<Object>) inputValue) {
                Schema inputChildSchema = getChildSchemaOfListAsRecord(inputSchema, field);
                Schema outputChildSchema = getChildSchemaOfListAsRecord(outputSchema, field);
                outputElements.add(duplicateRecord((IndexedRecord) element, inputChildSchema, outputChildSchema));
            }
            outputRecord.set(field.name(), outputElements);
        } else {
            // We are on a raw type, use it directly
            outputRecord.set(field.name(), inputValue);
        }
    }
    return outputRecord.build();
}

16 View Complete Implementation : SchemaCustomMetaDataSource.java
Copyright Apache License 2.0
Author : Talend
private Map<String, CustomFieldDesc> loadCustomFieldDescMap() {
    Map<String, CustomFieldDesc> customFieldDescMap = new HashMap<>();
    for (Schema.Field field : schema.getFields()) {
        CustomFieldDesc customFieldDesc = NetSuiteDatasetRuntimeImpl.readCustomField(field);
        if (customFieldDesc != null) {
            customFieldDescMap.put(customFieldDesc.getName(), customFieldDesc);
        }
    }
    return customFieldDescMap;
}

16 View Complete Implementation : ResultSetStringRecordConverter.java
Copyright Apache License 2.0
Author : Talend
/**
 * Creates new actual schema from incoming specification <code>schema</code>
 * Actual schema fields has the same names as specification schema, but they have String type
 * Note, getSchema() will return schema, which differs from specification schema preplaceded to this method
 */
// TODO this one more kind of copySchema() method which should be implemented in Daikon see TDKN-96
@Override
public void setSchema(Schema schema) {
    actualSchema = Schema.createRecord(schema.getName(), schema.getDoc(), schema.getNamespace(), schema.isError());
    List<Schema.Field> stringFields = new ArrayList<>();
    for (Schema.Field specField : schema.getFields()) {
        boolean nullable = AvroUtils.isNullable(specField.schema());
        Schema stringSchema = AvroUtils._string();
        if (nullable) {
            stringSchema = AvroUtils.wrapAsNullable(stringSchema);
        }
        Schema.Field stringField = new Schema.Field(specField.name(), stringSchema, specField.doc(), specField.defaultVal(), specField.order());
        for (Map.Entry<String, Object> entry : specField.getObjectProps().entrySet()) {
            stringField.addProp(entry.getKey(), entry.getValue());
        }
        stringFields.add(stringField);
    }
    actualSchema.setFields(stringFields);
    for (Map.Entry<String, Object> entry : schema.getObjectProps().entrySet()) {
        actualSchema.addProp(entry.getKey(), entry.getValue());
    }
}

16 View Complete Implementation : Excel97FileRecordReader.java
Copyright Apache License 2.0
Author : Talend
private boolean nextKeyValue4Excel97() throws IOException {
    if (!rowIterator.hasNext()) {
        return false;
    }
    currentRow++;
    Row row = rowIterator.next();
    if (ExcelUtils.isEmptyRow(row)) {
        // skip empty rows
        return next();
    }
    // if not fill the schema before as no header or invalid header, set it here and as no valid name as no header, so set a name like this : field1,field2,field3
    if (schema == null) {
        schema = createSchema(row, false);
    }
    value = new GenericData.Record(schema);
    List<Field> fields = schema.getFields();
    int lastColumn = Math.max(row.getLastCellNum(), fields.size());
    for (int i = 0; i < lastColumn; i++) {
        String content = ExcelUtils.getCellValuereplacedtring(row.getCell(i, MissingCellPolicy.RETURN_BLANK_AS_NULL), formulaEvaluator);
        value.put(i, content);
    }
    return true;
}

16 View Complete Implementation : TMarketoListOperationPropertiesTest.java
Copyright Apache License 2.0
Author : Talend
@Test
public void testGetRESTSchemaMain() throws Exception {
    Schema s = MarketoConstants.getListOperationRESTSchema();
    replacedertEquals(2, s.getFields().size());
}

16 View Complete Implementation : FieldDescriptionTest.java
Copyright Apache License 2.0
Author : Talend
@Test
public void testGetSchemaFromJson() throws Exception {
    String s = "schemaTest";
    String k = "";
    String f = "";
    Schema r = FieldDescription.getSchemaFromJson(s, f, k);
    replacedertNull(r);
    k = "[\"model\"]";
    f = "[{\"displayName\":\"Created At\",\"dataType\":\"datetime\",\"name\":\"createdAt\",\"updateable\":false}," + "{\"displayName\":\"Marketo GUID\",\"dataType\":\"string\",\"length\":36,\"name\":\"marketoGUID\"," + "\"updateable\":false},{\"displayName\":\"Updated At\",\"dataType\":\"datetime\",\"name\":\"updatedAt\",\"updateable\":false},{\"displayName\":\"Acquired at\",\"dataType\":\"date\",\"name\":\"acquiredAt\",\"updateable\":true},{\"displayName\":\"Brand\",\"dataType\":\"string\",\"length\":255,\"name\":\"brand\",\"updateable\":true},{\"displayName\":\"Customer Id\",\"dataType\":\"integer\",\"name\":\"customerId\",\"updateable\":true},{\"displayName\":\"Model\",\"dataType\":\"string\",\"length\":255,\"name\":\"model\",\"updateable\":true}]";
    r = FieldDescription.getSchemaFromJson(s, f, k);
    replacedertNotNull(r);
    replacedertEquals("createdAt", r.getFields().get(0).name());
    replacedertEquals("true", r.getField("model").getProp(SchemaConstants.TALEND_COLUMN_IS_KEY));
    replacedertEquals("STRING", r.getField("marketoGUID").schema().getTypes().get(1).getType().toString());
    replacedertEquals("STRING", r.getField("brand").schema().getTypes().get(1).getType().toString());
    replacedertEquals("LONG", r.getField("createdAt").schema().getTypes().get(1).getType().toString());
    replacedertEquals("java.util.Date", r.getField("createdAt").getProp(SchemaConstants.JAVA_CLreplaced_FLAG));
    replacedertEquals(DATETIME_PATTERN_REST, r.getField("createdAt").getProp(SchemaConstants.TALEND_COLUMN_PATTERN));
    replacedertEquals("LONG", r.getField("updatedAt").schema().getTypes().get(1).getType().toString());
    replacedertEquals("java.util.Date", r.getField("updatedAt").getProp(SchemaConstants.JAVA_CLreplaced_FLAG));
    replacedertEquals(DATETIME_PATTERN_REST, r.getField("updatedAt").getProp(SchemaConstants.TALEND_COLUMN_PATTERN));
}

16 View Complete Implementation : ExcelHTMLFileRecordReader.java
Copyright Apache License 2.0
Author : Talend
private boolean nextKeyValue4ExcelHtml() {
    if (!htmlRowIterator.hasNext()) {
        return false;
    }
    currentRow++;
    List<String> row = htmlRowIterator.next();
    // if not fill the schema before as no header or invalid header, set it here and as no valid name as no header, so set a name like this : field1,field2,field3
    if (schema == null) {
        schema = createSchema(row, false);
    }
    value = new GenericData.Record(schema);
    List<Field> fields = schema.getFields();
    for (int i = 0; i < row.size(); i++) {
        if (i < fields.size()) {
            value.put(i, row.get(i));
        }
    }
    return true;
}

16 View Complete Implementation : CouchbaseWriter.java
Copyright Apache License 2.0
Author : Talend
public JsonObject createHierarchicalJson(Schema schema, IndexedRecord record, int idPos) {
    JsonObject jsonObject = JsonObject.create();
    for (int i = 0; i < schema.getFields().size(); i++) {
        if (i == idPos)
            continue;
        Object value = record.get(i);
        String fieldName = schema.getFields().get(i).name();
        try {
            JsonObject innerJson = JsonObject.fromJson(value.toString());
            jsonObject.put(fieldName, innerJson);
        } catch (Exception e) {
            try {
                JsonArray jsonArray = JsonArray.fromJson(value.toString());
                jsonObject.put(fieldName, jsonArray);
            } catch (Exception e2) {
                // This mean it's not JSON object
                jsonObject.put(fieldName, value);
            }
        }
    }
    return jsonObject;
}

16 View Complete Implementation : Excel2007FileRecordReader.java
Copyright Apache License 2.0
Author : Talend
private boolean nextKeyValue4Excel2007() throws IOException {
    if (!rowIterator.hasNext()) {
        return false;
    }
    currentRow++;
    Row row = rowIterator.next();
    if (ExcelUtils.isEmptyRow4Stream(row)) {
        // skip empty rows
        return next();
    }
    // if not fill the schema before as no header or invalid header, set it here and as no valid name as no header, so set a name like this : field1,field2,field3
    if (schema == null) {
        schema = createSchema(row, false);
    }
    value = new GenericData.Record(schema);
    List<Field> fields = schema.getFields();
    for (int i = 0; i < fields.size(); i++) {
        Cell cell = row.getCell(i);
        String content = cell == null ? StringUtils.EMPTY : cell.getStringCellValue();
        value.put(i, content);
    }
    return true;
}

16 View Complete Implementation : FileDelimitedReaderTestIT.java
Copyright Apache License 2.0
Author : Talend
protected void testInputDynamic(TFileInputDelimitedProperties properties, int count) throws Throwable {
    List<IndexedRecord> records = readRows(properties);
    replacedertNotNull(records);
    replacedertEquals(count, records.size());
    Schema dynamicSchema = records.get(0).getSchema();
    replacedertNotNull(dynamicSchema);
    replacedertEquals(11, dynamicSchema.getFields().size());
    replacedertEquals("TestBoolean", dynamicSchema.getFields().get(0).name());
    replacedertEquals("TestBytes", dynamicSchema.getFields().get(2).name());
    replacedertEquals("TestDate", dynamicSchema.getFields().get(4).name());
    replacedertEquals("TestFloat", dynamicSchema.getFields().get(6).name());
    replacedertEquals("TestInteger", dynamicSchema.getFields().get(8).name());
    replacedertEquals("TestObject", dynamicSchema.getFields().get(10).name());
    if (properties.header.getValue() == 1 || properties.header.getValue() == 3) {
        replacedertEquals("1", records.get(0).get(1));
        replacedertEquals("n", records.get(0).get(3));
        replacedertEquals("2.75", records.get(0).get(5));
        replacedertEquals("4.797", records.get(0).get(7));
        replacedertEquals("1473147067519", records.get(0).get(9));
    }
    if (properties.limit.getValue() != null && properties.limit.getValue() == 10) {
        replacedertEquals("false", records.get(8).get(0));
        replacedertEquals("CqkDWKxfab9XJvd8l9", records.get(8).get(2));
        replacedertEquals("2016-09-06T15:31:07", records.get(8).get(4));
        replacedertEquals("10.578", records.get(8).get(6));
        replacedertEquals("4671", records.get(8).get(8));
        replacedertEquals("Herbert McKinley", records.get(8).get(10));
    }
    printLogRecords(records);
}

16 View Complete Implementation : SchemaGeneratorUtils.java
Copyright Apache License 2.0
Author : Talend
/**
 * Merge a KV-Schema into a single schema.
 *
 * For each level, the schema will contains the elements present in the keySchema first, then the ones present in
 * the valueSchema.
 *
 * @param keySchema an avro Schema
 * @param valueSchema an avro Schema
 * @return an avro Schema merging the two previous schema
 */
public static Schema mergeKeyValues(Schema keySchema, Schema valueSchema) {
    List<Schema.Field> fieldList = new ArrayList<>();
    for (Field field : keySchema.getFields()) {
        if (valueSchema.getField(field.name()) != null) {
            // element in both key and value => create sub element
            fieldList.add(new Field(field.name(), mergeKeyValues(field.schema(), valueSchema.getField(field.name()).schema()), "", ""));
        } else {
            // Element only present in the key
            fieldList.add(new Field(field.name(), field.schema(), field.doc(), field.defaultVal()));
        }
    }
    for (Field field : valueSchema.getFields()) {
        if (keySchema.getField(field.name()) == null) {
            // Element only present in the value
            fieldList.add(new Field(field.name(), field.schema(), field.doc(), field.defaultVal()));
        }
    }
    if (fieldList.size() > 0) {
        try {
            return Schema.createRecord(keySchema.getName(), keySchema.getDoc(), keySchema.getNamespace(), keySchema.isError(), fieldList);
        } catch (AvroRuntimeException e) {
            // this will be throw if we are trying to get the name of an anonymous type
            return Schema.createRecord(fieldList);
        }
    } else {
        return AvroUtils.createEmptySchema();
    }
}

16 View Complete Implementation : KeyValueUtils.java
Copyright Apache License 2.0
Author : Talend
/**
 * Generate a new Index Record which is the filtered result of the input record.
 *
 * The user can freely remove column, add empty column or change the place of column in the same hierarchical level.
 *
 * @return the new record
 */
public static IndexedRecord extractIndexedRecord(IndexedRecord inputRecord, Schema outputSchema) {
    GenericRecordBuilder outputRecord = new GenericRecordBuilder(outputSchema);
    Schema inputSchema = getUnwrappedSchema(inputRecord);
    for (Field field : outputSchema.getFields()) {
        if (inputSchema.getField(field.name()) != null) {
            // The column was existing on the input record, we forward it to the output record.
            Object inputValue = inputRecord.get(inputSchema.getField(field.name()).pos());
            // The current column can be a Record (an hierarchical sub-object) or directly a value.
            // If we are on a record, we need to recursively do the process
            // if we are on a object, we save it to the output.
            if (inputValue instanceof Record) {
                // The sub-schema at this level is a union of "empty" and a record,
                // so we need to get the true sub-schema
                Schema inputChildSchema = getUnwrappedSchema(inputSchema.getField(field.name()));
                Schema outputChildSchema = getUnwrappedSchema(outputSchema.getField(field.name()));
                if (inputChildSchema.getType().equals(Type.RECORD) && outputChildSchema.getType().equals(Type.RECORD)) {
                    Object childRecord = extractIndexedRecord((IndexedRecord) inputValue, outputChildSchema);
                    outputRecord.set(field.name(), childRecord);
                }
            } else {
                outputRecord.set(field.name(), inputValue);
            }
        } else {
            // element not found => set to the value and its hierarchy to null
            outputRecord.set(field.name(), KeyValueUtils.generateEmptyRecord(outputSchema, field.name()));
        }
    }
    return outputRecord.build();
}

16 View Complete Implementation : KeyValueUtils.java
Copyright Apache License 2.0
Author : Talend
/**
 * Use a Schema to generate a hierarchical GenericRecord that contains only null values.
 *
 * @param schema the parent schema of the field to set as null
 * @param fieldName the name of the field to set as null
 * @return if fieldName is a Record of the schema, the method will return a GenericRecord with any leaf set as null,
 * otherwise return null
 */
public static IndexedRecord generateEmptyRecord(Schema schema, String fieldName) {
    if (schema.getType().equals(Type.RECORD)) {
        Schema unwrappedSchema = getUnwrappedSchema(schema.getField(fieldName));
        if (unwrappedSchema.getType().equals(Type.RECORD)) {
            GenericRecordBuilder outputRecord = new GenericRecordBuilder(unwrappedSchema);
            for (Field field : unwrappedSchema.getFields()) {
                IndexedRecord value = generateEmptyRecord(unwrappedSchema, field.name());
                outputRecord.set(field.name(), value);
            }
            return outputRecord.build();
        } else {
            return null;
        }
    } else {
        return null;
    }
}

16 View Complete Implementation : CommonUtils.java
Copyright Apache License 2.0
Author : Talend
public static List<String> getAllSchemaFieldNames(Schema schema) {
    List<String> values = new ArrayList<>();
    if (schema == null) {
        return values;
    }
    for (Schema.Field field : schema.getFields()) {
        values.add(field.name());
    }
    return values;
}

16 View Complete Implementation : TJDBCRowProperties.java
Copyright Apache License 2.0
Author : Talend
private List<String> getFieldNames(Property<Schema> schema) {
    Schema s = schema.getValue();
    List<String> fieldNames = new ArrayList<>();
    for (Schema.Field f : s.getFields()) {
        fieldNames.add(f.name());
    }
    return fieldNames;
}

15 View Complete Implementation : MarketoUtils.java
Copyright Apache License 2.0
Author : Talend
public static Schema newSchema(Schema metadataSchema, String newSchemaName, List<Schema.Field> moreFields) {
    Schema newSchema = Schema.createRecord(newSchemaName, metadataSchema.getDoc(), metadataSchema.getNamespace(), metadataSchema.isError());
    List<Schema.Field> copyFieldList = new ArrayList<>();
    for (Schema.Field se : metadataSchema.getFields()) {
        Schema.Field field = new Schema.Field(se.name(), se.schema(), se.doc(), se.defaultVal(), se.order());
        field.getObjectProps().putAll(se.getObjectProps());
        for (Map.Entry<String, Object> entry : se.getObjectProps().entrySet()) {
            field.addProp(entry.getKey(), entry.getValue());
        }
        copyFieldList.add(field);
    }
    copyFieldList.addAll(moreFields);
    newSchema.setFields(copyFieldList);
    for (Map.Entry<String, Object> entry : metadataSchema.getObjectProps().entrySet()) {
        newSchema.addProp(entry.getKey(), entry.getValue());
    }
    return newSchema;
}

15 View Complete Implementation : JDBCSPIndexedRecordCreator.java
Copyright Apache License 2.0
Author : Talend
public IndexedRecord createOutputIndexedRecord(CallableStatement value, IndexedRecord inputRecord) {
    if (!firstRowHaveCame) {
        firstRowHaveCame = true;
        Schema inputSchema = null;
        if (inputRecord != null) {
            inputSchema = inputRecord.getSchema();
        }
        Map<String, Field> inputFieldMap = null;
        for (Schema.Field outputField : outputSchema.getFields()) {
            if (outputFieldLocation2AvroConverter.containsKey(outputField.pos()) || (resultSetPostionOfOutputSchema == outputField.pos())) {
                continue;
            }
            if (inputSchema == null) {
                break;
            }
            List<Field> inputFields = inputSchema.getFields();
            if (inputFieldMap == null) {
                inputFieldMap = new HashMap<>();
                for (Field inputField : inputFields) {
                    inputFieldMap.put(inputField.name(), inputField);
                }
            }
            Field inputField = inputFieldMap.get(outputField.name());
            if (inputField != null) {
                autoPropagatedFieldsFromInputToOutput.put(outputField.pos(), inputField.pos());
            }
        }
    }
    return new ResultSetIndexedRecord(value, inputRecord);
}

15 View Complete Implementation : CommonUtils.java
Copyright Apache License 2.0
Author : Talend
public static List<String> getAllSchemaFieldDBNames(Schema schema) {
    List<String> values = new ArrayList<>();
    if (schema == null) {
        return values;
    }
    for (Schema.Field field : schema.getFields()) {
        values.add(field.getProp(SchemaConstants.TALEND_COLUMN_DB_COLUMN_NAME));
    }
    return values;
}

15 View Complete Implementation : MarketoSourceOrSinkTest.java
Copyright Apache License 2.0
Author : Talend
@Test
public void testMergeDynamicSchemas() throws Exception {
    Schema lead = MarketoConstants.getRESTSchemaForGetLeadOrGetMultipleLeads();
    Schema co = MarketoConstants.getCustomObjectDescribeSchema();
    Schema merged = MarketoSourceOrSink.mergeDynamicSchemas(lead, co);
    replacedertNotNull(merged);
    replacedertEquals(10, merged.getFields().size());
    merged = MarketoSourceOrSink.mergeDynamicSchemas(lead, getLeadDynamicSchema());
    replacedertEquals(2, merged.getFields().size());
}