org.apache.avro.Schema.getField() - java examples

Here are the examples of the java api org.apache.avro.Schema.getField() taken from open source projects. By voting up you can indicate which examples are most useful and appropriate.

155 Examples 7

19 View Complete Implementation : MarketoCampaignClientTestIT.java
Copyright Apache License 2.0
Author : Talend
public void testRecord(IndexedRecord record) {
    LOG.debug("record = {}.", record);
    replacedertNotNull(record.get(s.getField("id").pos()));
    replacedertTrue(record.get(s.getField("id").pos()) instanceof Integer);
    replacedertNotNull(record.get(s.getField("name").pos()));
    replacedertTrue(record.get(s.getField("name").pos()) instanceof String);
    replacedertNotNull(record.get(s.getField("createdAt").pos()));
    replacedertTrue(record.get(s.getField("createdAt").pos()) instanceof Long);
    replacedertNotNull(record.get(s.getField("updatedAt").pos()));
    replacedertTrue(record.get(s.getField("updatedAt").pos()) instanceof Long);
    if (record.get(s.getField("active").pos()).toString().equals("true")) {
        LOG.debug("record = {}.", record);
    }
}

19 View Complete Implementation : SalesforceInputReaderTestIT.java
Copyright Apache License 2.0
Author : Talend
/*
     * Test nested query of SOQL. Checking if data was placed correctly by guessed schema method.
     */
@Test
public void testComplexSOQLQuery() throws Throwable {
    TSalesforceInputProperties props = createTSalesforceInputProperties(false, false);
    props.manualQuery.setValue(true);
    // Manual query with foreign key
    // Need to specify where clause to be sure that this record exists and has parent-to-child relation.
    props.query.setValue("Select Id, Name,(Select Contact.Id,Contact.Name from Account.Contacts) from Account WHERE Name = 'United Oil & Gas, UK' Limit 1");
    props.validateGuessSchema();
    List<IndexedRecord> rows = readRows(props);
    if (rows.size() > 0) {
        for (IndexedRecord row : rows) {
            Schema schema = row.getSchema();
            replacedertNotNull(schema.getField("Id"));
            replacedertNotNull(schema.getField("Name"));
            replacedertNotNull(schema.getField("Account_Contacts_records_Contact_Id"));
            replacedertNotNull(schema.getField("Account_Contacts_records_Contact_Name"));
            replacedertNotNull(row.get(schema.getField("Id").pos()));
            replacedertNotNull(row.get(schema.getField("Name").pos()));
            replacedertNotNull(row.get(schema.getField("Account_Contacts_records_Contact_Id").pos()));
            replacedertNotNull(row.get(schema.getField("Account_Contacts_records_Contact_Name").pos()));
        }
    } else {
        LOGGER.warn("Query result is empty!");
    }
}

19 View Complete Implementation : SalesforceInputReaderTestIT.java
Copyright Apache License 2.0
Author : Talend
@Test
public void testManualQuery() throws Throwable {
    TSalesforceInputProperties props = createTSalesforceInputProperties(false, false);
    props.manualQuery.setValue(true);
    props.query.setValue("select Id from Account WHERE Name = '" + randomizedValue + "'");
    List<IndexedRecord> outputRows = readRows(props);
    replacedertEquals(100, outputRows.size());
    props.module.main.schema.setValue(// 
    SchemaBuilder.builder().record("MakeRowRecord").fields().name("Id").type().nullable().stringType().noDefault().name("Name").type().nullable().stringType().noDefault().name("Owner_Name").type().nullable().stringType().noDefault().name("Owner_Id").type().nullable().stringType().noDefault().endRecord());
    props.query.setValue("SELECT Id, Name, Owner.Name ,Owner.Id FROM Account WHERE Name = '" + randomizedValue + "'");
    List<IndexedRecord> rowsWithForeignKey = readRows(props);
    props.module.main.schema.setValue(// 
    SchemaBuilder.builder().record("MakeRowRecord").fields().name("Id").type().nullable().stringType().noDefault().name("Name").type().nullable().stringType().noDefault().name("OwnerId").type().nullable().stringType().noDefault().endRecord());
    props.query.setValue("SELECT Id, Name, OwnerId FROM Account WHERE Name = '" + randomizedValue + "'");
    outputRows = readRows(props);
    replacedertEquals(rowsWithForeignKey.size(), outputRows.size());
    replacedertEquals(100, rowsWithForeignKey.size());
    IndexedRecord fkRecord = rowsWithForeignKey.get(0);
    IndexedRecord commonRecord = outputRows.get(0);
    replacedertNotNull(fkRecord);
    replacedertNotNull(commonRecord);
    Schema schemaFK = fkRecord.getSchema();
    Schema schemaCommon = commonRecord.getSchema();
    replacedertNotNull(schemaFK);
    replacedertNotNull(schemaCommon);
    replacedertEquals(commonRecord.get(schemaCommon.getField("OwnerId").pos()), fkRecord.get(schemaFK.getField("Owner_Id").pos()));
}

19 View Complete Implementation : TMarketoOutputPropertiesTest.java
Copyright Apache License 2.0
Author : Talend
@Test
public void testDeleteLeadsSchemas() throws Exception {
    props.outputOperation.setValue(OutputOperation.deleteLeads);
    props.afterOutputOperation();
    replacedertEquals(MarketoConstants.getDeleteLeadsSchema(), props.schemaInput.schema.getValue());
    Schema flow = props.schemaFlow.schema.getValue();
    replacedertNotNull(flow.getField("Status"));
    props.deleteLeadsInBatch.setValue(true);
    props.afterDeleteLeadsInBatch();
    replacedertEquals(MarketoConstants.getDeleteLeadsSchema(), props.schemaInput.schema.getValue());
}

19 View Complete Implementation : FetchParquetTest.java
Copyright Apache License 2.0
Author : apache
private void writeParquetUsersWithArray(final File parquetFile, int numUsers) throws IOException {
    if (parquetFile.exists()) {
        replacedert.replacedertTrue(parquetFile.delete());
    }
    final AvroParquetWriter.Builder<GenericRecord> writerBuilder = createAvroParquetWriter(parquetFile, schemaWithArray);
    final Schema favoriteColorsSchema = schemaWithArray.getField("favorite_colors").schema();
    try (final ParquetWriter<GenericRecord> writer = writerBuilder.build()) {
        for (int i = 0; i < numUsers; i++) {
            final GenericRecord user = new GenericData.Record(schema);
            user.put("name", "Bob" + i);
            user.put("favorite_number", i);
            final GenericData.Array<String> colors = new GenericData.Array<>(1, favoriteColorsSchema);
            colors.add("blue" + i);
            user.put("favorite_color", colors);
            writer.write(user);
        }
    }
}

19 View Complete Implementation : MarketoClientCustomObjectsTestIT.java
Copyright Apache License 2.0
Author : Talend
/*
     *
     * Custom Objects
     *
     */
public void checkCustomObject(IndexedRecord r, Boolean isDescribe) {
    Schema s = MarketoConstants.getCustomObjectDescribeSchema();
    replacedertNotNull(r.get(s.getField("name").pos()));
    replacedertNotNull(r.get(s.getField("createdAt").pos()));
    replacedertEquals("java.util.Date", r.get(s.getField("createdAt").pos()).getClreplaced().getCanonicalName());
    replacedertTrue(r.get(s.getField("createdAt").pos()) instanceof Date);
    replacedertNotNull(r.get(s.getField("updatedAt").pos()));
    replacedertTrue(r.get(s.getField("updatedAt").pos()) instanceof Date);
    replacedertNotNull(r.get(s.getField("idField").pos()));
    replacedertNotNull(r.get(s.getField("dedupeFields").pos()));
    replacedertNotNull(r.get(s.getField("searchableFields").pos()));
    replacedertNotNull(r.get(s.getField("relationships").pos()));
    replacedertEquals(true, r.get(s.getField("fields").pos()) != null);
}

19 View Complete Implementation : FetchParquetTest.java
Copyright Apache License 2.0
Author : apache
private void writeParquetUsersWithNullableArray(final File parquetFile, int numUsers) throws IOException {
    if (parquetFile.exists()) {
        replacedert.replacedertTrue(parquetFile.delete());
    }
    final AvroParquetWriter.Builder<GenericRecord> writerBuilder = createAvroParquetWriter(parquetFile, schemaWithNullableArray);
    // use the schemaWithArray here just to get the schema for the array part of the favorite_colors fields, the overall
    // schemaWithNullableArray has a union of the array schema and null
    final Schema favoriteColorsSchema = schemaWithArray.getField("favorite_colors").schema();
    try (final ParquetWriter<GenericRecord> writer = writerBuilder.build()) {
        for (int i = 0; i < numUsers; i++) {
            final GenericRecord user = new GenericData.Record(schema);
            user.put("name", "Bob" + i);
            user.put("favorite_number", i);
            final GenericData.Array<String> colors = new GenericData.Array<>(1, favoriteColorsSchema);
            colors.add("blue" + i);
            user.put("favorite_color", colors);
            writer.write(user);
        }
    }
}

19 View Complete Implementation : TSplunkEventCollectorWriterTestIT.java
Copyright Apache License 2.0
Author : Talend
/**
 * Create indexed record with default static schema.
 */
private IndexedRecord createIndexedRecord() {
    Schema schema = createSchema();
    IndexedRecord record = new GenericData.Record(schema);
    record.put(schema.getField("FieldString").pos(), "String");
    record.put(schema.getField("FieldInt").pos(), 12);
    return record;
}

19 View Complete Implementation : SalesforceWriter.java
Copyright Apache License 2.0
Author : Talend
private void handleSuccess(IndexedRecord input, String id, String status) {
    successCount++;
    Schema outSchema = sprops.schemaFlow.schema.getValue();
    if (outSchema == null || outSchema.getFields().size() == 0) {
        return;
    }
    if (input.getSchema().equals(outSchema)) {
        successfulWrites.add(input);
    } else {
        IndexedRecord successful = null;
        if (AvroUtils.isIncludeAllFields(outSchema)) {
            Schema runtimeSchema = input.getSchema();
            List<Schema.Field> addedFields = new ArrayList<>();
            // Check whether design schema has additional field
            Schema.Field idField = outSchema.getField(TSalesforceOutputProperties.FIELD_SALESFORCE_ID);
            Schema.Field statusField = outSchema.getField(TSalesforceOutputProperties.FIELD_STATUS);
            if (idField != null) {
                addedFields.add(new Schema.Field(idField.name(), idField.schema(), idField.doc(), idField.defaultVal()));
            }
            if (statusField != null) {
                addedFields.add(new Schema.Field(statusField.name(), statusField.schema(), statusField.doc(), statusField.defaultVal()));
            }
            if (addedFields.size() > 0) {
                // Append additional fields to the runtime schema
                runtimeSchema = AvroUtils.appendFields(runtimeSchema, addedFields.toArray(new Schema.Field[addedFields.size()]));
            }
            successful = new GenericData.Record(runtimeSchema);
        } else {
            successful = new GenericData.Record(outSchema);
        }
        for (Schema.Field outField : successful.getSchema().getFields()) {
            Object outValue = null;
            Schema.Field inField = input.getSchema().getField(outField.name());
            if (inField != null) {
                outValue = input.get(inField.pos());
            } else if (TSalesforceOutputProperties.FIELD_SALESFORCE_ID.equals(outField.name())) {
                outValue = id;
            } else if (TSalesforceOutputProperties.FIELD_STATUS.equals(outField.name())) {
                outValue = status;
            }
            successful.put(outField.pos(), outValue);
        }
        successfulWrites.add(successful);
    }
    LOGGER.info(MESSAGES.getMessage("info.successfulRecord", getPastForm(sprops.outputAction.getValue()), dataCount));
}

19 View Complete Implementation : TSplunkEventCollectorWriterTestIT.java
Copyright Apache License 2.0
Author : Talend
/**
 * create indexed record with Date presented as String.
 */
private IndexedRecord createIndexedRecordWithTimereplacedtring() {
    Schema schema = createSchemaWithTimereplacedtring();
    DateFormat format = new SimpleDateFormat(DATE_PATTERN);
    IndexedRecord record = new GenericData.Record(schema);
    record.put(schema.getField("FieldString").pos(), "String");
    record.put(schema.getField("FieldInt").pos(), 12);
    record.put(schema.getField("Description").pos(), "Dynamic time column test.");
    record.put(schema.getField("time").pos(), format.format(new Date()));
    return record;
}

19 View Complete Implementation : SalesforceGetDeletedUpdatedReaderTestIT.java
Copyright Apache License 2.0
Author : Talend
public List<IndexedRecord> makeUpdateRows(String random, boolean isDynamic, List<IndexedRecord> rows) {
    List<IndexedRecord> outputRows = new ArrayList<>();
    for (int i = 0; i < rows.size(); i++) {
        Schema schema = rows.get(i).getSchema();
        GenericData.Record row = new GenericData.Record(schema);
        row.put("Id", rows.get(i).get(schema.getField("Id").pos()));
        row.put("Name", rows.get(i).get(schema.getField("Name").pos()));
        row.put("ShippingStreet", rows.get(i).get(schema.getField("ShippingStreet").pos()));
        row.put("ShippingPostalCode", String.valueOf(rows.get(i).get(schema.getField("ShippingPostalCode").pos())));
        row.put("BillingStreet", rows.get(i).get(schema.getField("BillingStreet").pos()));
        row.put("BillingState", rows.get(i).get(schema.getField("BillingState").pos()));
        row.put("BillingPostalCode", random);
        if (isDynamic) {
            row.put("ShippingState", rows.get(i).get(schema.getField("ShippingState").pos()));
        }
        System.out.println(// 
        "Row to update: " + row.get("Name") + " id: " + // 
        row.get("Id") + " shippingPostalCode: " + // 
        row.get("ShippingPostalCode") + " billingPostalCode: " + // 
        row.get("BillingPostalCode") + " billingStreet: " + row.get("BillingStreet"));
        outputRows.add(row);
    }
    return outputRows;
}

19 View Complete Implementation : SalesforceWriter.java
Copyright Apache License 2.0
Author : Talend
private void handleReject(IndexedRecord input, Error[] resultErrors, String[] changedItemKeys, int batchIdx) throws IOException {
    String changedItemKey = null;
    if (batchIdx < changedItemKeys.length) {
        if (changedItemKeys[batchIdx] != null) {
            changedItemKey = changedItemKeys[batchIdx];
        } else {
            changedItemKey = String.valueOf(batchIdx + 1);
        }
    } else {
        changedItemKey = "Batch index out of bounds";
    }
    StringBuilder errors = SalesforceRuntime.addLog(resultErrors, changedItemKey, logWriter);
    if (exceptionForErrors) {
        if (errors.toString().length() > 0) {
            if (logWriter != null) {
                logWriter.close();
            }
            throw new IOException(errors.toString());
        }
    } else {
        rejectCount++;
        Schema outSchema = sprops.schemaReject.schema.getValue();
        if (outSchema == null || outSchema.getFields().size() == 0) {
            return;
        }
        if (input.getSchema().equals(outSchema)) {
            rejectedWrites.add(input);
        } else {
            IndexedRecord reject = null;
            if (AvroUtils.isIncludeAllFields(outSchema)) {
                Schema runtimeSchema = input.getSchema();
                List<Schema.Field> addedFields = new ArrayList<>();
                // Check whether design schema has additional field
                Schema.Field errorCodeField = outSchema.getField(TSalesforceOutputProperties.FIELD_ERROR_CODE);
                Schema.Field errorField = outSchema.getField(TSalesforceOutputProperties.FIELD_ERROR_FIELDS);
                Schema.Field errorMsgField = outSchema.getField(TSalesforceOutputProperties.FIELD_ERROR_MESSAGE);
                if (errorCodeField != null) {
                    addedFields.add(new Schema.Field(errorCodeField.name(), errorCodeField.schema(), errorCodeField.doc(), errorCodeField.defaultVal()));
                }
                if (errorField != null) {
                    addedFields.add(new Schema.Field(errorField.name(), errorField.schema(), errorField.doc(), errorField.defaultVal()));
                }
                if (errorMsgField != null) {
                    addedFields.add(new Schema.Field(errorMsgField.name(), errorMsgField.schema(), errorMsgField.doc(), errorMsgField.defaultVal()));
                }
                if (addedFields.size() > 0) {
                    // Append additional fields to the runtime schema
                    runtimeSchema = AvroUtils.appendFields(runtimeSchema, addedFields.toArray(new Schema.Field[addedFields.size()]));
                }
                reject = new GenericData.Record(runtimeSchema);
            } else {
                reject = new GenericData.Record(outSchema);
            }
            for (Schema.Field outField : reject.getSchema().getFields()) {
                Object outValue = null;
                Schema.Field inField = input.getSchema().getField(outField.name());
                if (inField != null) {
                    outValue = input.get(inField.pos());
                } else if (resultErrors.length > 0) {
                    Error error = resultErrors[0];
                    if (TSalesforceOutputProperties.FIELD_ERROR_CODE.equals(outField.name())) {
                        outValue = error.getStatusCode() != null ? error.getStatusCode().toString() : null;
                    } else if (TSalesforceOutputProperties.FIELD_ERROR_FIELDS.equals(outField.name())) {
                        StringBuffer fields = new StringBuffer();
                        for (String field : error.getFields()) {
                            fields.append(field);
                            fields.append(",");
                        }
                        if (fields.length() > 0) {
                            fields.deleteCharAt(fields.length() - 1);
                        }
                        outValue = fields.toString();
                    } else if (TSalesforceOutputProperties.FIELD_ERROR_MESSAGE.equals(outField.name())) {
                        outValue = error.getMessage();
                    }
                }
                reject.put(outField.pos(), outValue);
            }
            rejectedWrites.add(reject);
        }
        Property<OutputAction> outputAction = sprops.outputAction;
        LOGGER.info(MESSAGES.getMessage("info.rejectedRecord", sprops.outputAction.getPossibleValuesDisplayName(outputAction.getValue()).toLowerCase(), dataCount));
    }
}

19 View Complete Implementation : SalesforceInputReaderTestIT.java
Copyright Apache License 2.0
Author : Talend
/**
 * Test query mode fields of schema is not case sensitive
 */
@Test
public void testColumnNameCaseSensitive() throws Throwable {
    TSalesforceInputProperties props = createTSalesforceInputProperties(false, false);
    Schema schema = // 
    SchemaBuilder.builder().record("Schema").fields().name("ID").type().stringType().noDefault().name("type").type().stringType().noDefault().name("NAME").type().stringType().noDefault().endRecord();
    props.module.main.schema.setValue(schema);
    props.condition.setValue("Id != null and name != null and type!=null Limit 1");
    props.validateGuessSchema();
    List<IndexedRecord> rows = readRows(props);
    if (rows.size() > 0) {
        replacedertEquals(1, rows.size());
        IndexedRecord row = rows.get(0);
        Schema runtimeSchema = row.getSchema();
        replacedertEquals(3, runtimeSchema.getFields().size());
        replacedertNotNull(row.get(schema.getField("ID").pos()));
        replacedertNotNull(row.get(schema.getField("type").pos()));
        replacedertNotEquals("Account", row.get(schema.getField("type").pos()));
        replacedertNotNull(row.get(schema.getField("NAME").pos()));
    } else {
        LOGGER.warn("Query result is empty!");
    }
}

18 View Complete Implementation : SalesforceWriter.java
Copyright Apache License 2.0
Author : Talend
private SObject createSObject(IndexedRecord input) {
    SObject so = new SObject();
    so.setType(sprops.module.moduleName.getStringValue());
    nullValueFields.clear();
    for (Schema.Field f : input.getSchema().getFields()) {
        // For "Id" column, we should ignore it for "INSERT" action
        if (!("Id".equals(f.name()) && SalesforceOutputProperties.OutputAction.INSERT.equals(sprops.outputAction.getValue()))) {
            Object value = input.get(f.pos());
            Schema.Field se = moduleSchema.getField(f.name());
            if (se != null) {
                if (value != null && !value.toString().isEmpty()) {
                    addSObjectField(so, se.schema(), se.name(), value);
                } else {
                    if (UPDATE.equals(sprops.outputAction.getValue())) {
                        nullValueFields.add(f.name());
                    }
                }
            }
        }
    }
    if (!sprops.ignoreNull.getValue()) {
        so.setFieldsToNull(nullValueFields.toArray(new String[0]));
    }
    return so;
}

18 View Complete Implementation : SalesforceAvroRegistryTest.java
Copyright Apache License 2.0
Author : Talend
@Test
public void testStringToDecimalConverter() {
    Schema schema = // 
    SchemaBuilder.builder().record("Schema").fields().name("Field_A").type(AvroUtils._decimal()).noDefault().endRecord();
    AvroConverter<String, ?> converter = sRegistry.getConverterFromString(schema.getField("Field_A"));
    replacedertNotNull(converter);
    replacedertEquals(String.clreplaced, converter.getDatumClreplaced());
    replacedertEquals(schema.getField("Field_A").schema(), converter.getSchema());
    Object value = converter.convertToAvro("20000000000000000000000000.123456789");
    replacedertNotNull(value);
    replacedertThat(value, instanceOf(BigDecimal.clreplaced));
    replacedertThat((BigDecimal) value, equalTo(new BigDecimal("20000000000000000000000000.123456789")));
    replacedertNull(converter.convertToAvro(""));
    replacedertNull(converter.convertToAvro(null));
    String sValue = ((AvroConverter<String, BigDecimal>) converter).convertToDatum(new BigDecimal("20000000000000000000000000.123456789"));
    replacedertNotNull(sValue);
    replacedertThat(sValue, equalTo("20000000000000000000000000.123456789"));
}

18 View Complete Implementation : RootSchemaUtils.java
Copyright Apache License 2.0
Author : Talend
/**
 * Retrieves and returns Main data schema from Root schema.
 * If incoming schema is not Root schema throws an {@link IllegalArgumentException}
 *
 * @param rootSchema Root schema
 * @return Main data schema
 */
public static Schema getMainSchema(Schema rootSchema) {
    if (!isRootSchema(rootSchema)) {
        throw new IllegalArgumentException("Input schema should be Root schema");
    }
    Schema mainSchema = rootSchema.getField(MAIN_FIELD_NAME).schema();
    return mainSchema;
}

18 View Complete Implementation : SalesforceAvroRegistryTest.java
Copyright Apache License 2.0
Author : Talend
@Test(expected = ComponentException.clreplaced)
public void testStringToDateConverterParseError() throws ParseException {
    Schema schema = // 
    SchemaBuilder.builder().record("Schema").fields().name("Field_A").type(AvroUtils._date()).noDefault().endRecord();
    schema.getField("Field_A").addProp(SchemaConstants.TALEND_COLUMN_PATTERN, "yyyy-MM-dd");
    AvroConverter<String, ?> converter = sRegistry.getConverterFromString(schema.getField("Field_A"));
    converter.convertToAvro("2017/6/15");
}

18 View Complete Implementation : MarketoClientCustomObjectsTestIT.java
Copyright Apache License 2.0
Author : Talend
public void checkCustomObjectRecord(IndexedRecord r) {
    Schema s = MarketoConstants.getCustomObjectRecordSchema();
    replacedertNotNull(r.get(s.getField(FIELD_CO_MARKETO_GUID).pos()));
    replacedertNotNull(r.get(s.getField("seq").pos()));
    replacedertTrue(r.get(s.getField("seq").pos()) instanceof Integer);
    replacedertNotNull(r.get(s.getField("createdAt").pos()));
    replacedertEquals("java.util.Date", r.get(s.getField("createdAt").pos()).getClreplaced().getCanonicalName());
    replacedertTrue(r.get(s.getField("createdAt").pos()) instanceof Date);
    replacedertNotNull(r.get(s.getField("updatedAt").pos()));
    replacedertTrue(r.get(s.getField("updatedAt").pos()) instanceof Date);
}

18 View Complete Implementation : SalesforceWriter.java
Copyright Apache License 2.0
Author : Talend
private void addSObjectField(XmlObject xmlObject, Schema expected, String fieldName, Object value) {
    Object valueToAdd = null;
    // Convert stuff here
    // For Nillable base64 type field, we retrieve it as UNION type:[bytes,null]
    // So need to unwrap it and get its real type
    Schema unwrapSchema = AvroUtils.unwrapIfNullable(expected);
    switch(unwrapSchema.getType()) {
        case BYTES:
            if ((value instanceof String) || (value instanceof byte[])) {
                byte[] base64Data = null;
                if (value instanceof byte[]) {
                    base64Data = (byte[]) value;
                } else {
                    base64Data = ((String) value).getBytes();
                }
                if (Base64.isBase64(new String(base64Data))) {
                    valueToAdd = Base64.decode(base64Data);
                    break;
                }
            }
        default:
            valueToAdd = value;
            break;
    }
    if (valueToAdd instanceof Date) {
        xmlObject.setField(fieldName, SalesforceRuntime.convertDateToCalendar((Date) valueToAdd, true));
    } else {
        Schema.Field se = moduleSchema.getField(fieldName);
        if (se != null && valueToAdd instanceof String) {
            String datePattern = se.getProp(SchemaConstants.TALEND_COLUMN_PATTERN);
            if (datePattern != null && !datePattern.toString().isEmpty()) {
                if ("yyyy-MM-dd'T'HH:mm:ss'.000Z'".equals(datePattern)) {
                    xmlObject.setField(fieldName, calendarCodec.deserialize((String) valueToAdd));
                } else if ("yyyy-MM-dd".equals(datePattern)) {
                    xmlObject.setField(fieldName, dateCodec.deserialize((String) valueToAdd));
                } else {
                    xmlObject.setField(fieldName, new Time((String) valueToAdd));
                }
            } else {
                xmlObject.setField(fieldName, SalesforceAvroRegistry.get().getConverterFromString(se).convertToAvro((String) valueToAdd));
            }
        } else {
            xmlObject.setField(fieldName, valueToAdd);
        }
    }
}

18 View Complete Implementation : AggregateUtils.java
Copyright Apache License 2.0
Author : Talend
/**
 * Set the value to record based on the fieldPath, if the path is wrong nothing will be added on record
 *
 * @param fieldPath
 * @param value
 * @param record
 */
public static void setField(String fieldPath, Object value, IndexedRecord record) {
    String[] path = fieldPath.split("\\.");
    if (path.length <= 0) {
        return;
    }
    Schema schema = record.getSchema();
    IndexedRecord parentRecord = record;
    for (int i = 0; i < path.length - 1; i++) {
        if (schema.getField(path[i]) == null) {
            return;
        }
        Object parentRecordObj = parentRecord.get(schema.getField(path[i]).pos());
        if (parentRecordObj instanceof GenericData.Record) {
            parentRecord = (IndexedRecord) parentRecordObj;
            // The sub-schema at this level is a union of "empty" and a
            // record, so we need to get the true sub-schema
            if (schema.getField(path[i]).schema().getType().equals(Schema.Type.RECORD)) {
                schema = schema.getField(path[i]).schema();
            } else if (schema.getField(path[i]).schema().getType().equals(Schema.Type.UNION)) {
                for (Schema childSchema : schema.getField(path[i]).schema().getTypes()) {
                    if (childSchema.getType().equals(Schema.Type.RECORD)) {
                        schema = childSchema;
                        break;
                    }
                }
            }
        } else {
            // can't find parent record, can't fill value
            return;
        }
    }
    parentRecord.put(schema.getField(path[path.length - 1]).pos(), value);
}

18 View Complete Implementation : FetchParquetTest.java
Copyright Apache License 2.0
Author : apache
private void writeParquetUsersWithDecimal(final File parquetFile, int numUsers) throws IOException {
    if (parquetFile.exists()) {
        replacedert.replacedertTrue(parquetFile.delete());
    }
    final BigDecimal initialAmount = new BigDecimal("1234567.0123456789");
    final AvroParquetWriter.Builder<GenericRecord> writerBuilder = createAvroParquetWriter(parquetFile, schemaWithDecimal);
    final List<Schema> amountSchemaUnion = schemaWithDecimal.getField("amount").schema().getTypes();
    final Schema amountSchema = amountSchemaUnion.stream().filter(s -> s.getType() == Schema.Type.FIXED).findFirst().orElse(null);
    replacedert.replacedertNotNull(amountSchema);
    final Conversions.DecimalConversion decimalConversion = new Conversions.DecimalConversion();
    try (final ParquetWriter<GenericRecord> writer = writerBuilder.build()) {
        for (int i = 0; i < numUsers; i++) {
            final BigDecimal incrementedAmount = initialAmount.add(new BigDecimal("1"));
            final GenericRecord user = new GenericData.Record(schemaWithDecimal);
            user.put("name", "Bob" + i);
            user.put("amount", decimalConversion.toFixed(incrementedAmount, amountSchema, amountSchema.getLogicalType()));
            writer.write(user);
        }
    }
}

18 View Complete Implementation : TestAvroTypeUtil.java
Copyright Apache License 2.0
Author : apache
@Test
public void testAliasCreatedForInvalidField() {
    final List<RecordField> fields = new ArrayList<>();
    fields.add(new RecordField("valid", RecordFieldType.STRING.getDataType()));
    fields.add(new RecordField("$invalid2", RecordFieldType.STRING.getDataType()));
    fields.add(new RecordField("3invalid3", RecordFieldType.STRING.getDataType()));
    fields.add(new RecordField("  __ Another ONE!!", RecordFieldType.STRING.getDataType()));
    final RecordSchema recordSchema = new SimpleRecordSchema(fields);
    final Schema avroSchema = AvroTypeUtil.extractAvroSchema(recordSchema);
    replacedertNotNull(avroSchema.getField("valid"));
    replacedertNull(avroSchema.getField("$invalid"));
    final Field field2 = avroSchema.getField("_invalid2");
    replacedertNotNull(field2);
    replacedertEquals("_invalid2", field2.name());
    replacedertEquals(1, field2.aliases().size());
    replacedertTrue(field2.aliases().contains("$invalid2"));
    replacedertNull(avroSchema.getField("$invalid3"));
    final Field field3 = avroSchema.getField("_invalid3");
    replacedertNotNull(field3);
    replacedertEquals("_invalid3", field3.name());
    replacedertEquals(1, field3.aliases().size());
    replacedertTrue(field3.aliases().contains("3invalid3"));
    replacedertNull(avroSchema.getField("  __ Another ONE!!"));
    final Field field4 = avroSchema.getField("_____Another_ONE__");
    replacedertNotNull(field4);
    replacedertEquals("_____Another_ONE__", field4.name());
    replacedertEquals(1, field4.aliases().size());
    replacedertTrue(field4.aliases().contains("  __ Another ONE!!"));
}

18 View Complete Implementation : SalesforceAvroRegistryTest.java
Copyright Apache License 2.0
Author : Talend
@Test
public void testStringToDoubleConverter() {
    Schema schema = // 
    SchemaBuilder.builder().record("Schema").fields().name("Field_A").type(AvroUtils._double()).noDefault().endRecord();
    AvroConverter<String, ?> converter = sRegistry.getConverterFromString(schema.getField("Field_A"));
    replacedertNotNull(converter);
    replacedertEquals(String.clreplaced, converter.getDatumClreplaced());
    replacedertEquals(schema.getField("Field_A").schema(), converter.getSchema());
    Object value = converter.convertToAvro("102030405060.12345");
    replacedertNotNull(value);
    replacedertThat(value, instanceOf(Double.clreplaced));
    replacedertThat((Double) value, equalTo(Double.valueOf(102030405060.12345)));
    replacedertNull(converter.convertToAvro(""));
    replacedertNull(converter.convertToAvro(null));
}

18 View Complete Implementation : TSalesforceBulkExecPropertiesTest.java
Copyright Apache License 2.0
Author : Talend
@Test
public void testAfterSchema() throws Throwable {
    properties.init();
    properties.module.moduleName.setValue("Account");
    properties.module.main.schema.setValue(DEFAULT_SCHEMA);
    propertiesService.afterProperty("schema", properties.module.main);
    Schema schema = properties.getSchema();
    Schema flowSchema = properties.schemaFlow.schema.getValue();
    for (Schema.Field inputField : schema.getFields()) {
        Schema.Field field = flowSchema.getField(inputField.name());
        replacedertNotNull(field);
        replacedertEquals(inputField, field);
    }
    replacedertNotNull(flowSchema.getField("salesforce_id"));
    replacedertEquals("true", flowSchema.getField("salesforce_id").getProp(SchemaConstants.TALEND_FIELD_GENERATED));
    replacedertNotNull(flowSchema.getField("salesforce_created"));
    replacedertEquals("true", flowSchema.getField("salesforce_created").getProp(SchemaConstants.TALEND_FIELD_GENERATED));
    Schema rejectSchema = properties.schemaReject.schema.getValue();
    for (Schema.Field inputField : schema.getFields()) {
        Schema.Field field = rejectSchema.getField(inputField.name());
        replacedertNotNull(field);
        replacedertEquals(inputField, field);
    }
    replacedertNotNull(rejectSchema.getField("error"));
    replacedertEquals("true", rejectSchema.getField("error").getProp(SchemaConstants.TALEND_FIELD_GENERATED));
    replacedertThat(properties.upsertKeyColumn.getPossibleValues(), empty());
    replacedertThat((Iterable<String>) properties.upsertRelationTable.columnName.getPossibleValues(), contains("Id", "Name"));
}

18 View Complete Implementation : MarketoOutputWriterTestIT.java
Copyright Apache License 2.0
Author : Talend
public void testSyncLead(IndexedRecord record) throws Exception {
    props.outputOperation.setValue(OutputOperation.syncLead);
    writer = getWriter(props);
    writer.open("test");
    writer.write(record);
    MarketoResult result = (MarketoResult) writer.close();
    replacedertEquals(1, result.getApiCalls());
    replacedertEquals(1, result.getSuccessCount());
    replacedertEquals(0, result.getRejectCount());
    List<IndexedRecord> successes = writer.getSuccessfulWrites();
    List<IndexedRecord> rejects = writer.getRejectedWrites();
    replacedertEquals(Collections.emptyList(), rejects);
    replacedertEquals(1, successes.size());
    IndexedRecord success = successes.get(0);
    Schema s = props.schemaFlow.schema.getValue();
    replacedertNotNull(s);
    Integer leadId = (int) success.get(0);
    createdLeads.add(leadId);
    replacedertEquals("CREATED", success.get(s.getField("Status").pos()).toString().toUpperCase());
    LOG.debug("Added leadId = {} to createdLeads {}.", leadId, createdLeads);
}

18 View Complete Implementation : SchemaGeneratorUtils.java
Copyright Apache License 2.0
Author : Talend
/**
 * Merge a KV-SChema into a single schema
 *
 * @param inputSchema a Schema
 * @return an avro Schema resulting in a merge of the key and the value part of the KV schema
 */
public static Schema mergeKeyValues(Schema inputSchema) {
    return mergeKeyValues(inputSchema.getField(RECORD_KEY_PREFIX).schema(), inputSchema.getField(RECORD_VALUE_PREFIX).schema());
}

18 View Complete Implementation : TestNiFiOrcUtils.java
Copyright Apache License 2.0
Author : apache
@Test
public void test_getOrcField_union() throws Exception {
    final SchemaBuilder.Fieldreplacedembler<Schema> builder = SchemaBuilder.record("testRecord").namespace("any.data").fields();
    builder.name("union").type().unionOf().intType().and().booleanType().endUnion().noDefault();
    Schema testSchema = builder.endRecord();
    TypeInfo orcType = NiFiOrcUtils.getOrcField(testSchema.getField("union").schema());
    replacedertEquals(TypeInfoFactory.getUnionTypeInfo(Arrays.asList(TypeInfoCreator.createInt(), TypeInfoCreator.createBoolean())), orcType);
}

18 View Complete Implementation : RootSchemaUtils.java
Copyright Apache License 2.0
Author : Talend
/**
 * Retrieves and returns Out of band data schema from Root schema.
 * If incoming schema is not Root schema throws an {@link IllegalArgumentException}
 *
 * @param rootSchema Root schema
 * @return Main data schema
 */
public static Schema getOutOfBandSchema(Schema rootSchema) {
    if (!isRootSchema(rootSchema)) {
        throw new IllegalArgumentException("Input schema should be Root schema");
    }
    Schema outOfBandSchema = rootSchema.getField(OUTOFBAND_FIELD_NAME).schema();
    return outOfBandSchema;
}

17 View Complete Implementation : SalesforceAvroRegistryTest.java
Copyright Apache License 2.0
Author : Talend
@Test
public void testStringToBytesConverter() {
    Schema schema = // 
    SchemaBuilder.builder().record("Schema").fields().name("Field_A").type(AvroUtils._bytes()).noDefault().endRecord();
    AvroConverter<String, ?> converter = sRegistry.getConverterFromString(schema.getField("Field_A"));
    replacedertNotNull(converter);
    replacedertEquals(String.clreplaced, converter.getDatumClreplaced());
    replacedertEquals(schema.getField("Field_A").schema(), converter.getSchema());
    Object value = converter.convertToAvro("a1b2");
    replacedertNotNull(value);
    replacedertThat(value, instanceOf(byte[].clreplaced));
    replacedertArrayEquals(new byte[] { 97, 49, 98, 50 }, (byte[]) value);
    replacedertNull(converter.convertToAvro(null));
}

17 View Complete Implementation : MarketoConstantsTest.java
Copyright Apache License 2.0
Author : Talend
// Custom Objects
@Test
public void testCustomObjectsSyncSchema() throws Exception {
    Schema ref = MarketoConstants.getCustomObjectSyncSchema();
    replacedertEquals("STRING", ref.getField("marketoGUID").schema().getType().toString());
    replacedertEquals("true", ref.getField("marketoGUID").getProp(SchemaConstants.TALEND_COLUMN_IS_KEY));
}

17 View Complete Implementation : MarketoConstantsTest.java
Copyright Apache License 2.0
Author : Talend
@Test
public void testTriggerCampaignSchema() throws Exception {
    Schema ref = MarketoConstants.triggerCampaignSchema();
    replacedertEquals("INT", ref.getField("leadId").schema().getTypes().get(0).getType().toString());
    replacedertEquals("true", ref.getField("leadId").getProp(SchemaConstants.TALEND_COLUMN_IS_KEY));
}

17 View Complete Implementation : MarketoConstantsTest.java
Copyright Apache License 2.0
Author : Talend
@Test
public void testScheduleCampaignSchema() throws Exception {
    Schema ref = MarketoConstants.scheduleCampaignSchema();
    replacedertEquals("INT", ref.getField("campaignId").schema().getTypes().get(0).getType().toString());
    replacedertEquals("true", ref.getField("campaignId").getProp(SchemaConstants.TALEND_COLUMN_IS_KEY));
    replacedertEquals("STRING", ref.getField("Status").schema().getTypes().get(0).getType().toString());
}

17 View Complete Implementation : SalesforceAvroRegistryTest.java
Copyright Apache License 2.0
Author : Talend
@Test
public void testStringToBooleanConverter() {
    Schema schema = // 
    SchemaBuilder.builder().record("Schema").fields().name("Field_A").type(AvroUtils._boolean()).noDefault().endRecord();
    AvroConverter<String, ?> converter = sRegistry.getConverterFromString(schema.getField("Field_A"));
    replacedertNotNull(converter);
    replacedertEquals(String.clreplaced, converter.getDatumClreplaced());
    replacedertEquals(schema.getField("Field_A").schema(), converter.getSchema());
    Object value = converter.convertToAvro("true");
    replacedertNotNull(value);
    replacedertThat(value, instanceOf(Boolean.clreplaced));
    replacedertThat((Boolean) value, equalTo(Boolean.TRUE));
    replacedertNull(converter.convertToAvro(""));
    replacedertNull(converter.convertToAvro(null));
}

17 View Complete Implementation : MarketoConstantsTest.java
Copyright Apache License 2.0
Author : Talend
@Test
public void testOutputSchemaForSyncLeadREST() throws Exception {
    Schema ref = MarketoConstants.getRESTOutputSchemaForSyncLead();
    replacedertEquals("INT", ref.getField("id").schema().getTypes().get(0).getType().toString());
    replacedertEquals("true", ref.getField("id").getProp(SchemaConstants.TALEND_COLUMN_IS_KEY));
    replacedertEquals("STRING", ref.getField("email").schema().getTypes().get(0).getType().toString());
    replacedertEquals("STRING", ref.getField("firstName").schema().getTypes().get(0).getType().toString());
    replacedertEquals("STRING", ref.getField("lastName").schema().getTypes().get(0).getType().toString());
}

17 View Complete Implementation : MarketoConstantsTest.java
Copyright Apache License 2.0
Author : Talend
@Test
public void testSyncLeadOutputSchemaSOAP() throws Exception {
    Schema ref = MarketoConstants.getSOAPOutputSchemaForSyncLead();
    replacedertEquals("INT", ref.getField("Id").schema().getTypes().get(0).getType().toString());
    replacedertEquals("true", ref.getField("Id").getProp(SchemaConstants.TALEND_COLUMN_IS_KEY));
    replacedertEquals("STRING", ref.getField("Email").schema().getTypes().get(0).getType().toString());
    replacedertEquals("STRING", ref.getField("ForeignSysPersonId").schema().getTypes().get(0).getType().toString());
    replacedertEquals("STRING", ref.getField("ForeignSysType").schema().getTypes().get(0).getType().toString());
}

17 View Complete Implementation : SnowflakeRowReader.java
Copyright Apache License 2.0
Author : Talend
private void handleReject(SQLException e) {
    if (schemaReject == null) {
        schemaReject = properties.schemaReject.schema.getValue();
    }
    IndexedRecord reject = new GenericData.Record(schemaReject);
    reject.put(schemaReject.getField("errorCode").pos(), e.getSQLState());
    reject.put(schemaReject.getField("errorMessage").pos(), e.getMessage());
    rejectInfo = new HashMap<String, Object>();
    rejectInfo.put("error", e.getMessage());
    // Since Studio or Framework handles rejects in its own way.
    rejectInfo.put("errorCode", e.getSQLState());
    rejectInfo.put("errorMessage", e.getMessage());
    rejectInfo.put("exception", e.toString());
    rejectInfo.put("talend_record", reject);
    isRejectError = true;
}

17 View Complete Implementation : SalesforceAnyTypeFieldTestIT.java
Copyright Apache License 2.0
Author : Talend
public void testQueryAnyTypeHistory(boolean isBulkQuery, boolean isRetrieveSchema) throws Exception {
    TSalesforceInputProperties inputProps = (TSalesforceInputProperties) new TSalesforceInputProperties("bar").init();
    setupProps(inputProps.connection, !ADD_QUOTES);
    inputProps.module.moduleName.setValue(CUSTOM_MODULE_HISTORY);
    if (isRetrieveSchema) {
        inputProps.module.beforeModuleName();
        inputProps.module.afterModuleName();
    } else {
        inputProps.module.main.schema.setValue(SCHEMA_QUERY_HISTORY);
    }
    if (isBulkQuery) {
        inputProps.queryMode.setValue(TSalesforceInputProperties.QueryMode.Bulk);
    } else {
        inputProps.queryMode.setValue(TSalesforceInputProperties.QueryMode.Query);
    }
    inputProps.batchSize.setValue(200);
    inputProps.condition.setValue("ParentId='" + recordId + "' and Field != 'created'");
    List<IndexedRecord> records = readRows(inputProps);
    replacedertEquals(5, records.size());
    Schema runtimeSchema = records.get(0).getSchema();
    for (IndexedRecord record : records) {
        if (BOOLEAN_COLUMN_NAME.equals(record.get(runtimeSchema.getField("Field").pos()))) {
            replacedertEquals("false", record.get(runtimeSchema.getField("OldValue").pos()));
            replacedertEquals("true", record.get(runtimeSchema.getField("NewValue").pos()));
        }
        if (CURRENCY_COLUMN_NAME.equals(record.get(runtimeSchema.getField("Field").pos()))) {
            replacedertEquals("7.251", record.get(runtimeSchema.getField("OldValue").pos()));
            replacedertEquals("7.252", record.get(runtimeSchema.getField("NewValue").pos()));
        }
        if (DATE_COLUMN_NAME.equals(record.get(runtimeSchema.getField("Field").pos()))) {
            replacedertEquals("2018-07-25", record.get(runtimeSchema.getField("OldValue").pos()));
            replacedertEquals("2018-07-26", record.get(runtimeSchema.getField("NewValue").pos()));
        }
        if (DATETIME_COLUMN_NAME.equals(record.get(runtimeSchema.getField("Field").pos()))) {
            replacedertEquals("2018-07-25T10:10:10.000Z", record.get(runtimeSchema.getField("OldValue").pos()));
            replacedertEquals("2018-07-25T11:11:11.000Z", record.get(runtimeSchema.getField("NewValue").pos()));
        }
        if (PERCENT_COLUMN_NAME.equals(record.get(runtimeSchema.getField("Field").pos()))) {
            replacedertEquals("7.251", record.get(runtimeSchema.getField("OldValue").pos()));
            replacedertEquals("7.252", record.get(runtimeSchema.getField("NewValue").pos()));
        }
    }
}

17 View Complete Implementation : SalesforceAvroRegistryTest.java
Copyright Apache License 2.0
Author : Talend
@Test
public void testStringToIntegerConverter() {
    Schema schema = // 
    SchemaBuilder.builder().record("Schema").fields().name("Field_A").type(AvroUtils._int()).noDefault().endRecord();
    AvroConverter<String, ?> converter = sRegistry.getConverterFromString(schema.getField("Field_A"));
    replacedertNotNull(converter);
    replacedertEquals(String.clreplaced, converter.getDatumClreplaced());
    replacedertEquals(schema.getField("Field_A").schema(), converter.getSchema());
    Object value = converter.convertToAvro("1020304050");
    replacedertNotNull(value);
    replacedertThat(value, instanceOf(Integer.clreplaced));
    replacedertThat((Integer) value, equalTo(Integer.valueOf(1020304050)));
    replacedertNull(converter.convertToAvro(""));
    replacedertNull(converter.convertToAvro(null));
}

17 View Complete Implementation : MarketoConstantsTest.java
Copyright Apache License 2.0
Author : Talend
@Test
public void testTriggerCampaignSchemaFlow() throws Exception {
    Schema ref = MarketoConstants.triggerCampaignSchemaFlow();
    replacedertEquals("INT", ref.getField("campaignId").schema().getTypes().get(0).getType().toString());
    replacedertEquals("true", ref.getField("campaignId").getProp(SchemaConstants.TALEND_COLUMN_IS_KEY));
}

17 View Complete Implementation : SalesforceAvroRegistryTest.java
Copyright Apache License 2.0
Author : Talend
@Test
public void testInferSchemaWithReferenceField() {
    Field referenceField = new Field();
    referenceField.setName("reference");
    referenceField.setType(FieldType.string);
    referenceField.setReferenceTo(new String[] { "SomeRecord" });
    referenceField.setRelationshipName("relationship");
    DescribeSObjectResult dsor = new DescribeSObjectResult();
    dsor.setName("MySObjectRecord");
    dsor.setFields(new Field[] { referenceField });
    Schema schema = sRegistry.inferSchema(dsor);
    Schema.Field field = schema.getField("reference");
    replacedertThat(field.schema().getType(), is(Schema.Type.STRING));
    replacedertThat(field.getProp(SalesforceSchemaConstants.REF_MODULE_NAME), is("SomeRecord"));
    replacedertThat(field.getProp(SalesforceSchemaConstants.REF_FIELD_NAME), is("relationship"));
}

17 View Complete Implementation : MarketoConstantsTest.java
Copyright Apache License 2.0
Author : Talend
@Test
public void testListOperationSchemaSOAP() throws Exception {
    Schema ref = MarketoConstants.getListOperationSOAPSchema();
    replacedertEquals("STRING", ref.getField("ListKeyType").schema().getTypes().get(0).getType().toString());
    replacedertEquals("STRING", ref.getField("ListKeyValue").schema().getTypes().get(0).getType().toString());
    replacedertEquals("STRING", ref.getField("LeadKeyType").schema().getTypes().get(0).getType().toString());
    replacedertEquals("STRING", ref.getField("LeadKeyValue").schema().getTypes().get(0).getType().toString());
}

17 View Complete Implementation : MarketoConstantsTest.java
Copyright Apache License 2.0
Author : Talend
/*
     * SOAP
     */
@Test
public void testLeadSchemaSOAP() throws Exception {
    Schema ref = MarketoConstants.getSOAPSchemaForGetLeadOrGetMultipleLeads();
    replacedertEquals("INT", ref.getField("Id").schema().getTypes().get(0).getType().toString());
    replacedertEquals("true", ref.getField("Id").getProp(SchemaConstants.TALEND_COLUMN_IS_KEY));
    replacedertEquals("STRING", ref.getField("Email").schema().getTypes().get(0).getType().toString());
    replacedertEquals("STRING", ref.getField("ForeignSysPersonId").schema().getTypes().get(0).getType().toString());
    replacedertEquals("STRING", ref.getField("ForeignSysType").schema().getTypes().get(0).getType().toString());
}

17 View Complete Implementation : SalesforceAnyTypeFieldTestIT.java
Copyright Apache License 2.0
Author : Talend
private void inserRecord() throws Throwable {
    TSalesforceOutputProperties outProps = getOutputProperties(CUSTOM_MODULE);
    outProps.extendInsert.setValue(false);
    outProps.retrieveInsertId.setValue(true);
    outProps.afterRetrieveInsertId();
    outProps.outputAction.setValue(TSalesforceOutputProperties.OutputAction.INSERT);
    List<IndexedRecord> outputRows = new ArrayList<>();
    GenericData.Record row = new GenericData.Record(SCHEMA_CUSTOM_MODULE);
    row.put("Name", randomizedValue);
    row.put(BOOLEAN_COLUMN_NAME, false);
    row.put(CURRENCY_COLUMN_NAME, new BigDecimal("7.251"));
    row.put(DATE_COLUMN_NAME, dateFormat.parse("2018-07-25"));
    row.put(DATETIME_COLUMN_NAME, dateTimeFormat.parse("2018-07-25T10:10:10.000Z"));
    row.put(PERCENT_COLUMN_NAME, 7.251);
    outputRows.add(row);
    List<IndexedRecord> successRecords = writeRows(CUSTOM_MODULE, outputRows, outProps);
    replacedertNotNull(successRecords);
    replacedertEquals(1, successRecords.size());
    IndexedRecord record = successRecords.get(0);
    Schema schema = record.getSchema();
    Schema.Field idField = schema.getField("salesforce_id");
    replacedertNotNull(idField);
    Object idObj = record.get(idField.pos());
    replacedertNotNull(idObj);
    recordId = String.valueOf(idObj);
}

17 View Complete Implementation : MarketoConstantsTest.java
Copyright Apache License 2.0
Author : Talend
/*
     * REST
     */
@Test
public void testListOperationSchemaREST() throws Exception {
    Schema ref = MarketoConstants.getListOperationRESTSchema();
    replacedertEquals("INT", ref.getField("ListId").schema().getTypes().get(0).getType().toString());
    replacedertEquals("INT", ref.getField("LeadId").schema().getTypes().get(0).getType().toString());
}

17 View Complete Implementation : FileDelimitedReaderTestIT.java
Copyright Apache License 2.0
Author : Talend
// Test FileInputDelimited component read with delimited mode and source is compressed file
@Test
public void testGetDynamicSchema() throws Throwable {
    // Include all field in "ALL_FIELDS_NAME", and default type is String
    Schema schemaOnlyDynamic = FileSourceOrSink.getDynamicSchema(ALL_FIELDS_NAME, "dynamic", BASIC_DYNAMIC_SCHEMA);
    replacedert.replacedertEquals(11, schemaOnlyDynamic.getFields().size());
    replacedert.replacedertEquals("TestBoolean", schemaOnlyDynamic.getFields().get(0).name());
    replacedert.replacedertTrue(AvroUtils.isSameType(AvroUtils._string(), schemaOnlyDynamic.getFields().get(0).schema()));
    replacedert.replacedertEquals("TestObject", schemaOnlyDynamic.getFields().get(10).name());
    replacedert.replacedertTrue(AvroUtils.isSameType(AvroUtils._string(), schemaOnlyDynamic.getFields().get(10).schema()));
    // Column index 0~9 are included in dynamic field. The last field "test_end" is from "DYNAMIC_IS_FIRST_SCHEMA"
    Schema schemaDynamicInFirst = FileSourceOrSink.getDynamicSchema(ALL_FIELDS_NAME, "dynamic", DYNAMIC_IS_FIRST_SCHEMA);
    replacedert.replacedertEquals(11, schemaDynamicInFirst.getFields().size());
    replacedert.replacedertEquals("TestBoolean", schemaDynamicInFirst.getFields().get(0).name());
    replacedert.replacedertTrue(AvroUtils.isSameType(AvroUtils._string(), schemaDynamicInFirst.getFields().get(0).schema()));
    replacedert.replacedertEquals("test_end", schemaDynamicInFirst.getFields().get(10).name());
    replacedert.replacedertTrue(AvroUtils.isSameType(AvroUtils.wrapAsNullable(AvroUtils._int()), schemaDynamicInFirst.getFields().get(10).schema()));
    // Column index 1~9 are included in dynamic field. The first field "test_begin" is from DYNAMIC_IS_MID_SCHEMA,The last
    // field "test_end" is from "DYNAMIC_IS_MID_SCHEMA"
    Schema schemaDynamicInMiddle = FileSourceOrSink.getDynamicSchema(ALL_FIELDS_NAME, "dynamic", DYNAMIC_IS_MID_SCHEMA);
    replacedert.replacedertEquals(11, schemaDynamicInMiddle.getFields().size());
    replacedert.replacedertEquals("test_begin", schemaDynamicInMiddle.getFields().get(0).name());
    replacedert.replacedertTrue(AvroUtils.isSameType(AvroUtils.wrapAsNullable(AvroUtils._int()), schemaDynamicInMiddle.getFields().get(0).schema()));
    replacedert.replacedertEquals("TestByte", schemaDynamicInMiddle.getFields().get(1).name());
    replacedert.replacedertTrue(AvroUtils.isSameType(AvroUtils._string(), schemaDynamicInMiddle.getFields().get(1).schema()));
    replacedert.replacedertEquals("TestLong", schemaDynamicInMiddle.getFields().get(9).name());
    replacedert.replacedertTrue(AvroUtils.isSameType(AvroUtils._string(), schemaDynamicInMiddle.getFields().get(9).schema()));
    replacedert.replacedertEquals(10, schemaDynamicInFirst.getField("test_end").pos());
    replacedert.replacedertTrue(AvroUtils.isSameType(AvroUtils.wrapAsNullable(AvroUtils._int()), schemaDynamicInMiddle.getFields().get(10).schema()));
    // Column index 1~10 are included in dynamic field. The first field "test_begin" is from "DYNAMIC_IS_END_SCHEMA"
    Schema schemaDynamicInEnd = FileSourceOrSink.getDynamicSchema(ALL_FIELDS_NAME, "dynamic", DYNAMIC_IS_END_SCHEMA);
    replacedert.replacedertEquals(11, schemaDynamicInEnd.getFields().size());
    replacedert.replacedertEquals("test_begin", schemaDynamicInEnd.getFields().get(0).name());
    replacedert.replacedertTrue(AvroUtils.isSameType(AvroUtils.wrapAsNullable(AvroUtils._int()), schemaDynamicInEnd.getFields().get(0).schema()));
    replacedert.replacedertEquals("TestObject", schemaDynamicInEnd.getFields().get(10).name());
    replacedert.replacedertTrue(AvroUtils.isSameType(AvroUtils._string(), schemaDynamicInEnd.getFields().get(10).schema()));
}

16 View Complete Implementation : TMarketoListOperationPropertiesTest.java
Copyright Apache License 2.0
Author : Talend
@Test
public void testUpdateOutputSchemas() throws Exception {
    Schema s;
    int mainFieldsCount = props.schemaInput.schema.getValue().getFields().size();
    // REST
    props.updateOutputSchemas();
    s = props.schemaFlow.schema.getValue();
    replacedertEquals(MarketoConstants.getListOperationFlowRESTSchema(), s);
    s = props.schemaReject.schema.getValue();
    replacedertEquals(MarketoConstants.getListOperationRejectRESTSchema(), s);
    // SOAP
    props.connection.apiMode.setValue(APIMode.SOAP);
    props.afterListOperation();
    props.updateOutputSchemas();
    mainFieldsCount = props.schemaInput.schema.getValue().getFields().size();
    s = props.schemaFlow.schema.getValue();
    replacedertEquals(MarketoConstants.getListOperationFlowSOAPSchema(), s);
    replacedertNotNull(s.getField("Success"));
    replacedertTrue(s.getField("Success").schema().getTypes().get(0).getType().equals(Schema.Type.BOOLEAN));
    s = props.schemaReject.schema.getValue();
    replacedertEquals(MarketoConstants.getListOperationRejectSOAPSchema(), s);
    replacedertNotNull(s.getField("ERROR_MSG"));
    replacedertTrue(s.getField("ERROR_MSG").schema().getTypes().get(0).getType().equals(Schema.Type.STRING));
    props.listOperation.setValue(ListOperation.removeFrom);
    props.multipleOperation.setValue(true);
    props.afterMultipleOperation();
    replacedertEquals(props.schemaInput.schema.getValue().getFields().size(), props.schemaFlow.schema.getValue().getFields().size());
}

16 View Complete Implementation : SalesforceAvroRegistryTest.java
Copyright Apache License 2.0
Author : Talend
@Test
public void testStringToDateConverter() throws ParseException {
    Schema schema = // 
    SchemaBuilder.builder().record("Schema").fields().name("Field_A").type(AvroUtils._date()).noDefault().endRecord();
    schema.getField("Field_A").addProp(SchemaConstants.TALEND_COLUMN_PATTERN, "yyyy-MM-dd");
    AvroConverter<String, ?> converter = sRegistry.getConverterFromString(schema.getField("Field_A"));
    replacedertNotNull(converter);
    replacedertEquals(String.clreplaced, converter.getDatumClreplaced());
    replacedertEquals(schema.getField("Field_A").schema(), converter.getSchema());
    Object value = converter.convertToAvro("2017-06-15");
    replacedertNotNull(value);
    replacedertThat(value, instanceOf(Long.clreplaced));
    replacedertThat((Long) value, equalTo(Long.valueOf(dateFormat.parse("2017-06-15").getTime())));
    replacedertNull(converter.convertToAvro(""));
    replacedertNull(converter.convertToAvro(null));
    Date date = dateFormat.parse("2017-06-15");
    String sValue = ((AvroConverter<String, Long>) converter).convertToDatum(Long.valueOf(date.getTime()));
    replacedertNotNull(sValue);
    replacedertEquals(dateFormat.format(date), sValue);
    replacedertNull(converter.convertToDatum(null));
}

16 View Complete Implementation : NetSuiteOutputWriter.java
Copyright Apache License 2.0
Author : Talend
/**
 * Create record for outgoing {@code reject} flow.
 *
 * @param response write response
 * @param record indexed record which was submitted
 * @return result record
 */
private IndexedRecord createRejectRecord(NsWriteResponse<RefT> response, IndexedRecord record) {
    GenericData.Record targetRecord = new GenericData.Record(rejectSchema);
    for (Schema.Field field : schema.getFields()) {
        Schema.Field targetField = rejectSchema.getField(field.name());
        if (targetField != null) {
            Object value = record.get(field.pos());
            targetRecord.put(targetField.name(), value);
        }
    }
    String errorCode;
    String errorMessage;
    NsStatus status = response.getStatus();
    if (!status.getDetails().isEmpty()) {
        errorCode = status.getDetails().get(0).getCode();
        errorMessage = status.getDetails().get(0).getMessage();
    } else {
        errorCode = "GENERAL_ERROR";
        errorMessage = "Operation failed";
    }
    Schema.Field errorCodeField = NetSuiteDatasetRuntimeImpl.getNsFieldByName(rejectSchema, "errorCode");
    if (errorCodeField != null) {
        targetRecord.put(errorCodeField.pos(), errorCode);
    }
    Schema.Field errorMessageField = NetSuiteDatasetRuntimeImpl.getNsFieldByName(rejectSchema, "errorMessage");
    if (errorMessageField != null) {
        targetRecord.put(errorMessageField.pos(), errorMessage);
    }
    return targetRecord;
}

16 View Complete Implementation : CouchbaseWriter.java
Copyright Apache License 2.0
Author : Talend
@Override
public void write(Object datum) throws IOException {
    if (!opened) {
        throw new IOException("Writer is not opened");
    }
    result.totalCount++;
    if (datum == null) {
        return;
    }
    // Data object is always IndexedRecord
    IndexedRecord record = (IndexedRecord) datum;
    Schema schema = record.getSchema();
    Schema.Field idField = schema.getField(idFieldName);
    if (idField == null) {
        throw new IOException("Schema does not contain ID field: " + idFieldName);
    }
    int idPos = idField.pos();
    Object id = record.get(idPos);
    if (id == null) {
        handleException("Record is not processed. ID is null.", new IllegalArgumentException("ID field should not be null"));
        return;
    }
    try {
        if (containsJson) {
            connection.insertJsonDoreplacedent(id.toString(), createHierarchicalJson(schema, record, idPos));
        } else {
            connection.upsert(id.toString(), datum.toString());
        }
        result.successCount++;
    } catch (Exception e) {
        handleException("Record is not processed. Failed to upsert value - " + datum.toString(), e);
    }
}

16 View Complete Implementation : NormalizeUtils.java
Copyright Apache License 2.0
Author : Talend
/**
 * Get the child schema of a field, and check if this is correctly a Record. If not, thow a TalendRuntimeException.
 *
 * @param parentSchema the schema of the parent element
 * @param field the field to extract
 * @return the schema of the element extracted
 */
public static Schema getChildSchemaAsRecord(Schema parentSchema, Schema.Field field) {
    Schema childSchema = AvroUtils.unwrapIfNullable(parentSchema.getField(field.name()).schema());
    if (childSchema.getType().equals(Schema.Type.RECORD)) {
        return childSchema;
    } else {
        throw new TalendRuntimeException(CommonErrorCodes.UNEXPECTED_EXCEPTION, new Throwable(String.format("The field %s has the type %s but should be a Record on the schema %s", field.name(), childSchema.getType(), parentSchema.toString())));
    }
}