org.apache.iceberg.Schema - java examples

Here are the examples of the java api org.apache.iceberg.Schema taken from open source projects. By voting up you can indicate which examples are most useful and appropriate.

155 Examples 7

18 View Complete Implementation : TestTruncatesProjection.java
Copyright Apache License 2.0
Author : apache
@Test
public void testLongStrictUpperBound() {
    Long value = 99L;
    Schema schema = new Schema(optional(1, "value", Types.LongType.get()));
    ParreplacedionSpec spec = ParreplacedionSpec.builderFor(schema).truncate("value", 10).build();
    replacedertProjectionStrict(spec, lessThan("value", value), Expression.Operation.LT, "90");
    replacedertProjectionStrict(spec, lessThanOrEqual("value", value), Expression.Operation.LT, "100");
    replacedertProjectionStrict(spec, greaterThan("value", value), Expression.Operation.GT, "90");
    replacedertProjectionStrict(spec, greaterThanOrEqual("value", value), Expression.Operation.GT, "90");
    replacedertProjectionStrict(spec, notEqual("value", value), Expression.Operation.NOT_EQ, "90");
    replacedertProjectionStrictValue(spec, equal("value", value), Expression.Operation.FALSE);
    replacedertProjectionStrict(spec, notIn("value", value - 1, value, value + 1), Expression.Operation.NOT_IN, "[90, 90, 100]");
    replacedertProjectionStrictValue(spec, in("value", value, value - 1), Expression.Operation.FALSE);
}

18 View Complete Implementation : TestSparkParquetWriter.java
Copyright Apache License 2.0
Author : apache
public clreplaced TestSparkParquetWriter {

    @Rule
    public TemporaryFolder temp = new TemporaryFolder();

    private static final Schema COMPLEX_SCHEMA = new Schema(required(1, "roots", Types.LongType.get()), optional(3, "lime", Types.ListType.ofRequired(4, Types.DoubleType.get())), required(5, "strict", Types.StructType.of(required(9, "tangerine", Types.StringType.get()), optional(6, "hopeful", Types.StructType.of(required(7, "steel", Types.FloatType.get()), required(8, "lantern", Types.DateType.get()))), optional(10, "vehement", Types.LongType.get()))), optional(11, "metamorphosis", Types.MapType.ofRequired(12, 13, Types.StringType.get(), Types.TimestampType.withZone())), required(14, "winter", Types.ListType.ofOptional(15, Types.StructType.of(optional(16, "beet", Types.DoubleType.get()), required(17, "stamp", Types.FloatType.get()), optional(18, "wheeze", Types.StringType.get())))), optional(19, "renovate", Types.MapType.ofRequired(20, 21, Types.StringType.get(), Types.StructType.of(optional(22, "jumpy", Types.DoubleType.get()), required(23, "koala", Types.IntegerType.get()), required(24, "couch rope", Types.IntegerType.get())))), optional(2, "slide", Types.StringType.get()));

    @Test
    public void testCorrectness() throws IOException {
        int numRows = 250_000;
        Iterable<InternalRow> records = RandomData.generateSpark(COMPLEX_SCHEMA, numRows, 19981);
        File testFile = temp.newFile();
        replacedert.replacedertTrue("Delete should succeed", testFile.delete());
        try (FileAppender<InternalRow> writer = Parquet.write(Files.localOutput(testFile)).schema(COMPLEX_SCHEMA).createWriterFunc(msgType -> SparkParquetWriters.buildWriter(COMPLEX_SCHEMA, msgType)).build()) {
            writer.addAll(records);
        }
        try (CloseableIterable<InternalRow> reader = Parquet.read(Files.localInput(testFile)).project(COMPLEX_SCHEMA).createReaderFunc(type -> SparkParquetReaders.buildReader(COMPLEX_SCHEMA, type)).build()) {
            Iterator<InternalRow> expected = records.iterator();
            Iterator<InternalRow> rows = reader.iterator();
            for (int i = 0; i < numRows; i += 1) {
                replacedert.replacedertTrue("Should have expected number of rows", rows.hasNext());
                TestHelpers.replacedertEquals(COMPLEX_SCHEMA, expected.next(), rows.next());
            }
            replacedert.replacedertFalse("Should not have extra rows", rows.hasNext());
        }
    }
}

18 View Complete Implementation : TestTruncatesProjection.java
Copyright Apache License 2.0
Author : apache
@Test
public void testStringStrict() {
    String value = "abcdefg";
    Schema schema = new Schema(optional(1, "value", Types.StringType.get()));
    ParreplacedionSpec spec = ParreplacedionSpec.builderFor(schema).truncate("value", 5).build();
    replacedertProjectionStrict(spec, lessThan("value", value), Expression.Operation.LT, "abcde");
    replacedertProjectionStrict(spec, lessThanOrEqual("value", value), Expression.Operation.LT, "abcde");
    replacedertProjectionStrict(spec, greaterThan("value", value), Expression.Operation.GT, "abcde");
    replacedertProjectionStrict(spec, greaterThanOrEqual("value", value), Expression.Operation.GT, "abcde");
    replacedertProjectionStrict(spec, notEqual("value", value), Expression.Operation.NOT_EQ, "abcde");
    replacedertProjectionStrictValue(spec, equal("value", value), Expression.Operation.FALSE);
    replacedertProjectionStrict(spec, notIn("value", value, value + "abc"), Expression.Operation.NOT_IN, "[abcde, abcde]");
    replacedertProjectionStrictValue(spec, in("value", value, value + "abc"), Expression.Operation.FALSE);
}

18 View Complete Implementation : TestBucketingProjection.java
Copyright Apache License 2.0
Author : apache
@Test
public void testBucketIntegerStrict() {
    Integer value = 100;
    Schema schema = new Schema(optional(1, "value", Types.IntegerType.get()));
    ParreplacedionSpec spec = ParreplacedionSpec.builderFor(schema).bucket("value", 10).build();
    // the bucket number of the value (i.e. 100) is 6
    replacedertProjectionStrict(spec, notEqual("value", value), Expression.Operation.NOT_EQ, "6");
    replacedertProjectionStrictValue(spec, equal("value", value), Expression.Operation.FALSE);
    replacedertProjectionStrictValue(spec, lessThan("value", value), Expression.Operation.FALSE);
    replacedertProjectionStrictValue(spec, lessThanOrEqual("value", value), Expression.Operation.FALSE);
    replacedertProjectionStrictValue(spec, greaterThan("value", value), Expression.Operation.FALSE);
    replacedertProjectionStrictValue(spec, greaterThanOrEqual("value", value), Expression.Operation.FALSE);
    replacedertProjectionStrict(spec, notIn("value", value - 1, value, value + 1), Expression.Operation.NOT_IN, "[6, 7, 8]");
    replacedertProjectionStrictValue(spec, in("value", value, value + 1), Expression.Operation.FALSE);
}

18 View Complete Implementation : TestBucketingProjection.java
Copyright Apache License 2.0
Author : apache
// all types
@Test
public void testBucketLongStrict() {
    Long value = 100L;
    Schema schema = new Schema(optional(1, "value", Types.LongType.get()));
    ParreplacedionSpec spec = ParreplacedionSpec.builderFor(schema).bucket("value", 10).build();
    // the bucket number of the value (i.e. 100) is 6
    replacedertProjectionStrict(spec, notEqual("value", value), Expression.Operation.NOT_EQ, "6");
    replacedertProjectionStrictValue(spec, equal("value", value), Expression.Operation.FALSE);
    replacedertProjectionStrictValue(spec, lessThan("value", value), Expression.Operation.FALSE);
    replacedertProjectionStrictValue(spec, lessThanOrEqual("value", value), Expression.Operation.FALSE);
    replacedertProjectionStrictValue(spec, greaterThan("value", value), Expression.Operation.FALSE);
    replacedertProjectionStrictValue(spec, greaterThanOrEqual("value", value), Expression.Operation.FALSE);
    replacedertProjectionStrict(spec, notIn("value", value - 1, value, value + 1), Expression.Operation.NOT_IN, "[6, 7, 8]");
    replacedertProjectionStrictValue(spec, in("value", value, value + 1), Expression.Operation.FALSE);
}

18 View Complete Implementation : TestBucketingProjection.java
Copyright Apache License 2.0
Author : apache
@Test
public void testBucketIntegerInclusive() {
    Integer value = 100;
    Schema schema = new Schema(optional(1, "value", Types.IntegerType.get()));
    ParreplacedionSpec spec = ParreplacedionSpec.builderFor(schema).bucket("value", 10).build();
    // the bucket number of the value (i.e. 100) is 6
    replacedertProjectionInclusive(spec, equal("value", value), Expression.Operation.EQ, "6");
    replacedertProjectionInclusiveValue(spec, notEqual("value", value), Expression.Operation.TRUE);
    replacedertProjectionInclusiveValue(spec, lessThan("value", value), Expression.Operation.TRUE);
    replacedertProjectionInclusiveValue(spec, lessThanOrEqual("value", value), Expression.Operation.TRUE);
    replacedertProjectionInclusiveValue(spec, greaterThan("value", value), Expression.Operation.TRUE);
    replacedertProjectionInclusiveValue(spec, greaterThanOrEqual("value", value), Expression.Operation.TRUE);
    replacedertProjectionInclusive(spec, in("value", value - 1, value, value + 1), Expression.Operation.IN, "[6, 7, 8]");
    replacedertProjectionInclusiveValue(spec, notIn("value", value, value + 1), Expression.Operation.TRUE);
}

18 View Complete Implementation : TestBucketingProjection.java
Copyright Apache License 2.0
Author : apache
@Test
public void testBucketStringInclusive() {
    String value = "abcdefg";
    Schema schema = new Schema(optional(1, "value", Types.StringType.get()));
    ParreplacedionSpec spec = ParreplacedionSpec.builderFor(schema).bucket("value", 10).build();
    // the bucket number of the value (i.e. "abcdefg") is 4
    replacedertProjectionInclusive(spec, equal("value", value), Expression.Operation.EQ, "4");
    replacedertProjectionInclusiveValue(spec, notEqual("value", value), Expression.Operation.TRUE);
    replacedertProjectionInclusiveValue(spec, lessThan("value", value), Expression.Operation.TRUE);
    replacedertProjectionInclusiveValue(spec, lessThanOrEqual("value", value), Expression.Operation.TRUE);
    replacedertProjectionInclusiveValue(spec, greaterThan("value", value), Expression.Operation.TRUE);
    replacedertProjectionInclusiveValue(spec, greaterThanOrEqual("value", value), Expression.Operation.TRUE);
    replacedertProjectionInclusive(spec, in("value", value, value + "abc"), Expression.Operation.IN, "[4, 9]");
    replacedertProjectionInclusiveValue(spec, notIn("value", value, value + "abc"), Expression.Operation.TRUE);
}

18 View Complete Implementation : TestTruncatesProjection.java
Copyright Apache License 2.0
Author : apache
@Test
public void testIntegerStrictLowerBound() {
    Integer value = 100;
    Schema schema = new Schema(optional(1, "value", Types.IntegerType.get()));
    ParreplacedionSpec spec = ParreplacedionSpec.builderFor(schema).truncate("value", 10).build();
    replacedertProjectionStrict(spec, lessThan("value", value), Expression.Operation.LT, "100");
    replacedertProjectionStrict(spec, lessThanOrEqual("value", value), Expression.Operation.LT, "100");
    replacedertProjectionStrict(spec, greaterThan("value", value), Expression.Operation.GT, "100");
    replacedertProjectionStrict(spec, greaterThanOrEqual("value", value), Expression.Operation.GT, "90");
    replacedertProjectionStrict(spec, notEqual("value", value), Expression.Operation.NOT_EQ, "100");
    replacedertProjectionStrictValue(spec, equal("value", value), Expression.Operation.FALSE);
    replacedertProjectionStrict(spec, notIn("value", value - 1, value, value + 1), Expression.Operation.NOT_IN, "[90, 100, 100]");
    replacedertProjectionStrictValue(spec, in("value", value, value + 1), Expression.Operation.FALSE);
}

18 View Complete Implementation : ScanEvent.java
Copyright Apache License 2.0
Author : apache
/**
 * Event sent to listeners when a table scan is planned.
 */
public final clreplaced ScanEvent {

    private final String tableName;

    private final long snapshotId;

    private final Expression filter;

    private final Schema projection;

    public ScanEvent(String tableName, long snapshotId, Expression filter, Schema projection) {
        this.tableName = tableName;
        this.snapshotId = snapshotId;
        this.filter = filter;
        this.projection = projection;
    }

    public String tableName() {
        return tableName;
    }

    public long snapshotId() {
        return snapshotId;
    }

    public Expression filter() {
        return filter;
    }

    public Schema projection() {
        return projection;
    }
}

18 View Complete Implementation : TestTruncatesProjection.java
Copyright Apache License 2.0
Author : apache
@Test
public void testStringInclusive() {
    String value = "abcdefg";
    Schema schema = new Schema(optional(1, "value", Types.StringType.get()));
    ParreplacedionSpec spec = ParreplacedionSpec.builderFor(schema).truncate("value", 5).build();
    replacedertProjectionInclusive(spec, lessThan("value", value), Expression.Operation.LT_EQ, "abcde");
    replacedertProjectionInclusive(spec, lessThanOrEqual("value", value), Expression.Operation.LT_EQ, "abcde");
    replacedertProjectionInclusive(spec, greaterThan("value", value), Expression.Operation.GT_EQ, "abcde");
    replacedertProjectionInclusive(spec, greaterThanOrEqual("value", value), Expression.Operation.GT_EQ, "abcde");
    replacedertProjectionInclusive(spec, equal("value", value), Expression.Operation.EQ, "abcde");
    replacedertProjectionInclusiveValue(spec, notEqual("value", value), Expression.Operation.TRUE);
    replacedertProjectionInclusive(spec, in("value", value, value + "abc"), Expression.Operation.IN, "[abcde, abcde]");
    replacedertProjectionInclusiveValue(spec, notIn("value", value, value + "abc"), Expression.Operation.TRUE);
}

18 View Complete Implementation : TestTruncatesResiduals.java
Copyright Apache License 2.0
Author : apache
@Test
public void testStringTruncateTransformResiduals() {
    Schema schema = new Schema(Types.NestedField.optional(50, "value", Types.StringType.get()));
    // valid parreplacedions would be two letter strings for eg: ab, bc etc
    ParreplacedionSpec spec = ParreplacedionSpec.builderFor(schema).truncate("value", 2).build();
    // less than
    replacedertResidualValue(spec, lessThan("value", "bcd"), "ab", Expression.Operation.TRUE);
    replacedertResidualPredicate(spec, lessThan("value", "bcd"), "bc");
    replacedertResidualValue(spec, lessThan("value", "bcd"), "cd", Expression.Operation.FALSE);
    // less than equals
    replacedertResidualValue(spec, lessThanOrEqual("value", "bcd"), "ab", Expression.Operation.TRUE);
    replacedertResidualPredicate(spec, lessThanOrEqual("value", "bcd"), "bc");
    replacedertResidualValue(spec, lessThanOrEqual("value", "bcd"), "cd", Expression.Operation.FALSE);
    // greater than
    replacedertResidualValue(spec, greaterThan("value", "bcd"), "ab", Expression.Operation.FALSE);
    replacedertResidualPredicate(spec, greaterThan("value", "bcd"), "bc");
    replacedertResidualValue(spec, greaterThan("value", "bcd"), "cd", Expression.Operation.TRUE);
    // greater than
    replacedertResidualValue(spec, greaterThanOrEqual("value", "bcd"), "ab", Expression.Operation.FALSE);
    replacedertResidualPredicate(spec, greaterThanOrEqual("value", "bcd"), "bc");
    replacedertResidualValue(spec, greaterThanOrEqual("value", "bcd"), "cd", Expression.Operation.TRUE);
    // equal
    replacedertResidualValue(spec, equal("value", "bcd"), "ab", Expression.Operation.FALSE);
    replacedertResidualPredicate(spec, equal("value", "bcd"), "bc");
    replacedertResidualValue(spec, equal("value", "bcd"), "cd", Expression.Operation.FALSE);
    // not equal
    replacedertResidualValue(spec, notEqual("value", "bcd"), "ab", Expression.Operation.TRUE);
    replacedertResidualPredicate(spec, notEqual("value", "bcd"), "bc");
    replacedertResidualValue(spec, notEqual("value", "bcd"), "cd", Expression.Operation.TRUE);
    // starts with
    replacedertResidualValue(spec, startsWith("value", "bcd"), "ab", Expression.Operation.FALSE);
    replacedertResidualPredicate(spec, startsWith("value", "bcd"), "bc");
    replacedertResidualValue(spec, startsWith("value", "bcd"), "cd", Expression.Operation.FALSE);
}

17 View Complete Implementation : TestReadProjection.java
Copyright Apache License 2.0
Author : apache
@Test
public void testFullProjection() throws Exception {
    Schema schema = new Schema(Types.NestedField.required(0, "id", Types.LongType.get()), Types.NestedField.optional(1, "data", Types.StringType.get()));
    Record record = new Record(AvroSchemaUtil.convert(schema, "table"));
    record.put("id", 34L);
    record.put("data", "test");
    Record projected = writeAndRead("full_projection", schema, schema, record);
    replacedert.replacedertEquals("Should contain the correct id value", 34L, (long) projected.get("id"));
    int cmp = Comparators.charSequences().compare("test", (CharSequence) projected.get("data"));
    replacedert.replacedertTrue("Should contain the correct data value", cmp == 0);
}

17 View Complete Implementation : TestReadProjection.java
Copyright Apache License 2.0
Author : apache
@Test
public void testRename() throws Exception {
    Schema writeSchema = new Schema(Types.NestedField.required(0, "id", Types.LongType.get()), Types.NestedField.optional(1, "data", Types.StringType.get()));
    Record record = GenericRecord.create(writeSchema.replacedtruct());
    record.setField("id", 34L);
    record.setField("data", "test");
    Schema readSchema = new Schema(Types.NestedField.required(0, "id", Types.LongType.get()), Types.NestedField.optional(1, "renamed", Types.StringType.get()));
    Record projected = writeAndRead("project_and_rename", writeSchema, readSchema, record);
    replacedert.replacedertEquals("Should contain the correct id value", 34L, (long) projected.getField("id"));
    int cmp = Comparators.charSequences().compare("test", (CharSequence) projected.getField("renamed"));
    replacedert.replacedertTrue("Should contain the correct data/renamed value", cmp == 0);
}

17 View Complete Implementation : TestBucketingProjection.java
Copyright Apache License 2.0
Author : apache
@Test
public void testBucketUUIDInclusive() {
    UUID value = new UUID(123L, 456L);
    Schema schema = new Schema(optional(1, "value", Types.UUIDType.get()));
    ParreplacedionSpec spec = ParreplacedionSpec.builderFor(schema).bucket("value", 10).build();
    // the bucket number of the value (i.e. UUID(123L, 456L)) is 4
    replacedertProjectionInclusive(spec, equal("value", value), Expression.Operation.EQ, "4");
    replacedertProjectionInclusiveValue(spec, notEqual("value", value), Expression.Operation.TRUE);
    replacedertProjectionInclusiveValue(spec, lessThan("value", value), Expression.Operation.TRUE);
    replacedertProjectionInclusiveValue(spec, lessThanOrEqual("value", value), Expression.Operation.TRUE);
    replacedertProjectionInclusiveValue(spec, greaterThan("value", value), Expression.Operation.TRUE);
    replacedertProjectionInclusiveValue(spec, greaterThanOrEqual("value", value), Expression.Operation.TRUE);
    UUID anotherValue = new UUID(456L, 123L);
    replacedertProjectionInclusive(spec, in("value", value, anotherValue), Expression.Operation.IN, "[4, 6]");
    replacedertProjectionInclusiveValue(spec, notIn("value", value, anotherValue), Expression.Operation.TRUE);
}

17 View Complete Implementation : TestReadProjection.java
Copyright Apache License 2.0
Author : apache
@Test
public void testReorderedFullProjection() throws Exception {
    Schema schema = new Schema(Types.NestedField.required(0, "id", Types.LongType.get()), Types.NestedField.optional(1, "data", Types.StringType.get()));
    Record record = GenericRecord.create(schema.replacedtruct());
    record.setField("id", 34L);
    record.setField("data", "test");
    Schema reordered = new Schema(Types.NestedField.optional(1, "data", Types.StringType.get()), Types.NestedField.required(0, "id", Types.LongType.get()));
    Record projected = writeAndRead("full_projection", schema, reordered, record);
    replacedert.replacedertEquals("Should contain the correct 0 value", "test", projected.get(0).toString());
    replacedert.replacedertEquals("Should contain the correct 1 value", 34L, projected.get(1));
}

17 View Complete Implementation : TestBucketingProjection.java
Copyright Apache License 2.0
Author : apache
@Test
public void testBucketByteBufferInclusive() throws Exception {
    ByteBuffer value = ByteBuffer.wrap("abcdefg".getBytes("UTF-8"));
    Schema schema = new Schema(optional(1, "value", Types.BinaryType.get()));
    ParreplacedionSpec spec = ParreplacedionSpec.builderFor(schema).bucket("value", 10).build();
    // the bucket number of the value (i.e. "abcdefg") is 4
    replacedertProjectionInclusive(spec, equal("value", value), Expression.Operation.EQ, "4");
    replacedertProjectionInclusiveValue(spec, notEqual("value", value), Expression.Operation.TRUE);
    replacedertProjectionInclusiveValue(spec, lessThan("value", value), Expression.Operation.TRUE);
    replacedertProjectionInclusiveValue(spec, lessThanOrEqual("value", value), Expression.Operation.TRUE);
    replacedertProjectionInclusiveValue(spec, greaterThan("value", value), Expression.Operation.TRUE);
    replacedertProjectionInclusiveValue(spec, greaterThanOrEqual("value", value), Expression.Operation.TRUE);
    ByteBuffer anotherValue = ByteBuffer.wrap("abcdehij".getBytes("UTF-8"));
    replacedertProjectionInclusive(spec, in("value", value, anotherValue), Expression.Operation.IN, "[4, 6]");
    replacedertProjectionInclusiveValue(spec, notIn("value", value, anotherValue), Expression.Operation.TRUE);
}

17 View Complete Implementation : TestReadProjection.java
Copyright Apache License 2.0
Author : apache
@Test
public void testFullProjection() throws Exception {
    Schema schema = new Schema(Types.NestedField.required(0, "id", Types.LongType.get()), Types.NestedField.optional(1, "data", Types.StringType.get()));
    Record record = GenericRecord.create(schema.replacedtruct());
    record.setField("id", 34L);
    record.setField("data", "test");
    Record projected = writeAndRead("full_projection", schema, schema, record);
    replacedert.replacedertEquals("Should contain the correct id value", 34L, (long) projected.getField("id"));
    int cmp = Comparators.charSequences().compare("test", (CharSequence) projected.getField("data"));
    replacedert.replacedertTrue("Should contain the correct data value", cmp == 0);
}

17 View Complete Implementation : TestBucketingProjection.java
Copyright Apache License 2.0
Author : apache
@Test
public void testBucketUUIDStrict() {
    UUID value = new UUID(123L, 456L);
    Schema schema = new Schema(optional(1, "value", Types.UUIDType.get()));
    ParreplacedionSpec spec = ParreplacedionSpec.builderFor(schema).bucket("value", 10).build();
    // the bucket number of the value (i.e. UUID(123L, 456L)) is 4
    replacedertProjectionStrict(spec, notEqual("value", value), Expression.Operation.NOT_EQ, "4");
    replacedertProjectionStrictValue(spec, equal("value", value), Expression.Operation.FALSE);
    replacedertProjectionStrictValue(spec, lessThan("value", value), Expression.Operation.FALSE);
    replacedertProjectionStrictValue(spec, lessThanOrEqual("value", value), Expression.Operation.FALSE);
    replacedertProjectionStrictValue(spec, greaterThan("value", value), Expression.Operation.FALSE);
    replacedertProjectionStrictValue(spec, greaterThanOrEqual("value", value), Expression.Operation.FALSE);
    UUID anotherValue = new UUID(456L, 123L);
    replacedertProjectionStrict(spec, notIn("value", value, anotherValue), Expression.Operation.NOT_IN, "[4, 6]");
    replacedertProjectionStrictValue(spec, in("value", value, anotherValue), Expression.Operation.FALSE);
}

17 View Complete Implementation : ParquetFilters.java
Copyright Apache License 2.0
Author : apache
static FilterCompat.Filter convert(Schema schema, Expression expr, boolean caseSensitive) {
    FilterPredicate pred = ExpressionVisitors.visit(expr, new ConvertFilterToParquet(schema, caseSensitive));
    // TODO: handle AlwaysFalse.INSTANCE
    if (pred != null && pred != AlwaysTrue.INSTANCE) {
        // FilterCompat will apply LogicalInverseRewriter
        return FilterCompat.get(pred);
    } else {
        return FilterCompat.NOOP;
    }
}

17 View Complete Implementation : TestBucketingProjection.java
Copyright Apache License 2.0
Author : apache
@Test
public void testBucketByteBufferStrict() throws Exception {
    ByteBuffer value = ByteBuffer.wrap("abcdefg".getBytes("UTF-8"));
    Schema schema = new Schema(optional(1, "value", Types.BinaryType.get()));
    ParreplacedionSpec spec = ParreplacedionSpec.builderFor(schema).bucket("value", 10).build();
    // the bucket number of the value (i.e. "abcdefg") is 4
    replacedertProjectionStrict(spec, notEqual("value", value), Expression.Operation.NOT_EQ, "4");
    replacedertProjectionStrictValue(spec, equal("value", value), Expression.Operation.FALSE);
    replacedertProjectionStrictValue(spec, lessThan("value", value), Expression.Operation.FALSE);
    replacedertProjectionStrictValue(spec, lessThanOrEqual("value", value), Expression.Operation.FALSE);
    replacedertProjectionStrictValue(spec, greaterThan("value", value), Expression.Operation.FALSE);
    replacedertProjectionStrictValue(spec, greaterThanOrEqual("value", value), Expression.Operation.FALSE);
    ByteBuffer anotherValue = ByteBuffer.wrap("abcdehij".getBytes("UTF-8"));
    replacedertProjectionStrict(spec, notIn("value", value, anotherValue), Expression.Operation.NOT_IN, "[4, 6]");
    replacedertProjectionStrictValue(spec, in("value", value, anotherValue), Expression.Operation.FALSE);
}

17 View Complete Implementation : TestTruncatesProjection.java
Copyright Apache License 2.0
Author : apache
@Test
public void testBinaryStrict() throws Exception {
    ByteBuffer value = ByteBuffer.wrap("abcdefg".getBytes("UTF-8"));
    Schema schema = new Schema(optional(1, "value", Types.BinaryType.get()));
    ParreplacedionSpec spec = ParreplacedionSpec.builderFor(schema).truncate("value", 5).build();
    String expectedValue = TransformUtil.base64encode(ByteBuffer.wrap("abcde".getBytes("UTF-8")));
    replacedertProjectionStrict(spec, lessThan("value", value), Expression.Operation.LT, expectedValue);
    replacedertProjectionStrict(spec, lessThanOrEqual("value", value), Expression.Operation.LT, expectedValue);
    replacedertProjectionStrict(spec, greaterThan("value", value), Expression.Operation.GT, expectedValue);
    replacedertProjectionStrict(spec, greaterThanOrEqual("value", value), Expression.Operation.GT, expectedValue);
    replacedertProjectionStrict(spec, notEqual("value", value), Expression.Operation.NOT_EQ, expectedValue);
    replacedertProjectionStrictValue(spec, equal("value", value), Expression.Operation.FALSE);
    ByteBuffer anotherValue = ByteBuffer.wrap("abcdehij".getBytes("UTF-8"));
    replacedertProjectionStrict(spec, notIn("value", value, anotherValue), Expression.Operation.NOT_IN, String.format("[%s, %s]", expectedValue, expectedValue));
    replacedertProjectionStrictValue(spec, in("value", value, anotherValue), Expression.Operation.FALSE);
}

17 View Complete Implementation : TestORCSchemaUtil.java
Copyright Apache License 2.0
Author : apache
@Test
public void testRoundtripConversionNested() {
    Types.StructType leafStructType = Types.StructType.of(optional(6, "leafLongCol", Types.LongType.get()), optional(7, "leafBinaryCol", Types.BinaryType.get()));
    Types.StructType nestedStructType = Types.StructType.of(optional(4, "longCol", Types.LongType.get()), optional(5, "leafStructCol", leafStructType));
    Types.StructType structPrimTypeForList = Types.StructType.of(optional(506, "leafLongCol", Types.LongType.get()), optional(507, "leafBinaryCol", Types.BinaryType.get()));
    Types.StructType leafStructTypeForList = Types.StructType.of(optional(516, "leafLongCol", Types.LongType.get()), optional(517, "leafBinaryCol", Types.BinaryType.get()));
    Types.StructType nestedStructTypeForList = Types.StructType.of(optional(504, "longCol", Types.LongType.get()), optional(505, "leafStructCol", leafStructTypeForList));
    Types.StructType structPrimTypeForMap = Types.StructType.of(optional(606, "leafLongCol", Types.LongType.get()), optional(607, "leafBinaryCol", Types.BinaryType.get()));
    Types.StructType leafStructTypeForMap = Types.StructType.of(optional(616, "leafLongCol", Types.LongType.get()), optional(617, "leafBinaryCol", Types.BinaryType.get()));
    Types.StructType nestedStructTypeForMap = Types.StructType.of(optional(604, "longCol", Types.LongType.get()), optional(605, "leafStructCol", leafStructTypeForMap));
    Types.StructType leafStructTypeForStruct = Types.StructType.of(optional(716, "leafLongCol", Types.LongType.get()), optional(717, "leafBinaryCol", Types.BinaryType.get()));
    Types.StructType nestedStructTypeForStruct = Types.StructType.of(optional(704, "longCol", Types.LongType.get()), optional(705, "leafStructCol", leafStructTypeForStruct));
    // all fields in expected iceberg schema will be optional since we don't have a column mapping
    Schema expectedSchema = new Schema(optional(1, "intCol", Types.IntegerType.get()), optional(2, "longCol", Types.LongType.get()), optional(3, "nestedStructCol", nestedStructType), optional(8, "intCol3", Types.IntegerType.get()), optional(9, "doubleCol", Types.DoubleType.get()), required(10, "uuidCol", Types.UUIDType.get()), optional(20, "booleanCol", Types.BooleanType.get()), optional(21, "fixedCol", Types.FixedType.ofLength(4096)), required(22, "binaryCol", Types.BinaryType.get()), required(23, "stringCol", Types.StringType.get()), required(24, "decimalCol", Types.DecimalType.of(15, 3)), required(25, "floatCol", Types.FloatType.get()), optional(30, "dateCol", Types.DateType.get()), required(32, "timeCol", Types.TimeType.get()), required(34, "timestampCol", Types.TimestampType.withZone()), required(35, "listPrimCol", Types.ListType.ofRequired(135, Types.LongType.get())), required(36, "listPrimNestCol", Types.ListType.ofRequired(136, structPrimTypeForList)), required(37, "listNestedCol", Types.ListType.ofRequired(137, nestedStructTypeForList)), optional(38, "mapPrimCol", Types.MapType.ofRequired(138, 238, Types.StringType.get(), Types.FixedType.ofLength(4096))), required(39, "mapPrimNestCol", Types.MapType.ofRequired(139, 239, Types.StringType.get(), structPrimTypeForMap)), required(40, "mapNestedCol", Types.MapType.ofRequired(140, 240, Types.StringType.get(), nestedStructTypeForMap)), required(41, "structListNestCol", Types.ListType.ofRequired(241, Types.StructType.of(optional(816, "leafLongCol", Types.LongType.get()), optional(817, "leafBinaryCol", Types.BinaryType.get())))), required(42, "structMapNestCol", Types.MapType.ofRequired(242, 342, Types.StringType.get(), Types.StructType.of(optional(916, "leafLongCol", Types.LongType.get()), optional(917, "leafBinaryCol", Types.BinaryType.get())))), required(43, "structStructNestCol", Types.StructType.of(required(243, "innerStructNest", Types.StructType.of(optional(1016, "leafLongCol", Types.LongType.get()), optional(1017, "leafBinaryCol", Types.BinaryType.get()))))), required(44, "structStructComplexNestCol", Types.StructType.of(required(244, "innerStructNest", Types.StructType.of(optional(1116, "leafLongCol", Types.LongType.get()), optional(1117, "leftMapOfListStructCol", Types.MapType.ofRequired(1150, 1151, Types.StringType.get(), Types.ListType.ofRequired(1250, nestedStructTypeForStruct))))))));
    TypeDescription orcSchema = ORCSchemaUtil.convert(expectedSchema);
    replacedertEquals(expectedSchema.replacedtruct(), ORCSchemaUtil.convert(orcSchema).replacedtruct());
}

17 View Complete Implementation : ParquetUtil.java
Copyright Apache License 2.0
Author : apache
// we allow struct nesting, but not maps or arrays
private static boolean shouldStoreBounds(ColumnPath columnPath, Schema schema) {
    Iterator<String> pathIterator = columnPath.iterator();
    Type currentType = schema.replacedtruct();
    while (pathIterator.hasNext()) {
        if (currentType == null || !currentType.isStructType()) {
            return false;
        }
        String fieldName = pathIterator.next();
        currentType = currentType.replacedtructType().fieldType(fieldName);
    }
    return currentType != null && currentType.isPrimitiveType();
}

17 View Complete Implementation : TestReadProjection.java
Copyright Apache License 2.0
Author : apache
@Test
public void testFullProjection() throws Exception {
    Schema schema = new Schema(Types.NestedField.required(0, "id", Types.LongType.get()), Types.NestedField.optional(1, "data", Types.StringType.get()));
    Record record = new Record(AvroSchemaUtil.convert(schema, "table"));
    record.put("id", 34L);
    record.put("data", "test");
    Record projected = writeAndRead("full_projection", schema, schema, record);
    replacedert.replacedertEquals("Should contain the correct id value", 34L, (long) projected.get("id"));
    int cmp = Comparators.charSequences().compare("test", (CharSequence) projected.get("data"));
    replacedert.replacedertTrue("Should contain the correct data value", cmp == 0);
}

17 View Complete Implementation : TestReadProjection.java
Copyright Apache License 2.0
Author : apache
@Test
public void testFullProjection() throws Exception {
    Schema schema = new Schema(Types.NestedField.required(0, "id", Types.LongType.get()), Types.NestedField.optional(1, "data", Types.StringType.get()));
    Record record = new Record(AvroSchemaUtil.convert(schema, "table"));
    record.put("id", 34L);
    record.put("data", "test");
    Record projected = writeAndRead("full_projection", schema, schema, record);
    replacedert.replacedertEquals("Should contain the correct id value", 34L, (long) projected.get("id"));
    int cmp = Comparators.charSequences().compare("test", (CharSequence) projected.get("data"));
    replacedert.replacedertEquals("Should contain the correct data value", 0, cmp);
}

17 View Complete Implementation : TestReadProjection.java
Copyright Apache License 2.0
Author : apache
@Test
public void testRename() throws Exception {
    Schema writeSchema = new Schema(Types.NestedField.required(0, "id", Types.LongType.get()), Types.NestedField.optional(1, "data", Types.StringType.get()));
    Record record = new Record(AvroSchemaUtil.convert(writeSchema, "table"));
    record.put("id", 34L);
    record.put("data", "test");
    Schema readSchema = new Schema(Types.NestedField.required(0, "id", Types.LongType.get()), Types.NestedField.optional(1, "renamed", Types.StringType.get()));
    Record projected = writeAndRead("project_and_rename", writeSchema, readSchema, record);
    replacedert.replacedertEquals("Should contain the correct id value", 34L, (long) projected.get("id"));
    int cmp = Comparators.charSequences().compare("test", (CharSequence) projected.get("renamed"));
    replacedert.replacedertTrue("Should contain the correct data/renamed value", cmp == 0);
}

17 View Complete Implementation : HadoopTables.java
Copyright Apache License 2.0
Author : apache
/**
 * Create a table using the FileSystem implementation resolve from
 * location.
 *
 * @param schema iceberg schema used to create the table
 * @param spec parreplacedioning spec, if null the table will be unparreplacedioned
 * @param properties a string map of table properties, initialized to empty if null
 * @param location a path URI (e.g. hdfs:///warehouse/my_table)
 * @return newly created table implementation
 */
@Override
public Table create(Schema schema, ParreplacedionSpec spec, Map<String, String> properties, String location) {
    Preconditions.checkNotNull(schema, "A table schema is required");
    TableOperations ops = newTableOps(location);
    if (ops.current() != null) {
        throw new AlreadyExistsException("Table already exists at location: " + location);
    }
    Map<String, String> tableProps = properties == null ? ImmutableMap.of() : properties;
    ParreplacedionSpec parreplacedionSpec = spec == null ? ParreplacedionSpec.unparreplacedioned() : spec;
    TableMetadata metadata = TableMetadata.newTableMetadata(schema, parreplacedionSpec, location, tableProps);
    ops.commit(null, metadata);
    return new BaseTable(ops, location);
}

17 View Complete Implementation : SparkSchemaUtil.java
Copyright Apache License 2.0
Author : apache
/**
 * Convert a Spark {@link StructType struct} to a {@link Schema} based on the given schema.
 * <p>
 * This conversion does not replacedign new ids; it uses ids from the base schema.
 * <p>
 * Data types, field order, and nullability will match the spark type. This conversion may return
 * a schema that is not compatible with base schema.
 *
 * @param baseSchema a Schema on which conversion is based
 * @param sparkType a Spark StructType
 * @return the equivalent Schema
 * @throws IllegalArgumentException if the type cannot be converted or there are missing ids
 */
public static Schema convert(Schema baseSchema, StructType sparkType) {
    // convert to a type with fresh ids
    Types.StructType struct = SparkTypeVisitor.visit(sparkType, new SparkTypeToType(sparkType)).replacedtructType();
    // rereplacedign ids to match the base schema
    Schema schema = TypeUtil.rereplacedignIds(new Schema(struct.fields()), baseSchema);
    // fix types that can't be represented in Spark (UUID and Fixed)
    return FixupTypes.fixup(schema, baseSchema);
}

17 View Complete Implementation : TestReadProjection.java
Copyright Apache License 2.0
Author : apache
@Test
public void testRename() throws Exception {
    Schema writeSchema = new Schema(Types.NestedField.required(0, "id", Types.LongType.get()), Types.NestedField.optional(1, "data", Types.StringType.get()));
    Record record = new Record(AvroSchemaUtil.convert(writeSchema, "table"));
    record.put("id", 34L);
    record.put("data", "test");
    Schema readSchema = new Schema(Types.NestedField.required(0, "id", Types.LongType.get()), Types.NestedField.optional(1, "renamed", Types.StringType.get()));
    Record projected = writeAndRead("project_and_rename", writeSchema, readSchema, record);
    replacedert.replacedertEquals("Should contain the correct id value", 34L, (long) projected.get("id"));
    int cmp = Comparators.charSequences().compare("test", (CharSequence) projected.get("renamed"));
    replacedert.replacedertTrue("Should contain the correct data/renamed value", cmp == 0);
}

17 View Complete Implementation : TestReadProjection.java
Copyright Apache License 2.0
Author : apache
@Test
public void testRename() throws Exception {
    Schema writeSchema = new Schema(Types.NestedField.required(0, "id", Types.LongType.get()), Types.NestedField.optional(1, "data", Types.StringType.get()));
    Record record = new Record(AvroSchemaUtil.convert(writeSchema, "table"));
    record.put("id", 34L);
    record.put("data", "test");
    Schema readSchema = new Schema(Types.NestedField.required(0, "id", Types.LongType.get()), Types.NestedField.optional(1, "renamed", Types.StringType.get()));
    Record projected = writeAndRead("project_and_rename", writeSchema, readSchema, record);
    replacedert.replacedertEquals("Should contain the correct id value", 34L, (long) projected.get("id"));
    int cmp = Comparators.charSequences().compare("test", (CharSequence) projected.get("renamed"));
    replacedert.replacedertEquals("Should contain the correct data/renamed value", 0, cmp);
}

17 View Complete Implementation : SparkSchemaUtil.java
Copyright Apache License 2.0
Author : apache
/**
 * Prune columns from a {@link Schema} using a {@link StructType Spark type} projection.
 * <p>
 * This requires that the Spark type is a projection of the Schema. Nullability and types must
 * match.
 * <p>
 * The filters list of {@link Expression} is used to ensure that columns referenced by filters
 * are projected.
 *
 * @param schema a Schema
 * @param requestedType a projection of the Spark representation of the Schema
 * @param filters a list of filters
 * @return a Schema corresponding to the Spark projection
 * @throws IllegalArgumentException if the Spark type does not match the Schema
 */
public static Schema prune(Schema schema, StructType requestedType, List<Expression> filters) {
    Set<Integer> filterRefs = Binder.boundReferences(schema.replacedtruct(), filters, true);
    return new Schema(TypeUtil.visit(schema, new PruneColumnsWithoutReordering(requestedType, filterRefs)).asNestedType().replacedtructType().fields());
}

17 View Complete Implementation : TestTruncatesProjection.java
Copyright Apache License 2.0
Author : apache
@Test
public void testBinaryInclusive() throws Exception {
    ByteBuffer value = ByteBuffer.wrap("abcdefg".getBytes("UTF-8"));
    Schema schema = new Schema(optional(1, "value", Types.BinaryType.get()));
    ParreplacedionSpec spec = ParreplacedionSpec.builderFor(schema).truncate("value", 5).build();
    String expectedValue = TransformUtil.base64encode(ByteBuffer.wrap("abcde".getBytes("UTF-8")));
    replacedertProjectionInclusive(spec, lessThan("value", value), Expression.Operation.LT_EQ, expectedValue);
    replacedertProjectionInclusive(spec, lessThanOrEqual("value", value), Expression.Operation.LT_EQ, expectedValue);
    replacedertProjectionInclusive(spec, greaterThan("value", value), Expression.Operation.GT_EQ, expectedValue);
    replacedertProjectionInclusive(spec, greaterThanOrEqual("value", value), Expression.Operation.GT_EQ, expectedValue);
    replacedertProjectionInclusive(spec, equal("value", value), Expression.Operation.EQ, expectedValue);
    replacedertProjectionInclusiveValue(spec, notEqual("value", value), Expression.Operation.TRUE);
    ByteBuffer anotherValue = ByteBuffer.wrap("abcdehij".getBytes("UTF-8"));
    replacedertProjectionInclusive(spec, in("value", value, anotherValue), Expression.Operation.IN, String.format("[%s, %s]", expectedValue, expectedValue));
    replacedertProjectionInclusiveValue(spec, notIn("value", value, anotherValue), Expression.Operation.TRUE);
}

16 View Complete Implementation : TestReadProjection.java
Copyright Apache License 2.0
Author : apache
@Test
public void testSpecialCharacterProjection() throws Exception {
    Schema schema = new Schema(Types.NestedField.required(0, "user id", Types.LongType.get()), Types.NestedField.optional(1, "data%0", Types.StringType.get()));
    Record record = GenericRecord.create(schema.replacedtruct());
    record.setField("user id", 34L);
    record.setField("data%0", "test");
    Record full = writeAndRead("special_chars", schema, schema, record);
    replacedert.replacedertEquals("Should contain the correct id value", 34L, (long) full.getField("user id"));
    replacedert.replacedertEquals("Should contain the correct data value", 0, Comparators.charSequences().compare("test", (CharSequence) full.getField("data%0")));
    Record projected = writeAndRead("special_characters", schema, schema.select("data%0"), record);
    replacedert.replacedertNull("Should not contain id value", projected.getField("user id"));
    replacedert.replacedertEquals("Should contain the correct data value", 0, Comparators.charSequences().compare("test", (CharSequence) projected.getField("data%0")));
}

16 View Complete Implementation : TestNameMapping.java
Copyright Apache License 2.0
Author : apache
@Test
public void testMappingFindByName() {
    Schema schema = new Schema(required(1, "id", Types.LongType.get()), required(2, "data", Types.StringType.get()), required(3, "map", Types.MapType.ofRequired(4, 5, Types.DoubleType.get(), Types.StructType.of(required(6, "x", Types.DoubleType.get()), required(7, "y", Types.DoubleType.get())))), required(8, "list", Types.ListType.ofRequired(9, Types.StringType.get())), required(10, "location", Types.StructType.of(required(11, "lareplacedude", Types.FloatType.get()), required(12, "longitude", Types.FloatType.get()))));
    NameMapping mapping = MappingUtil.create(schema);
    replacedert.replacedertNull("Should not return a field mapping for a nested name", mapping.find("element"));
    replacedert.replacedertNull("Should not return a field mapping for a nested name", mapping.find("x"));
    replacedert.replacedertNull("Should not return a field mapping for a nested name", mapping.find("key"));
    replacedert.replacedertNull("Should not return a field mapping for a nested name", mapping.find("value"));
    replacedert.replacedertEquals(MappedField.of(2, "data"), mapping.find("data"));
    replacedert.replacedertEquals(MappedField.of(6, "x"), mapping.find("map", "value", "x"));
    replacedert.replacedertEquals(MappedField.of(9, "element"), mapping.find("list", "element"));
    replacedert.replacedertEquals(MappedField.of(11, "lareplacedude"), mapping.find("location", "lareplacedude"));
    replacedert.replacedertEquals(MappedField.of(10, "location", MappedFields.of(MappedField.of(11, "lareplacedude"), MappedField.of(12, "longitude"))), mapping.find("location"));
}

16 View Complete Implementation : TestAvroNameMapping.java
Copyright Apache License 2.0
Author : apache
@Test
public void testInferredMapping() throws IOException {
    Schema writeSchema = new Schema(Types.NestedField.required(0, "id", Types.LongType.get()), Types.NestedField.optional(1, "data", Types.StringType.get()));
    Record record = new Record(AvroSchemaUtil.convert(writeSchema, "table"));
    record.put("id", 34L);
    record.put("data", "data");
    Schema readSchema = writeSchema;
    // Preplaced null for nameMapping so that it is automatically inferred from read schema
    Record projected = writeAndRead(writeSchema, readSchema, record, null);
    replacedert.replacedertEquals(record, projected);
}

16 View Complete Implementation : TestTruncatesProjection.java
Copyright Apache License 2.0
Author : apache
@Test
public void testDecimalInclusiveUpperBound() {
    Types.DecimalType type = Types.DecimalType.of(9, 2);
    BigDecimal value = (BigDecimal) Literal.of("99.99").to(type).value();
    Schema schema = new Schema(optional(1, "value", type));
    ParreplacedionSpec spec = ParreplacedionSpec.builderFor(schema).truncate("value", 10).build();
    replacedertProjectionInclusive(spec, lessThan("value", value), Expression.Operation.LT_EQ, "99.90");
    replacedertProjectionInclusive(spec, lessThanOrEqual("value", value), Expression.Operation.LT_EQ, "99.90");
    replacedertProjectionInclusive(spec, greaterThan("value", value), Expression.Operation.GT_EQ, "100.00");
    replacedertProjectionInclusive(spec, greaterThanOrEqual("value", value), Expression.Operation.GT_EQ, "99.90");
    replacedertProjectionInclusive(spec, equal("value", value), Expression.Operation.EQ, "99.90");
    replacedertProjectionInclusiveValue(spec, notEqual("value", value), Expression.Operation.TRUE);
    BigDecimal delta = new BigDecimal(1);
    replacedertProjectionInclusive(spec, in("value", value.add(delta), value, value.subtract(delta)), Expression.Operation.IN, "[98.90, 99.90, 100.90]");
    replacedertProjectionInclusiveValue(spec, notIn("value", value, value.subtract(delta)), Expression.Operation.TRUE);
}

16 View Complete Implementation : TestReadProjection.java
Copyright Apache License 2.0
Author : apache
@Test
public void testBasicProjection() throws Exception {
    Schema writeSchema = new Schema(Types.NestedField.required(0, "id", Types.LongType.get()), Types.NestedField.optional(1, "data", Types.StringType.get()));
    Record record = GenericRecord.create(writeSchema.replacedtruct());
    record.setField("id", 34L);
    record.setField("data", "test");
    Schema idOnly = new Schema(Types.NestedField.required(0, "id", Types.LongType.get()));
    Record projected = writeAndRead("basic_projection_id", writeSchema, idOnly, record);
    replacedert.replacedertNull("Should not project data", projected.getField("data"));
    replacedert.replacedertEquals("Should contain the correct id value", 34L, (long) projected.getField("id"));
    Schema dataOnly = new Schema(Types.NestedField.optional(1, "data", Types.StringType.get()));
    projected = writeAndRead("basic_projection_data", writeSchema, dataOnly, record);
    replacedert.replacedertNull("Should not project id", projected.getField("id"));
    int cmp = Comparators.charSequences().compare("test", (CharSequence) projected.getField("data"));
    replacedert.replacedertTrue("Should contain the correct data value", cmp == 0);
}

16 View Complete Implementation : TestReadProjection.java
Copyright Apache License 2.0
Author : apache
@Test
public void testEmptyProjection() throws Exception {
    Schema schema = new Schema(Types.NestedField.required(0, "id", Types.LongType.get()), Types.NestedField.optional(1, "data", Types.StringType.get()));
    Record record = GenericRecord.create(schema.replacedtruct());
    record.setField("id", 34L);
    record.setField("data", "test");
    Record projected = writeAndRead("empty_projection", schema, schema.select(), record);
    replacedert.replacedertNotNull("Should read a non-null record", projected);
    try {
        projected.get(0);
        replacedert.fail("Should not retrieve value with ordinal 0");
    } catch (ArrayIndexOutOfBoundsException e) {
    // this is expected because there are no values
    }
}

16 View Complete Implementation : TestGenericAvro.java
Copyright Apache License 2.0
Author : apache
@Override
protected void writeAndValidate(Schema schema) throws IOException {
    List<Record> expected = RandomAvroData.generate(schema, 100, 0L);
    File testFile = temp.newFile();
    replacedert.replacedertTrue("Delete should succeed", testFile.delete());
    try (FileAppender<Record> writer = Avro.write(Files.localOutput(testFile)).schema(schema).named("test").build()) {
        for (Record rec : expected) {
            writer.add(rec);
        }
    }
    List<Record> rows;
    try (AvroIterable<Record> reader = Avro.read(Files.localInput(testFile)).project(schema).build()) {
        rows = Lists.newArrayList(reader);
    }
    for (int i = 0; i < expected.size(); i += 1) {
        AvroTestHelpers.replacedertEquals(schema.replacedtruct(), expected.get(i), rows.get(i));
    }
}

16 View Complete Implementation : TestReadProjection.java
Copyright Apache License 2.0
Author : apache
@Test
public void testReorderedFullProjection() throws Exception {
    Schema schema = new Schema(Types.NestedField.required(0, "id", Types.LongType.get()), Types.NestedField.optional(1, "data", Types.StringType.get()));
    Record record = new Record(AvroSchemaUtil.convert(schema, "table"));
    record.put("id", 34L);
    record.put("data", "test");
    Schema reordered = new Schema(Types.NestedField.optional(1, "data", Types.StringType.get()), Types.NestedField.required(0, "id", Types.LongType.get()));
    Record projected = writeAndRead("full_projection", schema, reordered, record);
    replacedert.replacedertEquals("Should contain the correct 0 value", "test", projected.get(0).toString());
    replacedert.replacedertEquals("Should contain the correct 1 value", 34L, projected.get(1));
}

16 View Complete Implementation : TestNameMapping.java
Copyright Apache License 2.0
Author : apache
@Test
public void testMappingFindById() {
    Schema schema = new Schema(required(1, "id", Types.LongType.get()), required(2, "data", Types.StringType.get()), required(3, "map", Types.MapType.ofRequired(4, 5, Types.DoubleType.get(), Types.StructType.of(required(6, "x", Types.DoubleType.get()), required(7, "y", Types.DoubleType.get())))), required(8, "list", Types.ListType.ofRequired(9, Types.StringType.get())), required(10, "location", Types.StructType.of(required(11, "lareplacedude", Types.FloatType.get()), required(12, "longitude", Types.FloatType.get()))));
    NameMapping mapping = MappingUtil.create(schema);
    replacedert.replacedertNull("Should not return a field mapping for a missing ID", mapping.find(100));
    replacedert.replacedertEquals(MappedField.of(2, "data"), mapping.find(2));
    replacedert.replacedertEquals(MappedField.of(6, "x"), mapping.find(6));
    replacedert.replacedertEquals(MappedField.of(9, "element"), mapping.find(9));
    replacedert.replacedertEquals(MappedField.of(11, "lareplacedude"), mapping.find(11));
    replacedert.replacedertEquals(MappedField.of(10, "location", MappedFields.of(MappedField.of(11, "lareplacedude"), MappedField.of(12, "longitude"))), mapping.find(10));
}

16 View Complete Implementation : TestTruncatesProjection.java
Copyright Apache License 2.0
Author : apache
@Test
public void testDecimalInclusiveLowerBound() {
    Types.DecimalType type = Types.DecimalType.of(9, 2);
    BigDecimal value = (BigDecimal) Literal.of("100.00").to(type).value();
    Schema schema = new Schema(optional(1, "value", type));
    ParreplacedionSpec spec = ParreplacedionSpec.builderFor(schema).truncate("value", 10).build();
    replacedertProjectionInclusive(spec, lessThan("value", value), Expression.Operation.LT_EQ, "99.90");
    replacedertProjectionInclusive(spec, lessThanOrEqual("value", value), Expression.Operation.LT_EQ, "100.00");
    replacedertProjectionInclusive(spec, greaterThan("value", value), Expression.Operation.GT_EQ, "100.00");
    replacedertProjectionInclusive(spec, greaterThanOrEqual("value", value), Expression.Operation.GT_EQ, "100.00");
    replacedertProjectionInclusive(spec, equal("value", value), Expression.Operation.EQ, "100.00");
    replacedertProjectionInclusiveValue(spec, notEqual("value", value), Expression.Operation.TRUE);
    BigDecimal delta = new BigDecimal(1);
    replacedertProjectionInclusive(spec, in("value", value.add(delta), value, value.subtract(delta)), Expression.Operation.IN, "[99.00, 100.00, 101.00]");
    replacedertProjectionInclusiveValue(spec, notIn("value", value, value.add(delta)), Expression.Operation.TRUE);
}

16 View Complete Implementation : TestTruncatesProjection.java
Copyright Apache License 2.0
Author : apache
@Test
public void testDecimalStrictLowerBound() {
    Types.DecimalType type = Types.DecimalType.of(9, 2);
    BigDecimal value = (BigDecimal) Literal.of("100.00").to(type).value();
    Schema schema = new Schema(optional(1, "value", type));
    ParreplacedionSpec spec = ParreplacedionSpec.builderFor(schema).truncate("value", 10).build();
    replacedertProjectionStrict(spec, lessThan("value", value), Expression.Operation.LT, "100.00");
    replacedertProjectionStrict(spec, lessThanOrEqual("value", value), Expression.Operation.LT, "100.00");
    replacedertProjectionStrict(spec, greaterThan("value", value), Expression.Operation.GT, "100.00");
    replacedertProjectionStrict(spec, greaterThanOrEqual("value", value), Expression.Operation.GT, "99.90");
    replacedertProjectionStrict(spec, notEqual("value", value), Expression.Operation.NOT_EQ, "100.00");
    replacedertProjectionStrictValue(spec, equal("value", value), Expression.Operation.FALSE);
    BigDecimal delta = new BigDecimal(1);
    replacedertProjectionStrict(spec, notIn("value", value.add(delta), value, value.subtract(delta)), Expression.Operation.NOT_IN, "[99.00, 100.00, 101.00]");
    replacedertProjectionStrictValue(spec, in("value", value, value.add(delta)), Expression.Operation.FALSE);
}

16 View Complete Implementation : TestReadProjection.java
Copyright Apache License 2.0
Author : apache
@Test
public void testReorderedProjection() throws Exception {
    Schema schema = new Schema(Types.NestedField.required(0, "id", Types.LongType.get()), Types.NestedField.optional(1, "data", Types.StringType.get()));
    Record record = GenericRecord.create(schema.replacedtruct());
    record.setField("id", 34L);
    record.setField("data", "test");
    Schema reordered = new Schema(Types.NestedField.optional(2, "missing_1", Types.StringType.get()), Types.NestedField.optional(1, "data", Types.StringType.get()), Types.NestedField.optional(3, "missing_2", Types.LongType.get()));
    Record projected = writeAndRead("full_projection", schema, reordered, record);
    replacedert.replacedertNull("Should contain the correct 0 value", projected.get(0));
    replacedert.replacedertEquals("Should contain the correct 1 value", "test", projected.get(1).toString());
    replacedert.replacedertNull("Should contain the correct 2 value", projected.get(2));
}

16 View Complete Implementation : TestTruncatesProjection.java
Copyright Apache License 2.0
Author : apache
@Test
public void testDecimalStrictUpperBound() {
    Types.DecimalType type = Types.DecimalType.of(9, 2);
    BigDecimal value = (BigDecimal) Literal.of("99.99").to(type).value();
    Schema schema = new Schema(optional(1, "value", type));
    ParreplacedionSpec spec = ParreplacedionSpec.builderFor(schema).truncate("value", 10).build();
    replacedertProjectionStrict(spec, lessThan("value", value), Expression.Operation.LT, "99.90");
    replacedertProjectionStrict(spec, lessThanOrEqual("value", value), Expression.Operation.LT, "100.00");
    replacedertProjectionStrict(spec, greaterThan("value", value), Expression.Operation.GT, "99.90");
    replacedertProjectionStrict(spec, greaterThanOrEqual("value", value), Expression.Operation.GT, "99.90");
    replacedertProjectionStrict(spec, notEqual("value", value), Expression.Operation.NOT_EQ, "99.90");
    replacedertProjectionStrictValue(spec, equal("value", value), Expression.Operation.FALSE);
    BigDecimal delta = new BigDecimal(1);
    replacedertProjectionStrict(spec, notIn("value", value.add(delta), value, value.subtract(delta)), Expression.Operation.NOT_IN, "[98.90, 99.90, 100.90]");
    replacedertProjectionStrictValue(spec, in("value", value, value.subtract(delta)), Expression.Operation.FALSE);
}

16 View Complete Implementation : TestBucketingProjection.java
Copyright Apache License 2.0
Author : apache
@Test
public void testBucketDecimalStrict() {
    Types.DecimalType type = Types.DecimalType.of(9, 2);
    BigDecimal value = (BigDecimal) Literal.of("100.00").to(type).value();
    Schema schema = new Schema(optional(1, "value", type));
    ParreplacedionSpec spec = ParreplacedionSpec.builderFor(schema).bucket("value", 10).build();
    // the bucket number of the value (i.e. 100.00) is 2
    replacedertProjectionStrict(spec, notEqual("value", value), Expression.Operation.NOT_EQ, "2");
    replacedertProjectionStrictValue(spec, equal("value", value), Expression.Operation.FALSE);
    replacedertProjectionStrictValue(spec, lessThan("value", value), Expression.Operation.FALSE);
    replacedertProjectionStrictValue(spec, lessThanOrEqual("value", value), Expression.Operation.FALSE);
    replacedertProjectionStrictValue(spec, greaterThan("value", value), Expression.Operation.FALSE);
    replacedertProjectionStrictValue(spec, greaterThanOrEqual("value", value), Expression.Operation.FALSE);
    BigDecimal delta = new BigDecimal(1);
    replacedertProjectionStrict(spec, notIn("value", value.add(delta), value, value.subtract(delta)), Expression.Operation.NOT_IN, "[2, 2, 6]");
    replacedertProjectionStrictValue(spec, in("value", value, value.add(delta)), Expression.Operation.FALSE);
}

16 View Complete Implementation : TestReadProjection.java
Copyright Apache License 2.0
Author : apache
@Test
public void testReorderedFullProjection() throws Exception {
    // replacedume.replacedumeTrue(
    // "Spark's Parquet read support does not support reordered columns",
    // !format.equalsIgnoreCase("parquet"));
    Schema schema = new Schema(Types.NestedField.required(0, "id", Types.LongType.get()), Types.NestedField.optional(1, "data", Types.StringType.get()));
    Record record = new Record(AvroSchemaUtil.convert(schema, "table"));
    record.put("id", 34L);
    record.put("data", "test");
    Schema reordered = new Schema(Types.NestedField.optional(1, "data", Types.StringType.get()), Types.NestedField.required(0, "id", Types.LongType.get()));
    Record projected = writeAndRead("reordered_full_projection", schema, reordered, record);
    replacedert.replacedertEquals("Should contain the correct 0 value", "test", projected.get(0).toString());
    replacedert.replacedertEquals("Should contain the correct 1 value", 34L, projected.get(1));
}

16 View Complete Implementation : TestORCSchemaUtil.java
Copyright Apache License 2.0
Author : apache
@Test
public void testTypePromotions() {
    Schema originalSchema = new Schema(optional(1, "a", Types.IntegerType.get()), optional(2, "b", Types.FloatType.get()), optional(3, "c", Types.DecimalType.of(10, 2)));
    // Original mapping (stored in ORC)
    TypeDescription orcSchema = ORCSchemaUtil.convert(originalSchema);
    // Evolve schema
    Schema evolveSchema = new Schema(optional(1, "a", Types.LongType.get()), optional(2, "b", Types.DoubleType.get()), optional(3, "c", Types.DecimalType.of(15, 2)));
    TypeDescription newOrcSchema = ORCSchemaUtil.buildOrcProjection(evolveSchema, orcSchema);
    replacedertEquals(3, newOrcSchema.getChildren().size());
    replacedertEquals(1, newOrcSchema.findSubtype("a").getId());
    replacedertEquals(TypeDescription.Category.LONG, newOrcSchema.findSubtype("a").getCategory());
    replacedertEquals(2, newOrcSchema.findSubtype("b").getId());
    replacedertEquals(TypeDescription.Category.DOUBLE, newOrcSchema.findSubtype("b").getCategory());
    TypeDescription decimalC = newOrcSchema.findSubtype("c");
    replacedertEquals(3, decimalC.getId());
    replacedertEquals(TypeDescription.Category.DECIMAL, decimalC.getCategory());
    replacedertEquals(15, decimalC.getPrecision());
    replacedertEquals(2, decimalC.getScale());
}

16 View Complete Implementation : TestReadProjection.java
Copyright Apache License 2.0
Author : apache
@Test
public void testMapProjection() throws IOException {
    Schema writeSchema = new Schema(Types.NestedField.required(0, "id", Types.LongType.get()), Types.NestedField.optional(5, "properties", Types.MapType.ofOptional(6, 7, Types.StringType.get(), Types.StringType.get())));
    Map<String, String> properties = ImmutableMap.of("a", "A", "b", "B");
    Record record = GenericRecord.create(writeSchema.replacedtruct());
    record.setField("id", 34L);
    record.setField("properties", properties);
    Schema idOnly = new Schema(Types.NestedField.required(0, "id", Types.LongType.get()));
    Record projected = writeAndRead("id_only", writeSchema, idOnly, record);
    replacedert.replacedertEquals("Should contain the correct id value", 34L, (long) projected.getField("id"));
    replacedert.replacedertNull("Should not project properties map", projected.getField("properties"));
    Schema keyOnly = writeSchema.select("properties.key");
    projected = writeAndRead("key_only", writeSchema, keyOnly, record);
    replacedert.replacedertNull("Should not project id", projected.getField("id"));
    replacedert.replacedertEquals("Should project entire map", properties, toStringMap((Map) projected.getField("properties")));
    Schema valueOnly = writeSchema.select("properties.value");
    projected = writeAndRead("value_only", writeSchema, valueOnly, record);
    replacedert.replacedertNull("Should not project id", projected.getField("id"));
    replacedert.replacedertEquals("Should project entire map", properties, toStringMap((Map) projected.getField("properties")));
    Schema mapOnly = writeSchema.select("properties");
    projected = writeAndRead("map_only", writeSchema, mapOnly, record);
    replacedert.replacedertNull("Should not project id", projected.getField("id"));
    replacedert.replacedertEquals("Should project entire map", properties, toStringMap((Map) projected.getField("properties")));
}

16 View Complete Implementation : TestHasIds.java
Copyright Apache License 2.0
Author : apache
@Test
public void test() {
    Schema schema = new Schema(Types.NestedField.required(0, "id", Types.LongType.get()), Types.NestedField.optional(5, "location", Types.MapType.ofOptional(6, 7, Types.StringType.get(), Types.StructType.of(Types.NestedField.required(1, "lat", Types.FloatType.get()), Types.NestedField.optional(2, "long", Types.FloatType.get())))));
    org.apache.avro.Schema avroSchema = RemoveIds.removeIds(schema);
    replacedert.replacedertFalse(AvroSchemaUtil.hasIds(avroSchema));
    avroSchema.getFields().get(0).addProp("field-id", 1);
    replacedert.replacedertTrue(AvroSchemaUtil.hasIds(avroSchema));
    // Create a fresh copy
    avroSchema = RemoveIds.removeIds(schema);
    avroSchema.getFields().get(1).schema().getTypes().get(1).getValueType().getTypes().get(1).getFields().get(1).addProp("field-id", 1);
    replacedert.replacedertTrue(AvroSchemaUtil.hasIds(avroSchema));
}