Here are the examples of the java api org.apache.flink.api.java.ExecutionEnvironment.getExecutionEnvironment() taken from open source projects. By voting up you can indicate which examples are most useful and appropriate.
155 Examples
16
View Complete Implementation : MiscellaneousIssuesITCase.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
@Test
public void testDisjointDataflows() {
try {
ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
env.setParallelism(5);
// generate two different flows
env.generateSequence(1, 10).output(new DiscardingOutputFormat<Long>());
env.generateSequence(1, 10).output(new DiscardingOutputFormat<Long>());
} catch (Exception e) {
e.printStackTrace();
fail(e.getMessage());
}
}
15
View Complete Implementation : AccumulatorErrorITCase.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
@Test
public void testInvalidTypeAcreplacedulator() throws Exception {
ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
// Test Exception forwarding with faulty Acreplacedulator implementation
env.generateSequence(0, 10000).map(new IncompatibleAcreplacedulatorTypesMapper()).map(new IncompatibleAcreplacedulatorTypesMapper2()).output(new DiscardingOutputFormat<>());
try {
env.execute();
fail("Should have failed.");
} catch (JobExecutionException e) {
replacedertTrue(findThrowable(e, UnsupportedOperationException.clreplaced).isPresent());
}
}
15
View Complete Implementation : PartitionOperatorTest.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
@Test
public void testRangeParreplacedionCustomParreplacedionerByFieldId() throws Exception {
final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
final DataSet<Tuple2<Integer, String>> ds = getTupleDataSet(env);
ds.parreplacedionCustom(new Parreplacedioner<Integer>() {
@Override
public int parreplacedion(Integer key, int numParreplacedions) {
return 1;
}
}, 0);
}
15
View Complete Implementation : SimpleRecoveryITCaseBase.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
@Test
public void testRestart() {
try {
List<Long> resultCollection = new ArrayList<Long>();
ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
env.setParallelism(4);
// the default restart strategy should be taken
env.generateSequence(1, 10).rebalance().map(new FailingMapper2<Long>()).reduce(new ReduceFunction<Long>() {
@Override
public Long reduce(Long value1, Long value2) {
return value1 + value2;
}
}).output(new LocalCollectionOutputFormat<Long>(resultCollection));
executeAndRunreplacedertions(env);
long sum = 0;
for (long l : resultCollection) {
sum += l;
}
replacedertEquals(55, sum);
} catch (Exception e) {
e.printStackTrace();
fail(e.getMessage());
} finally {
FailingMapper2.failuresBeforeSuccess = 1;
}
}
15
View Complete Implementation : CustomSerializationITCase.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
@Test
public void testIncorrectSerializer4() {
try {
ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
env.setParallelism(PARLLELISM);
env.generateSequence(1, 10 * PARLLELISM).map(new MapFunction<Long, ConsumesTooLittleSpanning>() {
@Override
public ConsumesTooLittleSpanning map(Long value) throws Exception {
return new ConsumesTooLittleSpanning();
}
}).rebalance().output(new DiscardingOutputFormat<ConsumesTooLittleSpanning>());
env.execute();
} catch (ProgramInvocationException e) {
Throwable rootCause = e.getCause().getCause();
replacedertTrue(rootCause instanceof IOException);
replacedertTrue(rootCause.getMessage().contains("broken serialization"));
} catch (Exception e) {
e.printStackTrace();
fail(e.getMessage());
}
}
15
View Complete Implementation : PartitionOperatorTest.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
@Test(expected = InvalidProgramException.clreplaced)
public void testRangeParreplacedionInvalidCustomParreplacedionerByFieldId() throws Exception {
final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
final DataSet<Tuple2<Integer, String>> ds = getTupleDataSet(env);
ds.parreplacedionCustom(new Parreplacedioner<Integer>() {
@Override
public int parreplacedion(Integer key, int numParreplacedions) {
return 1;
}
}, 1);
}
15
View Complete Implementation : HBaseReadExample.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
public static void main(String[] args) throws Exception {
ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
@SuppressWarnings("serial")
DataSet<Tuple2<String, String>> hbaseDs = env.createInput(new TableInputFormat<Tuple2<String, String>>() {
@Override
public String getTableName() {
return HBaseFlinkTestConstants.TEST_TABLE_NAME;
}
@Override
protected Scan getScanner() {
Scan scan = new Scan();
scan.addColumn(HBaseFlinkTestConstants.CF_SOME, HBaseFlinkTestConstants.Q_SOME);
return scan;
}
private Tuple2<String, String> reuse = new Tuple2<String, String>();
@Override
protected Tuple2<String, String> mapResultToTuple(Result r) {
String key = Bytes.toString(r.getRow());
String val = Bytes.toString(r.getValue(HBaseFlinkTestConstants.CF_SOME, HBaseFlinkTestConstants.Q_SOME));
reuse.setField(key, 0);
reuse.setField(val, 1);
return reuse;
}
}).filter(new FilterFunction<Tuple2<String, String>>() {
@Override
public boolean filter(Tuple2<String, String> t) throws Exception {
String val = t.getField(1);
if (val.startsWith("someStr")) {
return true;
}
return false;
}
});
hbaseDs.print();
}
15
View Complete Implementation : PartitionOperatorTest.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
@Test
public void testRangeParreplacedionCustomParreplacedionerByKeySelector() throws Exception {
final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
final DataSet<CustomPojo> ds = getPojoDataSet(env);
ds.parreplacedionCustom(new Parreplacedioner<Integer>() {
@Override
public int parreplacedion(Integer key, int numParreplacedions) {
return 1;
}
}, new KeySelector<CustomPojo, Integer>() {
@Override
public Integer getKey(CustomPojo value) throws Exception {
return value.getNumber();
}
});
}
15
View Complete Implementation : PartitionOperatorTest.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
@Test(expected = InvalidProgramException.clreplaced)
public void testRangeParreplacedionInvalidCustomParreplacedionerByFieldName() throws Exception {
final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
final DataSet<CustomPojo> ds = getPojoDataSet(env);
ds.parreplacedionCustom(new Parreplacedioner<Integer>() {
@Override
public int parreplacedion(Integer key, int numParreplacedions) {
return 1;
}
}, "name");
}
15
View Complete Implementation : PartitionOperatorTest.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
@Test
public void testRangeParreplacedionCustomParreplacedionerByFieldName() throws Exception {
final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
final DataSet<CustomPojo> ds = getPojoDataSet(env);
ds.parreplacedionCustom(new Parreplacedioner<Integer>() {
@Override
public int parreplacedion(Integer key, int numParreplacedions) {
return 1;
}
}, "number");
}
15
View Complete Implementation : HBaseConnectorITCase.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
// ------------------------------- Utilities -------------------------------------------------
/**
* Creates a Batch {@link TableEnvironment} depends on the {@link #planner} context.
*/
private TableEnvironment createBatchTableEnv() {
if (OLD_PLANNER.equals(planner)) {
ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
return BatchTableEnvironment.create(env, new TableConfig());
} else {
return TableEnvironment.create(batchSettings);
}
}
14
View Complete Implementation : PartitionOperatorTest.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
@Test
public void testRangeParreplacedionBySelectorComplexKeyWithOrders() throws Exception {
final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
final DataSet<NestedPojo> ds = getNestedPojoDataSet(env);
ds.parreplacedionByRange(new KeySelector<NestedPojo, CustomPojo>() {
@Override
public CustomPojo getKey(NestedPojo value) throws Exception {
return value.getNested();
}
}).withOrders(Order.ASCENDING);
}
14
View Complete Implementation : WordCountWithExternalClass.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
public static void main(String[] args) throws Exception {
// set up the execution environment
final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
// get input data
DataSet<String> text = StaticData.getDefaultTextLineDataSet(env);
DataSet<Tuple2<String, Integer>> counts = // split up the lines in pairs (2-tuples) containing: (word,1)
text.flatMap(new ExternalTokenizer()).groupBy(0).sum(1);
// emit result
counts.print();
// execute program
env.execute("WordCount Example");
}
14
View Complete Implementation : MaxByOperatorTest.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
/**
* This test validates that an index which is out of bounds throws an
* IndexOutOfBoundsException.
*/
@Test(expected = IndexOutOfBoundsException.clreplaced)
public void testOutOfTupleBoundsGrouping3() {
final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
UnsortedGrouping<Tuple5<Integer, Long, String, Long, Integer>> groupDs = env.fromCollection(emptyTupleData, tupleTypeInfo).groupBy(0);
// should not work, key out of tuple bounds
groupDs.maxBy(1, 2, 3, 4, -1);
}
14
View Complete Implementation : GroupingKeySelectorTranslationTest.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
@Test
public void testCustomParreplacedioningKeySelectorInvalidType() {
try {
ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
DataSet<Tuple2<Integer, Integer>> data = env.fromElements(new Tuple2<Integer, Integer>(0, 0)).rebalance().setParallelism(4);
try {
data.groupBy(new TestKeySelector<Tuple2<Integer, Integer>>()).withParreplacedioner(new TestParreplacedionerLong());
fail("Should throw an exception");
} catch (InvalidProgramException e) {
}
} catch (Exception e) {
e.printStackTrace();
fail(e.getMessage());
}
}
14
View Complete Implementation : MaxByOperatorTest.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
/**
* This test validates that an index which is out of bounds throws an
* IndexOutOfBoundsException.
*/
@Test(expected = IndexOutOfBoundsException.clreplaced)
public void testOutOfTupleBoundsDataset3() {
final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
DataSet<Tuple5<Integer, Long, String, Long, Integer>> tupleDs = env.fromCollection(emptyTupleData, tupleTypeInfo);
// should not work, key out of tuple bounds
tupleDs.maxBy(1, 2, 3, 4, -1);
}
14
View Complete Implementation : MinByOperatorTest.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
/**
* This test validates that an InvalidProgramException is thrown when minBy
* is used on a custom data type.
*/
@Test(expected = InvalidProgramException.clreplaced)
public void testCustomKeyFieldsDataset() {
final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
this.customTypeData.add(new CustomType());
DataSet<CustomType> customDs = env.fromCollection(customTypeData);
// should not work: groups on custom type
customDs.minBy(0);
}
14
View Complete Implementation : JoinOperatorTest.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
@Test(expected = InvalidProgramException.clreplaced)
public void testJoinKeyMixedWrong() {
final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
DataSet<CustomType> ds1 = env.fromCollection(customTypeData);
DataSet<CustomType> ds2 = env.fromCollection(customTypeData);
// wrongly mix String and Integer
ds1.join(ds2).where("myString").equalTo(new KeySelector<CustomType, Integer>() {
@Override
public Integer getKey(CustomType value) throws Exception {
return value.myInt;
}
});
}
14
View Complete Implementation : MaxByOperatorTest.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
/**
* This test validates that an index which is out of bounds throws an
* IndexOutOfBoundsException.
*/
@Test(expected = IndexOutOfBoundsException.clreplaced)
public void testOutOfTupleBoundsGrouping1() {
final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
UnsortedGrouping<Tuple5<Integer, Long, String, Long, Integer>> groupDs = env.fromCollection(emptyTupleData, tupleTypeInfo).groupBy(0);
// should not work, key out of tuple bounds
groupDs.maxBy(5);
}
14
View Complete Implementation : GroupingPojoTranslationTest.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
@Test
public void testCustomParreplacedioningTupleInvalidType() {
try {
ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
DataSet<Pojo2> data = env.fromElements(new Pojo2()).rebalance().setParallelism(4);
try {
data.groupBy("a").withParreplacedioner(new TestParreplacedionerLong());
fail("Should throw an exception");
} catch (InvalidProgramException e) {
}
} catch (Exception e) {
e.printStackTrace();
fail(e.getMessage());
}
}
14
View Complete Implementation : MaxByOperatorTest.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
/**
* This test validates that an index which is out of bounds throws an
* IndexOutOfBoundsException.
*/
@Test(expected = IndexOutOfBoundsException.clreplaced)
public void testOutOfTupleBoundsDataset2() {
final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
DataSet<Tuple5<Integer, Long, String, Long, Integer>> tupleDs = env.fromCollection(emptyTupleData, tupleTypeInfo);
// should not work, key out of tuple bounds
tupleDs.maxBy(-1);
}
14
View Complete Implementation : MaxByOperatorTest.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
/**
* This test validates that an index which is out of bounds throws an
* IndexOutOfBoundsException.
*/
@Test(expected = IndexOutOfBoundsException.clreplaced)
public void testOutOfTupleBoundsDataset1() {
final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
DataSet<Tuple5<Integer, Long, String, Long, Integer>> tupleDs = env.fromCollection(emptyTupleData, tupleTypeInfo);
// should not work, key out of tuple bounds
tupleDs.maxBy(5);
}
14
View Complete Implementation : JoinOperatorTest.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
@Test(expected = InvalidProgramException.clreplaced)
public void testJoinKeyMixing4() {
final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
DataSet<Tuple5<Integer, Long, String, Long, Integer>> ds1 = env.fromCollection(emptyTupleData, tupleTypeInfo);
DataSet<CustomType> ds2 = env.fromCollection(customTypeData);
// should not work, more than one key field position
ds1.join(ds2).where(1, 3).equalTo(new KeySelector<CustomType, Long>() {
@Override
public Long getKey(CustomType value) {
return value.myLong;
}
});
}
14
View Complete Implementation : MaxByOperatorTest.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
/**
* This test validates that an index which is out of bounds throws an
* IndexOutOfBoundsException.
*/
@Test(expected = IndexOutOfBoundsException.clreplaced)
public void testOutOfTupleBoundsGrouping2() {
final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
UnsortedGrouping<Tuple5<Integer, Long, String, Long, Integer>> groupDs = env.fromCollection(emptyTupleData, tupleTypeInfo).groupBy(0);
// should not work, key out of tuple bounds
groupDs.maxBy(-1);
}
14
View Complete Implementation : GroupingTupleTranslationTest.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
@Test
public void testCustomParreplacedioningTupleInvalidType() {
try {
ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
DataSet<Tuple2<Integer, Integer>> data = env.fromElements(new Tuple2<Integer, Integer>(0, 0)).rebalance().setParallelism(4);
try {
data.groupBy(0).withParreplacedioner(new TestParreplacedionerLong());
fail("Should throw an exception");
} catch (InvalidProgramException e) {
}
} catch (Exception e) {
e.printStackTrace();
fail(e.getMessage());
}
}
14
View Complete Implementation : WordCountWithAnonymousClass.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
public static void main(String[] args) throws Exception {
// set up the execution environment
final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
// get input data
DataSet<String> text = StaticData.getDefaultTextLineDataSet(env);
DataSet<Tuple2<String, Integer>> counts = // split up the lines in pairs (2-tuples) containing: (word,1)
text.flatMap(new FlatMapFunction<String, Tuple2<String, Integer>>() {
@Override
public void flatMap(String value, Collector<Tuple2<String, Integer>> out) throws Exception {
// normalize and split the line
String[] tokens = value.toLowerCase().split("\\W+");
// emit the pairs
for (String token : tokens) {
if (token.length() > 0) {
out.collect(new Tuple2<String, Integer>(token, 1));
}
}
}
}).groupBy(0).sum(1);
// emit result
counts.print();
// execute program
env.execute("WordCount Example");
}
14
View Complete Implementation : GroupingTupleTranslationTest.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
@Test
public void testCustomParreplacedioningTupleRejectCompositeKey() {
try {
ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
DataSet<Tuple3<Integer, Integer, Integer>> data = env.fromElements(new Tuple3<Integer, Integer, Integer>(0, 0, 0)).rebalance().setParallelism(4);
try {
data.groupBy(0, 1).withParreplacedioner(new TestParreplacedionerInt());
fail("Should throw an exception");
} catch (InvalidProgramException e) {
}
} catch (Exception e) {
e.printStackTrace();
fail(e.getMessage());
}
}
14
View Complete Implementation : GroupingKeySelectorTranslationTest.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
@Test
public void testCustomParreplacedioningTupleRejectCompositeKey() {
try {
ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
DataSet<Tuple3<Integer, Integer, Integer>> data = env.fromElements(new Tuple3<Integer, Integer, Integer>(0, 0, 0)).rebalance().setParallelism(4);
try {
data.groupBy(new TestBinaryKeySelector<Tuple3<Integer, Integer, Integer>>()).withParreplacedioner(new TestParreplacedionerInt());
fail("Should throw an exception");
} catch (InvalidProgramException e) {
}
} catch (Exception e) {
e.printStackTrace();
fail(e.getMessage());
}
}
14
View Complete Implementation : CoGroupOperatorTest.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
@Test(expected = InvalidProgramException.clreplaced)
public void testCoGroupKeyMixing4() {
final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
DataSet<Tuple5<Integer, Long, String, Long, Integer>> ds1 = env.fromCollection(emptyTupleData, tupleTypeInfo);
DataSet<CustomType> ds2 = env.fromCollection(customTypeData);
// should not work, more than one key field position
ds1.coGroup(ds2).where(1, 3).equalTo(new KeySelector<CustomType, Long>() {
@Override
public Long getKey(CustomType value) {
return value.myLong;
}
});
}
14
View Complete Implementation : CustomInputSplitProgram.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
public static void main(String[] args) throws Exception {
ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
DataSet<Integer> data = env.createInput(new CustomInputFormat());
data.map(new MapFunction<Integer, Tuple2<Integer, Double>>() {
@Override
public Tuple2<Integer, Double> map(Integer value) {
return new Tuple2<Integer, Double>(value, value * 0.5);
}
}).output(new DiscardingOutputFormat<Tuple2<Integer, Double>>());
env.execute();
}
14
View Complete Implementation : MinByOperatorTest.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
/**
* This test validates that an index which is out of bounds throws an
* IndexOutOfBoundsException.
*/
@Test(expected = IndexOutOfBoundsException.clreplaced)
public void testOutOfTupleBoundsDataset2() {
final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
DataSet<Tuple5<Integer, Long, String, Long, Integer>> tupleDs = env.fromCollection(emptyTupleData, tupleTypeInfo);
// should not work, key out of tuple bounds
tupleDs.minBy(-1);
}
14
View Complete Implementation : MinByOperatorTest.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
/**
* This test validates that an index which is out of bounds throws an
* IndexOutOfBoundsException.
*/
@Test(expected = IndexOutOfBoundsException.clreplaced)
public void testOutOfTupleBoundsDataset3() {
final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
DataSet<Tuple5<Integer, Long, String, Long, Integer>> tupleDs = env.fromCollection(emptyTupleData, tupleTypeInfo);
// should not work, key out of tuple bounds
tupleDs.minBy(1, 2, 3, 4, -1);
}
14
View Complete Implementation : CoGroupOperatorTest.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
@Test(expected = InvalidProgramException.clreplaced)
public void testCoGroupKeyMixing3() {
final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
DataSet<Tuple5<Integer, Long, String, Long, Integer>> ds1 = env.fromCollection(emptyTupleData, tupleTypeInfo);
DataSet<CustomType> ds2 = env.fromCollection(customTypeData);
// should not work, incompatible types
ds1.coGroup(ds2).where(2).equalTo(new KeySelector<CustomType, Long>() {
@Override
public Long getKey(CustomType value) {
return value.myLong;
}
});
}
14
View Complete Implementation : MaxByOperatorTest.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
/**
* This test validates that an InvalidProgramException is thrown when maxBy
* is used on a custom data type.
*/
@Test(expected = InvalidProgramException.clreplaced)
public void testCustomKeyFieldsGrouping() {
final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
this.customTypeData.add(new CustomType());
UnsortedGrouping<CustomType> groupDs = env.fromCollection(customTypeData).groupBy(0);
// should not work: groups on custom type
groupDs.maxBy(0);
}
14
View Complete Implementation : MinByOperatorTest.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
/**
* This test validates that an index which is out of bounds throws an
* IndexOutOfBoundsException.
*/
@Test(expected = IndexOutOfBoundsException.clreplaced)
public void testOutOfTupleBoundsDataset1() {
final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
DataSet<Tuple5<Integer, Long, String, Long, Integer>> tupleDs = env.fromCollection(emptyTupleData, tupleTypeInfo);
// should not work, key out of tuple bounds
tupleDs.minBy(5);
}
14
View Complete Implementation : ReduceWithCombinerITCase.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
@Test
public void testReduceOnNonKeyedDataset() throws Exception {
final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
env.setParallelism(4);
// creates the input data and distributes them evenly among the available downstream tasks
DataSet<Tuple2<Integer, Boolean>> input = createNonKeyedInput(env);
List<Tuple2<Integer, Boolean>> actual = input.reduceGroup(new NonKeyedCombReducer()).collect();
String expected = "10,true\n";
compareResultAsTuples(actual, expected);
}
14
View Complete Implementation : CustomPartitioningTest.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
@Test
public void testParreplacedionPojoInvalidType() {
try {
final int parallelism = 4;
ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
env.setParallelism(parallelism);
DataSet<Pojo> data = env.fromElements(new Pojo()).rebalance();
try {
data.parreplacedionCustom(new TestParreplacedionerLong(), "a");
fail("Should throw an exception");
} catch (InvalidProgramException e) {
// expected
}
} catch (Exception e) {
e.printStackTrace();
fail(e.getMessage());
}
}
14
View Complete Implementation : MinByOperatorTest.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
/**
* This test validates that an InvalidProgramException is thrown when minBy
* is used on a custom data type.
*/
@Test(expected = InvalidProgramException.clreplaced)
public void testCustomKeyFieldsGrouping() {
final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
this.customTypeData.add(new CustomType());
UnsortedGrouping<CustomType> groupDs = env.fromCollection(customTypeData).groupBy(0);
// should not work: groups on custom type
groupDs.minBy(0);
}
14
View Complete Implementation : MinByOperatorTest.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
/**
* This test validates that an index which is out of bounds throws an
* IndexOutOfBoundsException.
*/
@Test(expected = IndexOutOfBoundsException.clreplaced)
public void testOutOfTupleBoundsGrouping2() {
final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
UnsortedGrouping<Tuple5<Integer, Long, String, Long, Integer>> groupDs = env.fromCollection(emptyTupleData, tupleTypeInfo).groupBy(0);
// should not work, key out of tuple bounds
groupDs.minBy(-1);
}
14
View Complete Implementation : MaxByOperatorTest.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
/**
* This test validates that an InvalidProgramException is thrown when maxBy
* is used on a custom data type.
*/
@Test(expected = InvalidProgramException.clreplaced)
public void testCustomKeyFieldsDataset() {
final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
this.customTypeData.add(new CustomType());
DataSet<CustomType> customDs = env.fromCollection(customTypeData);
// should not work: groups on custom type
customDs.maxBy(0);
}
14
View Complete Implementation : MinByOperatorTest.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
/**
* This test validates that an index which is out of bounds throws an
* IndexOutOfBoundsException.
*/
@Test(expected = IndexOutOfBoundsException.clreplaced)
public void testOutOfTupleBoundsGrouping3() {
final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
UnsortedGrouping<Tuple5<Integer, Long, String, Long, Integer>> groupDs = env.fromCollection(emptyTupleData, tupleTypeInfo).groupBy(0);
// should not work, key out of tuple bounds
groupDs.minBy(1, 2, 3, 4, -1);
}
14
View Complete Implementation : WordCountWithExternalClass2.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
public static void main(String[] args) throws Exception {
// set up the execution environment
final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
// get input data
DataSet<String> text = StaticData.getDefaultTextLineDataSet(env);
DataSet<Tuple2<String, Integer>> counts = // split up the lines in pairs (2-tuples) containing: (word,1)
text.flatMap(new ExternalTokenizer2()).groupBy(0).sum(1);
// emit result
counts.print();
// execute program
env.execute("WordCount Example");
}
14
View Complete Implementation : ReduceWithCombinerITCase.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
@Test
public void testReduceOnKeyedDataset() throws Exception {
// set up the execution environment
final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
env.setParallelism(4);
// creates the input data and distributes them evenly among the available downstream tasks
DataSet<Tuple3<String, Integer, Boolean>> input = createKeyedInput(env);
List<Tuple3<String, Integer, Boolean>> actual = input.groupBy(0).reduceGroup(new KeyedCombReducer()).collect();
String expected = "k1,6,true\nk2,4,true\n";
compareResultAsTuples(actual, expected);
}
14
View Complete Implementation : GroupingPojoTranslationTest.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
@Test
public void testCustomParreplacedioningTupleRejectCompositeKey() {
try {
ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
DataSet<Pojo2> data = env.fromElements(new Pojo2()).rebalance().setParallelism(4);
try {
data.groupBy("a", "b").withParreplacedioner(new TestParreplacedionerInt());
fail("Should throw an exception");
} catch (InvalidProgramException e) {
}
} catch (Exception e) {
e.printStackTrace();
fail(e.getMessage());
}
}
14
View Complete Implementation : InputOutputITCase.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
@Override
protected void testProgram() throws Exception {
final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
TestNonRichOutputFormat output = new TestNonRichOutputFormat();
env.createInput(new TestNonRichInputFormat()).output(output);
try {
env.execute();
} catch (Exception e) {
// we didn't break anything by making everything rich.
e.printStackTrace();
fail(e.getMessage());
}
}
14
View Complete Implementation : WordCountWithInnerClass.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
public static void main(String[] args) throws Exception {
// set up the execution environment
final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
// get input data
DataSet<String> text = StaticData.getDefaultTextLineDataSet(env);
DataSet<Tuple2<String, Integer>> counts = // split up the lines in pairs (2-tuples) containing: (word,1)
text.flatMap(new Tokenizer()).groupBy(0).sum(1);
// emit result
counts.print();
// execute program
env.execute("WordCount Example");
}
14
View Complete Implementation : ReduceWithCombinerITCase.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
@Test
public void testReduceOnKeyedDatasetWithSelector() throws Exception {
// set up the execution environment
final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
env.setParallelism(4);
// creates the input data and distributes them evenly among the available downstream tasks
DataSet<Tuple3<String, Integer, Boolean>> input = createKeyedInput(env);
List<Tuple3<String, Integer, Boolean>> actual = input.groupBy(new KeySelectorX()).reduceGroup(new KeyedCombReducer()).collect();
String expected = "k1,6,true\nk2,4,true\n";
compareResultAsTuples(actual, expected);
}
14
View Complete Implementation : LargePlanTest.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
private static void runProgram(int depth, int width) throws Exception {
ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
DataSet<String> input = env.fromElements("a", "b", "c");
DataSet<String> stats = null;
for (int i = 0; i < depth; i++) {
stats = replacedyze(input, stats, width / (i + 1) + 1);
}
stats.output(new DiscardingOutputFormat<>());
env.createProgramPlan("depth " + depth + " width " + width);
}
14
View Complete Implementation : CustomDistributionITCase.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
/*
* Test the number of parreplacedion keys larger than the number of distribution fields
*/
@Test(expected = IllegalArgumentException.clreplaced)
public void testParreplacedionMoreThanDistribution() throws Exception {
final TestDataDist2 dist = new TestDataDist2();
ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
DataSet<Tuple3<Integer, Long, String>> input = CollectionDataSets.get3TupleDataSet(env);
DataSetUtils.parreplacedionByRange(input, dist, 0, 1, 2);
}
14
View Complete Implementation : JoinOperatorTest.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
@Test(expected = InvalidProgramException.clreplaced)
public void testJoinKeyMixing3() {
final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
DataSet<Tuple5<Integer, Long, String, Long, Integer>> ds1 = env.fromCollection(emptyTupleData, tupleTypeInfo);
DataSet<CustomType> ds2 = env.fromCollection(customTypeData);
// should not work, incompatible types
ds1.join(ds2).where(2).equalTo(new KeySelector<CustomType, Long>() {
@Override
public Long getKey(CustomType value) {
return value.myLong;
}
});
}