Here are the examples of the java api org.apache.flink.api.java.ExecutionEnvironment taken from open source projects. By voting up you can indicate which examples are most useful and appropriate.
155 Examples
19
View Complete Implementation : DataSetAnalyticBase.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
/**
* Base clreplaced for {@link DataSetreplacedytic}.
*
* @param <T> element type
* @param <R> the return type
*/
public abstract clreplaced DataSetreplacedyticBase<T, R> implements DataSetreplacedytic<T, R> {
protected ExecutionEnvironment env;
@Override
public DataSetreplacedyticBase<T, R> run(DataSet<T> input) throws Exception {
env = input.getExecutionEnvironment();
return this;
}
@Override
public R execute() throws Exception {
env.execute();
return getResult();
}
@Override
public R execute(String jobName) throws Exception {
Preconditions.checkNotNull(jobName);
env.execute(jobName);
return getResult();
}
}
19
View Complete Implementation : BipartiteGraph.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
/**
* Create biparreplacede graph from datasets.
*
* @param topVertices dataset of top vertices in the graph
* @param bottomVertices dataset of bottom vertices in the graph
* @param edges dataset of edges between vertices
* @param context Flink execution context
* @return new biparreplacede graph created from provided datasets
*/
public static <KT, KB, VVT, VVB, EV> BiparreplacedeGraph<KT, KB, VVT, VVB, EV> fromDataSet(DataSet<Vertex<KT, VVT>> topVertices, DataSet<Vertex<KB, VVB>> bottomVertices, DataSet<BiparreplacedeEdge<KT, KB, EV>> edges, ExecutionEnvironment context) {
return new BiparreplacedeGraph<>(topVertices, bottomVertices, edges, context);
}
19
View Complete Implementation : GraphGeneratorTestBase.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
/**
* Base clreplaced for graph generator tests.
*/
public abstract clreplaced GraphGeneratorTestBase {
protected ExecutionEnvironment env;
@Before
public void setup() {
env = ExecutionEnvironment.createCollectionsEnvironment();
}
}
19
View Complete Implementation : HBaseConnectorITCase.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
// ------------------------------- Utilities -------------------------------------------------
/**
* Creates a Batch {@link TableEnvironment} depends on the {@link #planner} context.
*/
private TableEnvironment createBatchTableEnv() {
if (OLD_PLANNER.equals(planner)) {
ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
return BatchTableEnvironment.create(env, new TableConfig());
} else {
return TableEnvironment.create(batchSettings);
}
}
19
View Complete Implementation : MiscellaneousIssuesITCase.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
@Test
public void testDisjointDataflows() {
try {
ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
env.setParallelism(5);
// generate two different flows
env.generateSequence(1, 10).output(new DiscardingOutputFormat<Long>());
env.generateSequence(1, 10).output(new DiscardingOutputFormat<Long>());
} catch (Exception e) {
e.printStackTrace();
fail(e.getMessage());
}
}
19
View Complete Implementation : CountTest.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
/**
* Tests for {@link Count}.
*/
public clreplaced CountTest {
private ExecutionEnvironment env;
@Before
public void setup() throws Exception {
env = ExecutionEnvironment.createCollectionsEnvironment();
env.getConfig().enableObjectReuse();
}
@Test
public void testList() throws Exception {
List<Long> list = Arrays.asList(ArrayUtils.toObject(new long[] { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 }));
DataSet<Long> dataset = env.fromCollection(list);
long count = new Count<Long>().run(dataset).execute();
replacedertEquals(list.size(), count);
}
@Test
public void testEmptyList() throws Exception {
DataSet<Long> dataset = env.fromCollection(Collections.emptyList(), Types.LONG);
long count = new Count<Long>().run(dataset).execute();
replacedertEquals(0, count);
}
}
19
View Complete Implementation : PathGraph.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
/**
* @see <a href="http://mathworld.wolfram.com/PathGraph.html">Path Graph at Wolfram MathWorld</a>
*/
public clreplaced PathGraph extends GraphGeneratorBase<LongValue, NullValue, NullValue> {
public static final int MINIMUM_VERTEX_COUNT = 2;
// Required to create the DataSource
private final ExecutionEnvironment env;
// Required configuration
private final long vertexCount;
/**
* An undirected {@link Graph} with {@code n} vertices where each vertex
* v<sub>i</sub> connects to adjacent vertices v<sub>i+1</sub> when
* {@code i < n-1} and v<sub>i-1</sub> when {@code i > 0}.
*
* <p>A {@code PathGraph} is distinguished from a {@code CycleGraph} in that
* the first and last vertex are not connected, breaking the cycle.
*
* @param env the Flink execution environment
* @param vertexCount number of vertices
*/
public PathGraph(ExecutionEnvironment env, long vertexCount) {
Preconditions.checkArgument(vertexCount >= MINIMUM_VERTEX_COUNT, "Vertex count must be at least " + MINIMUM_VERTEX_COUNT);
this.env = env;
this.vertexCount = vertexCount;
}
@Override
public Graph<LongValue, NullValue, NullValue> generate() {
return new GridGraph(env).addDimension(vertexCount, false).setParallelism(parallelism).generate();
}
}
19
View Complete Implementation : CycleGraph.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
/**
* @see <a href="http://mathworld.wolfram.com/CycleGraph.html">Cycle Graph at Wolfram MathWorld</a>
*/
public clreplaced CycleGraph extends GraphGeneratorBase<LongValue, NullValue, NullValue> {
public static final int MINIMUM_VERTEX_COUNT = 2;
// Required to create the DataSource
private final ExecutionEnvironment env;
// Required configuration
private final long vertexCount;
/**
* An undirected {@link Graph} with {@code n} vertices where each vertex
* v<sub>i</sub> is connected to adjacent vertices v<sub>(i+1)%n</sub> and
* v<sub>(i-1)%n</sub>.
*
* @param env the Flink execution environment
* @param vertexCount number of vertices
*/
public CycleGraph(ExecutionEnvironment env, long vertexCount) {
Preconditions.checkArgument(vertexCount >= MINIMUM_VERTEX_COUNT, "Vertex count must be at least " + MINIMUM_VERTEX_COUNT);
this.env = env;
this.vertexCount = vertexCount;
}
@Override
public Graph<LongValue, NullValue, NullValue> generate() {
return new GridGraph(env).addDimension(vertexCount, true).setParallelism(parallelism).generate();
}
}
19
View Complete Implementation : OrcTableSource.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
@Override
public DataSet<Row> getDataSet(ExecutionEnvironment execEnv) {
OrcRowInputFormat orcIF = buildOrcInputFormat();
orcIF.setNestedFileEnumeration(recursiveEnumeration);
if (selectedFields != null) {
orcIF.selectFields(selectedFields);
}
if (predicates != null) {
for (OrcSplitReader.Predicate pred : predicates) {
orcIF.addPredicate(pred);
}
}
return execEnv.createInput(orcIF).name(explainSource());
}
19
View Complete Implementation : InputOutputITCase.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
@Override
protected void testProgram() throws Exception {
final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
TestNonRichOutputFormat output = new TestNonRichOutputFormat();
env.createInput(new TestNonRichInputFormat()).output(output);
try {
env.execute();
} catch (Exception e) {
// we didn't break anything by making everything rich.
e.printStackTrace();
fail(e.getMessage());
}
}
19
View Complete Implementation : ChecksumHashCodeTest.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
/**
* Tests for {@link ChecksumHashCode}.
*/
public clreplaced ChecksumHashCodeTest {
private ExecutionEnvironment env;
@Before
public void setup() throws Exception {
env = ExecutionEnvironment.createCollectionsEnvironment();
env.getConfig().enableObjectReuse();
}
@Test
public void testList() throws Exception {
List<Long> list = Arrays.asList(ArrayUtils.toObject(new long[] { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 }));
DataSet<Long> dataset = env.fromCollection(list);
Checksum checksum = new ChecksumHashCode<Long>().run(dataset).execute();
replacedertEquals(list.size(), checksum.getCount());
replacedertEquals(list.size() * (list.size() - 1) / 2, checksum.getChecksum());
}
@Test
public void testEmptyList() throws Exception {
DataSet<Long> dataset = env.fromCollection(Collections.emptyList(), Types.LONG);
Checksum checksum = new ChecksumHashCode<Long>().run(dataset).execute();
replacedertEquals(0, checksum.getCount());
replacedertEquals(0, checksum.getChecksum());
}
}
19
View Complete Implementation : CompleteGraph.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
/**
* @see <a href="http://mathworld.wolfram.com/CompleteGraph.html">Complete Graph at Wolfram MathWorld</a>
*/
public clreplaced CompleteGraph extends GraphGeneratorBase<LongValue, NullValue, NullValue> {
public static final int MINIMUM_VERTEX_COUNT = 2;
// Required to create the DataSource
private final ExecutionEnvironment env;
// Required configuration
private final long vertexCount;
/**
* An undirected {@link Graph} connecting every distinct pair of vertices.
*
* @param env the Flink execution environment
* @param vertexCount number of vertices
*/
public CompleteGraph(ExecutionEnvironment env, long vertexCount) {
Preconditions.checkArgument(vertexCount >= MINIMUM_VERTEX_COUNT, "Vertex count must be at least " + MINIMUM_VERTEX_COUNT);
this.env = env;
this.vertexCount = vertexCount;
}
@Override
public Graph<LongValue, NullValue, NullValue> generate() {
return new CirculantGraph(env, vertexCount).addRange(1, vertexCount - 1).setParallelism(parallelism).generate();
}
}
19
View Complete Implementation : AccumulatorErrorITCase.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
@Test
public void testInvalidTypeAcreplacedulator() throws Exception {
ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
// Test Exception forwarding with faulty Acreplacedulator implementation
env.generateSequence(0, 10000).map(new IncompatibleAcreplacedulatorTypesMapper()).map(new IncompatibleAcreplacedulatorTypesMapper2()).output(new DiscardingOutputFormat<>());
try {
env.execute();
fail("Should have failed.");
} catch (JobExecutionException e) {
replacedertTrue(findThrowable(e, UnsupportedOperationException.clreplaced).isPresent());
}
}
19
View Complete Implementation : EmptyGraph.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
/**
* @see <a href="http://mathworld.wolfram.com/EmptyGraph.html">Empty Graph at Wolfram MathWorld</a>
*/
public clreplaced EmptyGraph extends GraphGeneratorBase<LongValue, NullValue, NullValue> {
public static final int MINIMUM_VERTEX_COUNT = 0;
// Required to create the DataSource
private final ExecutionEnvironment env;
// Required configuration
private final long vertexCount;
/**
* The {@link Graph} containing no edges.
*
* @param env the Flink execution environment
* @param vertexCount number of vertices
*/
public EmptyGraph(ExecutionEnvironment env, long vertexCount) {
Preconditions.checkArgument(vertexCount >= MINIMUM_VERTEX_COUNT, "Vertex count must be at least " + MINIMUM_VERTEX_COUNT);
this.env = env;
this.vertexCount = vertexCount;
}
@Override
public Graph<LongValue, NullValue, NullValue> generate() {
Preconditions.checkState(vertexCount >= 0);
// Vertices
DataSet<Vertex<LongValue, NullValue>> vertices = GraphGeneratorUtils.vertexSequence(env, parallelism, vertexCount);
// Edges
DataSource<Edge<LongValue, NullValue>> edges = env.fromCollection(Collections.<Edge<LongValue, NullValue>>emptyList(), TypeInformation.of(new TypeHint<Edge<LongValue, NullValue>>() {
})).setParallelism(parallelism).name("Empty edge set");
// Graph
return Graph.fromDataSet(vertices, edges, env);
}
}
19
View Complete Implementation : GraphGeneratorUtils.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
/**
* Generates {@link Vertex Vertices} with sequential, numerical labels.
*
* @param env the Flink execution environment.
* @param parallelism operator parallelism
* @param vertexCount number of sequential vertex labels
* @return {@link DataSet} of sequentially labeled {@link Vertex vertices}
*/
public static DataSet<Vertex<LongValue, NullValue>> vertexSequence(ExecutionEnvironment env, int parallelism, long vertexCount) {
Preconditions.checkArgument(vertexCount >= 0, "Vertex count must be non-negative");
if (vertexCount == 0) {
return env.fromCollection(Collections.emptyList(), TypeInformation.of(new TypeHint<Vertex<LongValue, NullValue>>() {
})).setParallelism(parallelism).name("Empty vertex set");
} else {
LongValueSequenceIterator iterator = new LongValueSequenceIterator(0, vertexCount - 1);
DataSource<LongValue> vertexLabels = env.fromParallelCollection(iterator, LongValue.clreplaced).setParallelism(parallelism).name("Vertex indices");
return vertexLabels.map(new CreateVertex()).setParallelism(parallelism).name("Vertex sequence");
}
}
19
View Complete Implementation : StreamPlanEnvironment.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
/**
* A special {@link StreamExecutionEnvironment} that is used in the web frontend when generating
* a user-inspectable graph of a streaming job.
*/
@PublicEvolving
public clreplaced StreamPlanEnvironment extends StreamExecutionEnvironment {
private ExecutionEnvironment env;
protected StreamPlanEnvironment(ExecutionEnvironment env) {
this.env = env;
int parallelism = env.getParallelism();
if (parallelism > 0) {
setParallelism(parallelism);
} else {
// determine parallelism
setParallelism(GlobalConfiguration.loadConfiguration().getInteger(CoreOptions.DEFAULT_PARALLELISM));
}
}
@Override
public JobClient executeAsync(StreamGraph streamGraph) throws Exception {
if (env instanceof OptimizerPlanEnvironment) {
((OptimizerPlanEnvironment) env).setPipeline(streamGraph);
}
throw new OptimizerPlanEnvironment.ProgramAbortException();
}
}
19
View Complete Implementation : CollectTest.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
/**
* Tests for {@link Collect}.
*/
public clreplaced CollectTest {
private ExecutionEnvironment env;
@Before
public void setup() throws Exception {
env = ExecutionEnvironment.createCollectionsEnvironment();
env.getConfig().enableObjectReuse();
}
@Test
public void testList() throws Exception {
List<Long> list = Arrays.asList(ArrayUtils.toObject(new long[] { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 }));
DataSet<Long> dataset = env.fromCollection(list);
List<Long> collected = new Collect<Long>().run(dataset).execute();
replacedertArrayEquals(list.toArray(), collected.toArray());
}
@Test
public void testEmptyList() throws Exception {
DataSet<Long> dataset = env.fromCollection(Collections.emptyList(), Types.LONG);
List<Long> collected = new Collect<Long>().run(dataset).execute();
replacedertEquals(0, collected.size());
}
}
19
View Complete Implementation : HypercubeGraph.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
/**
* @see <a href="http://mathworld.wolfram.com/HypercubeGraph.html">Hypercube Graph at Wolfram MathWorld</a>
*/
public clreplaced HypercubeGraph extends GraphGeneratorBase<LongValue, NullValue, NullValue> {
public static final int MINIMUM_DIMENSIONS = 1;
// Required to create the DataSource
private final ExecutionEnvironment env;
// Required configuration
private final long dimensions;
/**
* An undirected {@code Graph} where edges form an n-dimensional hypercube.
* Each vertex in a hypercube connects to one other vertex in each
* dimension.
*
* @param env the Flink execution environment
* @param dimensions number of dimensions
*/
public HypercubeGraph(ExecutionEnvironment env, long dimensions) {
Preconditions.checkArgument(dimensions >= MINIMUM_DIMENSIONS, "Number of dimensions must be at least " + MINIMUM_DIMENSIONS);
this.env = env;
this.dimensions = dimensions;
}
@Override
public Graph<LongValue, NullValue, NullValue> generate() {
Preconditions.checkState(dimensions > 0);
GridGraph graph = new GridGraph(env);
for (int i = 0; i < dimensions; i++) {
graph.addDimension(2, false);
}
return graph.setParallelism(parallelism).generate();
}
}
18
View Complete Implementation : MaxByOperatorTest.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
/**
* This test validates that an InvalidProgramException is thrown when maxBy
* is used on a custom data type.
*/
@Test(expected = InvalidProgramException.clreplaced)
public void testCustomKeyFieldsDataset() {
final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
this.customTypeData.add(new CustomType());
DataSet<CustomType> customDs = env.fromCollection(customTypeData);
// should not work: groups on custom type
customDs.maxBy(0);
}
18
View Complete Implementation : GroupReduceOperatorTest.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
@Test
public void testSemanticPropsWithKeySelector6() {
final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
DataSet<Tuple5<Integer, Long, String, Long, Integer>> tupleDs = env.fromCollection(emptyTupleData, tupleTypeInfo);
GroupReduceOperator<Tuple5<Integer, Long, String, Long, Integer>, Tuple5<Integer, Long, String, Long, Integer>> reduceOp = tupleDs.groupBy(new DummyTestKeySelector()).sortGroup(new DummyTestKeySelector(), Order.ASCENDING).reduceGroup(new DummyGroupReduceFunction3()).withForwardedFields("4->0;3;3->1;2");
SemanticProperties semProps = reduceOp.getSemanticProperties();
replacedertTrue(semProps.getForwardingTargetFields(0, 0).size() == 0);
replacedertTrue(semProps.getForwardingTargetFields(0, 1).size() == 0);
replacedertTrue(semProps.getForwardingTargetFields(0, 2).size() == 0);
replacedertTrue(semProps.getForwardingTargetFields(0, 3).size() == 0);
replacedertTrue(semProps.getForwardingTargetFields(0, 4).size() == 0);
replacedertTrue(semProps.getForwardingTargetFields(0, 5).size() == 0);
replacedertTrue(semProps.getForwardingTargetFields(0, 6).size() == 1);
replacedertTrue(semProps.getForwardingTargetFields(0, 6).contains(2));
replacedertTrue(semProps.getForwardingTargetFields(0, 7).size() == 2);
replacedertTrue(semProps.getForwardingTargetFields(0, 7).contains(1));
replacedertTrue(semProps.getForwardingTargetFields(0, 7).contains(3));
replacedertTrue(semProps.getForwardingTargetFields(0, 8).size() == 1);
replacedertTrue(semProps.getForwardingTargetFields(0, 8).contains(0));
replacedertTrue(semProps.getForwardingSourceField(0, 0) == 8);
replacedertTrue(semProps.getForwardingSourceField(0, 1) == 7);
replacedertTrue(semProps.getForwardingSourceField(0, 2) == 6);
replacedertTrue(semProps.getForwardingSourceField(0, 3) == 7);
replacedertTrue(semProps.getForwardingSourceField(0, 4) < 0);
replacedertTrue(semProps.getReadFields(0) == null);
}
18
View Complete Implementation : ValueCollectionDataSets.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
public static DataSet<PojoWithDateAndEnum> getPojoWithDateAndEnum(ExecutionEnvironment env) {
List<PojoWithDateAndEnum> data = new ArrayList<PojoWithDateAndEnum>();
PojoWithDateAndEnum one = new PojoWithDateAndEnum();
one.group = new StringValue("a");
one.date = new Date(666);
one.cat = Category.CAT_A;
data.add(one);
PojoWithDateAndEnum two = new PojoWithDateAndEnum();
two.group = new StringValue("a");
two.date = new Date(666);
two.cat = Category.CAT_A;
data.add(two);
PojoWithDateAndEnum three = new PojoWithDateAndEnum();
three.group = new StringValue("b");
three.date = new Date(666);
three.cat = Category.CAT_B;
data.add(three);
return env.fromCollection(data);
}
18
View Complete Implementation : CoGroupOperatorTest.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
@Test(expected = InvalidProgramException.clreplaced)
public void testCoGroupKeyMixing3() {
final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
DataSet<Tuple5<Integer, Long, String, Long, Integer>> ds1 = env.fromCollection(emptyTupleData, tupleTypeInfo);
DataSet<CustomType> ds2 = env.fromCollection(customTypeData);
// should not work, incompatible types
ds1.coGroup(ds2).where(2).equalTo(new KeySelector<CustomType, Long>() {
@Override
public Long getKey(CustomType value) {
return value.myLong;
}
});
}
18
View Complete Implementation : WordCountSimplePOJOITCase.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
@Override
protected void testProgram() throws Exception {
final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
DataSet<String> text = env.readTextFile(textPath);
DataSet<WC> counts = text.flatMap(new Tokenizer()).groupBy("word").reduce(new ReduceFunction<WC>() {
private static final long serialVersionUID = 1L;
public WC reduce(WC value1, WC value2) {
return new WC(value1.word, value1.count + value2.count);
}
});
counts.writeAsText(resultPath);
env.execute("WordCount with custom data types example");
}
18
View Complete Implementation : GroupCombineOperatorTest.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
@Test
public void testSemanticPropsWithKeySelector5() {
final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
DataSet<Tuple5<Integer, Long, String, Long, Integer>> tupleDs = env.fromCollection(emptyTupleData, tupleTypeInfo);
GroupCombineOperator<Tuple5<Integer, Long, String, Long, Integer>, Tuple5<Integer, Long, String, Long, Integer>> combineOp = tupleDs.groupBy(new DummyTestKeySelector()).combineGroup(new DummyGroupCombineFunction3()).withForwardedFields("4->0;3;3->1;2");
SemanticProperties semProps = combineOp.getSemanticProperties();
replacedertTrue(semProps.getForwardingTargetFields(0, 0).size() == 0);
replacedertTrue(semProps.getForwardingTargetFields(0, 1).size() == 0);
replacedertTrue(semProps.getForwardingTargetFields(0, 2).size() == 0);
replacedertTrue(semProps.getForwardingTargetFields(0, 3).size() == 0);
replacedertTrue(semProps.getForwardingTargetFields(0, 4).size() == 1);
replacedertTrue(semProps.getForwardingTargetFields(0, 4).contains(2));
replacedertTrue(semProps.getForwardingTargetFields(0, 5).size() == 2);
replacedertTrue(semProps.getForwardingTargetFields(0, 5).contains(1));
replacedertTrue(semProps.getForwardingTargetFields(0, 5).contains(3));
replacedertTrue(semProps.getForwardingTargetFields(0, 6).size() == 1);
replacedertTrue(semProps.getForwardingTargetFields(0, 6).contains(0));
replacedertTrue(semProps.getForwardingSourceField(0, 0) == 6);
replacedertTrue(semProps.getForwardingSourceField(0, 1) == 5);
replacedertTrue(semProps.getForwardingSourceField(0, 2) == 4);
replacedertTrue(semProps.getForwardingSourceField(0, 3) == 5);
replacedertTrue(semProps.getForwardingSourceField(0, 4) < 0);
replacedertTrue(semProps.getReadFields(0) == null);
}
18
View Complete Implementation : HBaseReadExample.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
public static void main(String[] args) throws Exception {
ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
@SuppressWarnings("serial")
DataSet<Tuple2<String, String>> hbaseDs = env.createInput(new TableInputFormat<Tuple2<String, String>>() {
@Override
public String getTableName() {
return HBaseFlinkTestConstants.TEST_TABLE_NAME;
}
@Override
protected Scan getScanner() {
Scan scan = new Scan();
scan.addColumn(HBaseFlinkTestConstants.CF_SOME, HBaseFlinkTestConstants.Q_SOME);
return scan;
}
private Tuple2<String, String> reuse = new Tuple2<String, String>();
@Override
protected Tuple2<String, String> mapResultToTuple(Result r) {
String key = Bytes.toString(r.getRow());
String val = Bytes.toString(r.getValue(HBaseFlinkTestConstants.CF_SOME, HBaseFlinkTestConstants.Q_SOME));
reuse.setField(key, 0);
reuse.setField(val, 1);
return reuse;
}
}).filter(new FilterFunction<Tuple2<String, String>>() {
@Override
public boolean filter(Tuple2<String, String> t) throws Exception {
String val = t.getField(1);
if (val.startsWith("someStr")) {
return true;
}
return false;
}
});
hbaseDs.print();
}
18
View Complete Implementation : GroupCombineOperatorTest.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
@Test
public void testSemanticPropsWithKeySelector3() {
final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
DataSet<Tuple5<Integer, Long, String, Long, Integer>> tupleDs = env.fromCollection(emptyTupleData, tupleTypeInfo);
GroupCombineOperator<Tuple5<Integer, Long, String, Long, Integer>, Tuple5<Integer, Long, String, Long, Integer>> combineOp = tupleDs.groupBy(new DummyTestKeySelector()).combineGroup(new DummyGroupCombineFunction2()).withForwardedFields("0->4;1;1->3;2");
SemanticProperties semProps = combineOp.getSemanticProperties();
replacedertTrue(semProps.getForwardingTargetFields(0, 0).size() == 0);
replacedertTrue(semProps.getForwardingTargetFields(0, 1).size() == 0);
replacedertTrue(semProps.getForwardingTargetFields(0, 2).size() == 1);
replacedertTrue(semProps.getForwardingTargetFields(0, 2).contains(4));
replacedertTrue(semProps.getForwardingTargetFields(0, 3).size() == 2);
replacedertTrue(semProps.getForwardingTargetFields(0, 3).contains(1));
replacedertTrue(semProps.getForwardingTargetFields(0, 3).contains(3));
replacedertTrue(semProps.getForwardingTargetFields(0, 4).size() == 1);
replacedertTrue(semProps.getForwardingTargetFields(0, 4).contains(2));
replacedertTrue(semProps.getForwardingTargetFields(0, 5).size() == 0);
replacedertTrue(semProps.getForwardingTargetFields(0, 6).size() == 0);
replacedertTrue(semProps.getForwardingSourceField(0, 0) < 0);
replacedertTrue(semProps.getForwardingSourceField(0, 1) == 3);
replacedertTrue(semProps.getForwardingSourceField(0, 2) == 4);
replacedertTrue(semProps.getForwardingSourceField(0, 3) == 3);
replacedertTrue(semProps.getForwardingSourceField(0, 4) == 2);
replacedertTrue(semProps.getReadFields(0).size() == 3);
replacedertTrue(semProps.getReadFields(0).contains(2));
replacedertTrue(semProps.getReadFields(0).contains(5));
replacedertTrue(semProps.getReadFields(0).contains(6));
}
18
View Complete Implementation : ValueCollectionDataSets.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
public static DataSet<Tuple3<Tuple2<IntValue, IntValue>, StringValue, IntValue>> getGroupSortedNestedTupleDataSet2(ExecutionEnvironment env) {
List<Tuple3<Tuple2<IntValue, IntValue>, StringValue, IntValue>> data = new ArrayList<>();
data.add(new Tuple3<>(new Tuple2<IntValue, IntValue>(new IntValue(1), new IntValue(3)), new StringValue("a"), new IntValue(2)));
data.add(new Tuple3<>(new Tuple2<IntValue, IntValue>(new IntValue(1), new IntValue(2)), new StringValue("a"), new IntValue(1)));
data.add(new Tuple3<>(new Tuple2<IntValue, IntValue>(new IntValue(2), new IntValue(1)), new StringValue("a"), new IntValue(3)));
data.add(new Tuple3<>(new Tuple2<IntValue, IntValue>(new IntValue(2), new IntValue(2)), new StringValue("b"), new IntValue(4)));
data.add(new Tuple3<>(new Tuple2<IntValue, IntValue>(new IntValue(3), new IntValue(3)), new StringValue("c"), new IntValue(5)));
data.add(new Tuple3<>(new Tuple2<IntValue, IntValue>(new IntValue(3), new IntValue(6)), new StringValue("c"), new IntValue(6)));
data.add(new Tuple3<>(new Tuple2<IntValue, IntValue>(new IntValue(4), new IntValue(9)), new StringValue("c"), new IntValue(7)));
TupleTypeInfo<Tuple3<Tuple2<IntValue, IntValue>, StringValue, IntValue>> type = new TupleTypeInfo<>(new TupleTypeInfo<Tuple2<IntValue, IntValue>>(ValueTypeInfo.INT_VALUE_TYPE_INFO, ValueTypeInfo.INT_VALUE_TYPE_INFO), ValueTypeInfo.STRING_VALUE_TYPE_INFO, ValueTypeInfo.INT_VALUE_TYPE_INFO);
return env.fromCollection(data, type);
}
18
View Complete Implementation : CustomDistributionITCase.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
/*
* Test the number of parreplacedion keys larger than the number of distribution fields
*/
@Test(expected = IllegalArgumentException.clreplaced)
public void testParreplacedionMoreThanDistribution() throws Exception {
final TestDataDist2 dist = new TestDataDist2();
ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
DataSet<Tuple3<Integer, Long, String>> input = CollectionDataSets.get3TupleDataSet(env);
DataSetUtils.parreplacedionByRange(input, dist, 0, 1, 2);
}
18
View Complete Implementation : ValueCollectionDataSets.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
public static DataSet<Tuple3<IntValue, CrazyNested, POJO>> getTupleContainingPojos(ExecutionEnvironment env) {
List<Tuple3<IntValue, CrazyNested, POJO>> data = new ArrayList<>();
// 3x
data.add(new Tuple3<IntValue, CrazyNested, POJO>(new IntValue(1), new CrazyNested("one", "uno", 1L), new POJO(1, "First", 10, 100, 1000L, "One", 10000L)));
data.add(new Tuple3<IntValue, CrazyNested, POJO>(new IntValue(1), new CrazyNested("one", "uno", 1L), new POJO(1, "First", 10, 100, 1000L, "One", 10000L)));
data.add(new Tuple3<IntValue, CrazyNested, POJO>(new IntValue(1), new CrazyNested("one", "uno", 1L), new POJO(1, "First", 10, 100, 1000L, "One", 10000L)));
// POJO is not initialized according to the first two fields.
// 1x
data.add(new Tuple3<IntValue, CrazyNested, POJO>(new IntValue(2), new CrazyNested("two", "duo", 2L), new POJO(1, "First", 10, 100, 1000L, "One", 10000L)));
return env.fromCollection(data);
}
18
View Complete Implementation : BatchExample.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
/*
* table script: "CREATE TABLE test.batches (number int, strings text, PRIMARY KEY(number, strings));"
*/
public static void main(String[] args) throws Exception {
ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
env.setParallelism(1);
ArrayList<Tuple2<Integer, String>> collection = new ArrayList<>(20);
for (int i = 0; i < 20; i++) {
collection.add(new Tuple2<>(i, "string " + i));
}
DataSet<Tuple2<Integer, String>> dataSet = env.fromCollection(collection);
dataSet.output(new CreplacedandraTupleOutputFormat<Tuple2<Integer, String>>(INSERT_QUERY, new ClusterBuilder() {
@Override
protected Cluster buildCluster(Builder builder) {
return builder.addContactPoints("127.0.0.1").build();
}
}));
env.execute("Write");
DataSet<Tuple2<Integer, String>> inputDS = env.createInput(new CreplacedandraInputFormat<Tuple2<Integer, String>>(SELECT_QUERY, new ClusterBuilder() {
@Override
protected Cluster buildCluster(Builder builder) {
return builder.addContactPoints("127.0.0.1").build();
}
}), TupleTypeInfo.of(new TypeHint<Tuple2<Integer, String>>() {
}));
inputDS.print();
}
18
View Complete Implementation : AvroExternalJarProgram.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
// public static void main(String[] args) throws Exception {
// String testDataFile = new File("src/test/resources/testdata.avro").getAbsolutePath();
// writeTestData(new File(testDataFile), 50);
// }
public static void main(String[] args) throws Exception {
String inputPath = args[0];
ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
DataSet<MyUser> input = env.createInput(new AvroInputFormat<MyUser>(new Path(inputPath), MyUser.clreplaced));
DataSet<Tuple2<String, MyUser>> result = input.map(new NameExtractor()).groupBy(0).reduce(new NameGrouper());
result.output(new DiscardingOutputFormat<Tuple2<String, MyUser>>());
env.execute();
}
18
View Complete Implementation : GroupReduceOperatorTest.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
@Test
public void testSemanticPropsWithKeySelector1() {
final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
DataSet<Tuple5<Integer, Long, String, Long, Integer>> tupleDs = env.fromCollection(emptyTupleData, tupleTypeInfo);
GroupReduceOperator<Tuple5<Integer, Long, String, Long, Integer>, Tuple5<Integer, Long, String, Long, Integer>> reduceOp = tupleDs.groupBy(new DummyTestKeySelector()).reduceGroup(new DummyGroupReduceFunction1());
SemanticProperties semProps = reduceOp.getSemanticProperties();
replacedertTrue(semProps.getForwardingTargetFields(0, 0).size() == 0);
replacedertTrue(semProps.getForwardingTargetFields(0, 1).size() == 0);
replacedertTrue(semProps.getForwardingTargetFields(0, 2).size() == 1);
replacedertTrue(semProps.getForwardingTargetFields(0, 2).contains(4));
replacedertTrue(semProps.getForwardingTargetFields(0, 3).size() == 2);
replacedertTrue(semProps.getForwardingTargetFields(0, 3).contains(1));
replacedertTrue(semProps.getForwardingTargetFields(0, 3).contains(3));
replacedertTrue(semProps.getForwardingTargetFields(0, 4).size() == 1);
replacedertTrue(semProps.getForwardingTargetFields(0, 4).contains(2));
replacedertTrue(semProps.getForwardingTargetFields(0, 5).size() == 0);
replacedertTrue(semProps.getForwardingTargetFields(0, 6).size() == 0);
replacedertTrue(semProps.getForwardingSourceField(0, 0) < 0);
replacedertTrue(semProps.getForwardingSourceField(0, 1) == 3);
replacedertTrue(semProps.getForwardingSourceField(0, 2) == 4);
replacedertTrue(semProps.getForwardingSourceField(0, 3) == 3);
replacedertTrue(semProps.getForwardingSourceField(0, 4) == 2);
replacedertTrue(semProps.getReadFields(0).size() == 3);
replacedertTrue(semProps.getReadFields(0).contains(2));
replacedertTrue(semProps.getReadFields(0).contains(5));
replacedertTrue(semProps.getReadFields(0).contains(6));
}
18
View Complete Implementation : SummarizationData.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
/**
* Creates a set of vertices with attached {@link String} values.
*
* @param env execution environment
* @return vertex data set with string values
*/
public static DataSet<Vertex<Long, String>> getVertices(ExecutionEnvironment env) {
List<Vertex<Long, String>> vertices = new ArrayList<>(INPUT_VERTICES.length);
for (String vertex : INPUT_VERTICES) {
String[] tokens = vertex.split(";");
vertices.add(new Vertex<>(Long.parseLong(tokens[0]), tokens[1]));
}
return env.fromCollection(vertices);
}
18
View Complete Implementation : CollectionDataSets.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
public static DataSet<Tuple3<Integer, Long, String>> get3TupleDataSet(ExecutionEnvironment env) {
List<Tuple3<Integer, Long, String>> data = new ArrayList<>();
data.add(new Tuple3<>(1, 1L, "Hi"));
data.add(new Tuple3<>(2, 2L, "Hello"));
data.add(new Tuple3<>(3, 2L, "Hello world"));
data.add(new Tuple3<>(4, 3L, "Hello world, how are you?"));
data.add(new Tuple3<>(5, 3L, "I am fine."));
data.add(new Tuple3<>(6, 3L, "Luke Skywalker"));
data.add(new Tuple3<>(7, 4L, "Comment#1"));
data.add(new Tuple3<>(8, 4L, "Comment#2"));
data.add(new Tuple3<>(9, 4L, "Comment#3"));
data.add(new Tuple3<>(10, 4L, "Comment#4"));
data.add(new Tuple3<>(11, 5L, "Comment#5"));
data.add(new Tuple3<>(12, 5L, "Comment#6"));
data.add(new Tuple3<>(13, 5L, "Comment#7"));
data.add(new Tuple3<>(14, 5L, "Comment#8"));
data.add(new Tuple3<>(15, 5L, "Comment#9"));
data.add(new Tuple3<>(16, 6L, "Comment#10"));
data.add(new Tuple3<>(17, 6L, "Comment#11"));
data.add(new Tuple3<>(18, 6L, "Comment#12"));
data.add(new Tuple3<>(19, 6L, "Comment#13"));
data.add(new Tuple3<>(20, 6L, "Comment#14"));
data.add(new Tuple3<>(21, 6L, "Comment#15"));
Collections.shuffle(data);
return env.fromCollection(data);
}
18
View Complete Implementation : ValueCollectionDataSets.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
public static DataSet<PojoContainingTupleAndWritable> getPojoContainingTupleAndWritable(ExecutionEnvironment env) {
List<PojoContainingTupleAndWritable> data = new ArrayList<>();
// 1x
data.add(new PojoContainingTupleAndWritable(1, 10L, 100L));
// 5x
data.add(new PojoContainingTupleAndWritable(2, 20L, 200L));
data.add(new PojoContainingTupleAndWritable(2, 20L, 200L));
data.add(new PojoContainingTupleAndWritable(2, 20L, 200L));
data.add(new PojoContainingTupleAndWritable(2, 20L, 200L));
data.add(new PojoContainingTupleAndWritable(2, 20L, 200L));
return env.fromCollection(data);
}
18
View Complete Implementation : CustomInputSplitProgram.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
public static void main(String[] args) throws Exception {
ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
DataSet<Integer> data = env.createInput(new CustomInputFormat());
data.map(new MapFunction<Integer, Tuple2<Integer, Double>>() {
@Override
public Tuple2<Integer, Double> map(Integer value) {
return new Tuple2<Integer, Double>(value, value * 0.5);
}
}).output(new DiscardingOutputFormat<Tuple2<Integer, Double>>());
env.execute();
}
18
View Complete Implementation : CustomSerializationITCase.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
@Test
public void testIncorrectSerializer4() {
try {
ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
env.setParallelism(PARLLELISM);
env.generateSequence(1, 10 * PARLLELISM).map(new MapFunction<Long, ConsumesTooLittleSpanning>() {
@Override
public ConsumesTooLittleSpanning map(Long value) throws Exception {
return new ConsumesTooLittleSpanning();
}
}).rebalance().output(new DiscardingOutputFormat<ConsumesTooLittleSpanning>());
env.execute();
} catch (ProgramInvocationException e) {
Throwable rootCause = e.getCause().getCause();
replacedertTrue(rootCause instanceof IOException);
replacedertTrue(rootCause.getMessage().contains("broken serialization"));
} catch (Exception e) {
e.printStackTrace();
fail(e.getMessage());
}
}
18
View Complete Implementation : TestGraphUtils.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
public static DataSet<Tuple2<Long, Long>> getLongLongTuple2SourceData(ExecutionEnvironment env) {
List<Tuple2<Long, Long>> tuples = new ArrayList<>();
tuples.add(new Tuple2<>(1L, 10L));
tuples.add(new Tuple2<>(1L, 20L));
tuples.add(new Tuple2<>(2L, 30L));
tuples.add(new Tuple2<>(3L, 40L));
tuples.add(new Tuple2<>(3L, 50L));
tuples.add(new Tuple2<>(4L, 60L));
tuples.add(new Tuple2<>(6L, 70L));
return env.fromCollection(tuples);
}
18
View Complete Implementation : LabelPropagationData.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
public static final DataSet<Edge<Long, NullValue>> getDefaultEdgeDataSet(ExecutionEnvironment env) {
List<Edge<Long, NullValue>> edges = new ArrayList<>();
edges.add(new Edge<>(1L, 3L, NullValue.getInstance()));
edges.add(new Edge<>(2L, 3L, NullValue.getInstance()));
edges.add(new Edge<>(4L, 7L, NullValue.getInstance()));
edges.add(new Edge<>(5L, 7L, NullValue.getInstance()));
edges.add(new Edge<>(6L, 7L, NullValue.getInstance()));
edges.add(new Edge<>(7L, 3L, NullValue.getInstance()));
return env.fromCollection(edges);
}
18
View Complete Implementation : GroupCombineOperatorTest.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
@Test
public void testSemanticPropsWithKeySelector7() {
final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
DataSet<Tuple5<Integer, Long, String, Long, Integer>> tupleDs = env.fromCollection(emptyTupleData, tupleTypeInfo);
GroupCombineOperator<Tuple5<Integer, Long, String, Long, Integer>, Tuple5<Integer, Long, String, Long, Integer>> combineOp = tupleDs.groupBy(new DummyTestKeySelector()).combineGroup(new DummyGroupCombineFunction4());
SemanticProperties semProps = combineOp.getSemanticProperties();
replacedertTrue(semProps.getForwardingTargetFields(0, 0).size() == 0);
replacedertTrue(semProps.getForwardingTargetFields(0, 1).size() == 0);
replacedertTrue(semProps.getForwardingTargetFields(0, 2).size() == 1);
replacedertTrue(semProps.getForwardingTargetFields(0, 2).contains(0));
replacedertTrue(semProps.getForwardingTargetFields(0, 3).size() == 1);
replacedertTrue(semProps.getForwardingTargetFields(0, 3).contains(1));
replacedertTrue(semProps.getForwardingTargetFields(0, 4).size() == 0);
replacedertTrue(semProps.getForwardingTargetFields(0, 5).size() == 1);
replacedertTrue(semProps.getForwardingTargetFields(0, 5).contains(3));
replacedertTrue(semProps.getForwardingTargetFields(0, 6).size() == 0);
replacedertTrue(semProps.getForwardingSourceField(0, 0) == 2);
replacedertTrue(semProps.getForwardingSourceField(0, 1) == 3);
replacedertTrue(semProps.getForwardingSourceField(0, 2) < 0);
replacedertTrue(semProps.getForwardingSourceField(0, 3) == 5);
replacedertTrue(semProps.getForwardingSourceField(0, 4) < 0);
replacedertTrue(semProps.getReadFields(0) == null);
}
18
View Complete Implementation : SummarizationData.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
/**
* Creates a set of edges with attached {@link String} values.
*
* @param env execution environment
* @return edge data set with string values
*/
public static DataSet<Edge<Long, String>> getEdges(ExecutionEnvironment env) {
List<Edge<Long, String>> edges = new ArrayList<>(INPUT_EDGES.length);
for (String edge : INPUT_EDGES) {
String[] tokens = edge.split(";");
edges.add(new Edge<>(Long.parseLong(tokens[0]), Long.parseLong(tokens[1]), tokens[2]));
}
return env.fromCollection(edges);
}
18
View Complete Implementation : TestGraphUtils.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
public static DataSet<Tuple3<Long, Long, Long>> getLongLongLongTuple3Data(ExecutionEnvironment env) {
List<Tuple3<Long, Long, Long>> tuples = new ArrayList<>();
tuples.add(new Tuple3<>(1L, 2L, 12L));
tuples.add(new Tuple3<>(1L, 3L, 13L));
tuples.add(new Tuple3<>(2L, 3L, 23L));
tuples.add(new Tuple3<>(3L, 4L, 34L));
tuples.add(new Tuple3<>(3L, 6L, 36L));
tuples.add(new Tuple3<>(4L, 6L, 46L));
tuples.add(new Tuple3<>(6L, 1L, 61L));
return env.fromCollection(tuples);
}
18
View Complete Implementation : WordCountWithInnerClass.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
public static void main(String[] args) throws Exception {
// set up the execution environment
final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
// get input data
DataSet<String> text = StaticData.getDefaultTextLineDataSet(env);
DataSet<Tuple2<String, Integer>> counts = // split up the lines in pairs (2-tuples) containing: (word,1)
text.flatMap(new Tokenizer()).groupBy(0).sum(1);
// emit result
counts.print();
// execute program
env.execute("WordCount Example");
}
18
View Complete Implementation : CommunityDetectionData.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
public static DataSet<Edge<Long, Double>> getSimpleEdgeDataSet(ExecutionEnvironment env) {
List<Edge<Long, Double>> edges = new ArrayList<>();
edges.add(new Edge<>(1L, 2L, 1.0));
edges.add(new Edge<>(1L, 3L, 2.0));
edges.add(new Edge<>(1L, 4L, 3.0));
edges.add(new Edge<>(1L, 5L, 4.0));
edges.add(new Edge<>(2L, 6L, 5.0));
edges.add(new Edge<>(6L, 7L, 6.0));
edges.add(new Edge<>(6L, 8L, 7.0));
edges.add(new Edge<>(7L, 8L, 8.0));
return env.fromCollection(edges);
}
18
View Complete Implementation : GroupReduceOperatorTest.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
@Test
public void testSemanticPropsWithKeySelector3() {
final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
DataSet<Tuple5<Integer, Long, String, Long, Integer>> tupleDs = env.fromCollection(emptyTupleData, tupleTypeInfo);
GroupReduceOperator<Tuple5<Integer, Long, String, Long, Integer>, Tuple5<Integer, Long, String, Long, Integer>> reduceOp = tupleDs.groupBy(new DummyTestKeySelector()).reduceGroup(new DummyGroupReduceFunction2()).withForwardedFields("0->4;1;1->3;2");
SemanticProperties semProps = reduceOp.getSemanticProperties();
replacedertTrue(semProps.getForwardingTargetFields(0, 0).size() == 0);
replacedertTrue(semProps.getForwardingTargetFields(0, 1).size() == 0);
replacedertTrue(semProps.getForwardingTargetFields(0, 2).size() == 1);
replacedertTrue(semProps.getForwardingTargetFields(0, 2).contains(4));
replacedertTrue(semProps.getForwardingTargetFields(0, 3).size() == 2);
replacedertTrue(semProps.getForwardingTargetFields(0, 3).contains(1));
replacedertTrue(semProps.getForwardingTargetFields(0, 3).contains(3));
replacedertTrue(semProps.getForwardingTargetFields(0, 4).size() == 1);
replacedertTrue(semProps.getForwardingTargetFields(0, 4).contains(2));
replacedertTrue(semProps.getForwardingTargetFields(0, 5).size() == 0);
replacedertTrue(semProps.getForwardingTargetFields(0, 6).size() == 0);
replacedertTrue(semProps.getForwardingSourceField(0, 0) < 0);
replacedertTrue(semProps.getForwardingSourceField(0, 1) == 3);
replacedertTrue(semProps.getForwardingSourceField(0, 2) == 4);
replacedertTrue(semProps.getForwardingSourceField(0, 3) == 3);
replacedertTrue(semProps.getForwardingSourceField(0, 4) == 2);
replacedertTrue(semProps.getReadFields(0).size() == 3);
replacedertTrue(semProps.getReadFields(0).contains(2));
replacedertTrue(semProps.getReadFields(0).contains(5));
replacedertTrue(semProps.getReadFields(0).contains(6));
}
18
View Complete Implementation : TestGraphUtils.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
public static DataSet<Edge<Long, Long>> getLongLongEdgeInvalidSrcTrgData(ExecutionEnvironment env) {
List<Edge<Long, Long>> edges = getLongLongEdges();
edges.remove(0);
edges.remove(1);
edges.remove(2);
edges.add(new Edge<>(13L, 3L, 13L));
edges.add(new Edge<>(1L, 12L, 12L));
edges.add(new Edge<>(13L, 33L, 13L));
return env.fromCollection(edges);
}
18
View Complete Implementation : WordCountSubclassInterfacePOJOITCase.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
@Override
protected void testProgram() throws Exception {
final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
DataSet<String> text = env.readTextFile(textPath);
DataSet<WCBase> counts = text.flatMap(new Tokenizer()).groupBy("word").reduce(new ReduceFunction<WCBase>() {
private static final long serialVersionUID = 1L;
public WCBase reduce(WCBase value1, WCBase value2) {
WC wc1 = (WC) value1;
WC wc2 = (WC) value2;
int c = wc1.secretCount.getCount() + wc2.secretCount.getCount();
wc1.secretCount.setCount(c);
return wc1;
}
}).map(new MapFunction<WCBase, WCBase>() {
@Override
public WCBase map(WCBase value) throws Exception {
WC wc = (WC) value;
wc.count = wc.secretCount.getCount();
return wc;
}
});
counts.writeAsText(resultPath);
env.execute("WordCount with custom data types example");
}
18
View Complete Implementation : IncrementalSSSPData.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
public static final DataSet<Edge<Long, Double>> getDefaultEdgeDataSet(ExecutionEnvironment env) {
List<Edge<Long, Double>> edges = new ArrayList<>();
edges.add(new Edge<>(1L, 3L, 3.0));
edges.add(new Edge<>(2L, 4L, 3.0));
edges.add(new Edge<>(2L, 5L, 2.0));
edges.add(new Edge<>(3L, 2L, 1.0));
edges.add(new Edge<>(3L, 5L, 5.0));
edges.add(new Edge<>(4L, 5L, 1.0));
return env.fromCollection(edges);
}
18
View Complete Implementation : DeltaIterationNotDependingOnSolutionSetITCase.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
@Override
protected void testProgram() throws Exception {
try {
ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
env.setParallelism(1);
DataSet<Tuple2<Long, Long>> input = env.generateSequence(0, 9).map(new Duplicator<Long>());
DeltaIteration<Tuple2<Long, Long>, Tuple2<Long, Long>> iteration = input.iterateDelta(input, 5, 1);
iteration.closeWith(iteration.getWorkset(), iteration.getWorkset().map(new TestMapper())).output(new LocalCollectionOutputFormat<Tuple2<Long, Long>>(result));
env.execute();
} catch (Exception e) {
e.printStackTrace();
fail(e.getMessage());
}
}
18
View Complete Implementation : MiscellaneousIssuesITCase.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
@Test
public void testAcreplacedulatorsAfterNoOp() {
final String accName = "test_acreplacedulator";
try {
ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
env.setParallelism(6);
env.generateSequence(1, 1000000).rebalance().flatMap(new RichFlatMapFunction<Long, Long>() {
private LongCounter counter;
@Override
public void open(Configuration parameters) {
counter = getRuntimeContext().getLongCounter(accName);
}
@Override
public void flatMap(Long value, Collector<Long> out) {
counter.add(1L);
}
}).output(new DiscardingOutputFormat<Long>());
JobExecutionResult result = env.execute();
replacedertEquals(1000000L, result.getAllAcreplacedulatorResults().get(accName));
} catch (Exception e) {
e.printStackTrace();
fail(e.getMessage());
}
}