org.apache.tajo.ExecutionBlockId - java examples

Here are the examples of the java api org.apache.tajo.ExecutionBlockId taken from open source projects. By voting up you can indicate which examples are most useful and appropriate.

102 Examples 7

19 View Complete Implementation : FetchImpl.java
Copyright Apache License 2.0
Author : apache
public void setExecutionBlockId(ExecutionBlockId executionBlockId) {
    this.executionBlockId = executionBlockId;
}

19 View Complete Implementation : TajoResourceAllocator.java
Copyright Apache License 2.0
Author : apache
private void launchTaskRunners(ExecutionBlockId executionBlockId, Collection<Container> containers) {
    // Query in standby mode doesn't need launch Worker.
    // But, replacedign ExecutionBlock to replacedigned tajo worker
    for (Container eachContainer : containers) {
        TajoContainerProxy containerProxy = new TajoContainerProxy(queryTaskContext, tajoConf, eachContainer, executionBlockId);
        executorService.submit(new LaunchRunner(eachContainer.getId(), containerProxy));
    }
}

19 View Complete Implementation : MasterPlan.java
Copyright Apache License 2.0
Author : apache
public boolean isLeaf(ExecutionBlockId id) {
    return execBlockGraph.isLeaf(id);
}

19 View Complete Implementation : TaskRequestEvent.java
Copyright Apache License 2.0
Author : apache
public clreplaced TaskRequestEvent extends AbstractEvent<TaskRequestEventType> {

    public enum TaskRequestEventType {

        TASK_REQ
    }

    private final AllocationResourceProto responseProto;

    private final ExecutionBlockId executionBlockId;

    private final int workerId;

    public TaskRequestEvent(int workerId, AllocationResourceProto responseProto, ExecutionBlockId executionBlockId) {
        super(TaskRequestEventType.TASK_REQ);
        this.workerId = workerId;
        this.responseProto = responseProto;
        this.executionBlockId = executionBlockId;
    }

    public ExecutionBlockId getExecutionBlockId() {
        return executionBlockId;
    }

    public AllocationResourceProto getResponseProto() {
        return responseProto;
    }

    public int getWorkerId() {
        return workerId;
    }
}

19 View Complete Implementation : MasterPlan.java
Copyright Apache License 2.0
Author : apache
public void removeExecBlock(ExecutionBlockId execBlockId) throws IllegalStateException {
    List<DataChannel> channels = getIncomingChannels(execBlockId);
    if (channels != null && channels.size() > 0) {
        throw new IllegalStateException("Cannot remove execution blocks because some other execution blocks are connected");
    }
    channels = getOutgoingChannels(execBlockId);
    if (channels != null && channels.size() > 0) {
        throw new IllegalStateException("Cannot remove execution blocks because some other execution blocks are connected");
    }
    execBlockMap.remove(execBlockId);
}

19 View Complete Implementation : Query.java
Copyright Apache License 2.0
Author : apache
public SubQuery getSubQuery(ExecutionBlockId id) {
    return this.subqueries.get(id);
}

19 View Complete Implementation : TestNodeResourceManager.java
Copyright Apache License 2.0
Author : apache
@Test(timeout = 30000)
public void testParallelRequest() throws Exception {
    final int parallelCount = conf.getIntVar(TajoConf.ConfVars.WORKER_RESOURCE_AVAILABLE_CPU_CORES) * 2;
    final int taskSize = 100000;
    resourceManager.setTaskHandlerEvent(true);
    final AtomicInteger totalComplete = new AtomicInteger();
    final AtomicInteger totalCanceled = new AtomicInteger();
    final ExecutionBlockId ebId = new ExecutionBlockId(LocalTajoTestingUtility.newQueryId(), 0);
    final Queue<TaskAllocationProto> totalTasks = MockNodeResourceManager.createTaskRequests(ebId, taskMemory, taskSize);
    TaskAllocationProto task = totalTasks.poll();
    BatchAllocationRequest.Builder requestProto = BatchAllocationRequest.newBuilder();
    requestProto.addTaskRequest(task);
    requestProto.setExecutionBlockId(ebId.getProto());
    CallFuture<BatchAllocationResponse> callFuture = new CallFuture<>();
    dispatcher.getEventHandler().handle(new NodeResourceAllocateEvent(requestProto.build(), callFuture));
    replacedertTrue(callFuture.get().getCancellationTaskCount() == 0);
    totalComplete.incrementAndGet();
    // start parallel request
    ExecutorService executor = Executors.newFixedThreadPool(parallelCount);
    List<Future> futureList = Lists.newArrayList();
    for (int i = 0; i < parallelCount; i++) {
        futureList.add(executor.submit(new Runnable() {

            @Override
            public void run() {
                int complete = 0;
                while (true) {
                    TaskAllocationProto task = totalTasks.poll();
                    if (task == null)
                        break;
                    BatchAllocationRequest.Builder requestProto = BatchAllocationRequest.newBuilder();
                    requestProto.addTaskRequest(task);
                    requestProto.setExecutionBlockId(ebId.getProto());
                    CallFuture<BatchAllocationResponse> callFuture = new CallFuture<>();
                    dispatcher.getEventHandler().handle(new NodeResourceAllocateEvent(requestProto.build(), callFuture));
                    try {
                        BatchAllocationResponse proto = callFuture.get();
                        if (proto.getCancellationTaskCount() > 0) {
                            totalTasks.addAll(proto.getCancellationTaskList());
                            totalCanceled.addAndGet(proto.getCancellationTaskCount());
                        } else {
                            complete++;
                        }
                    } catch (Exception e) {
                        fail(e.getMessage());
                    }
                }
                totalComplete.addAndGet(complete);
            }
        }));
    }
    for (Future future : futureList) {
        future.get();
    }
    executor.shutdown();
    replacedertEquals(taskSize, totalComplete.get());
}

19 View Complete Implementation : MasterPlan.java
Copyright Apache License 2.0
Author : apache
public void disconnect(ExecutionBlockId src, ExecutionBlockId target) {
    execBlockGraph.removeEdge(src, target);
}

19 View Complete Implementation : ExecutionBlock.java
Copyright Apache License 2.0
Author : apache
/**
 * A distributed execution plan (DEP) is a direct acyclic graph (DAG) of ExecutionBlocks.
 * An ExecutionBlock is a basic execution unit that could be distributed across a number of nodes.
 * An ExecutionBlock clreplaced contains input information (e.g., child execution blocks or input
 * tables), and output information (e.g., parreplacedion type, parreplacedion key, and parreplacedion number).
 * In addition, it includes a logical plan to be executed in each node.
 */
public clreplaced ExecutionBlock {

    private ExecutionBlockId executionBlockId;

    private LogicalNode plan = null;

    private Enforcer enforcer = new Enforcer();

    // Actual ScanNode's ExecutionBlockId -> Delegated ScanNode's ExecutionBlockId.
    private Map<ExecutionBlockId, ExecutionBlockId> unionScanMap = new HashMap<>();

    // map of table name and corresponding scan node
    private Map<String, ScanNode> broadcastRelations = new HashMap<>();

    private PlanContext planContext;

    /*
   * An execution block is null-supplying or preserved-row when its output is used as an input for outer join.
   * These properties are decided based on the type of parent execution block's outer join.
   * Here are brief descriptions for these properties.
   *
   * 1) left outer join
   *
   *              parent eb
   *         -------------------
   *         | left outer join |
   *         -------------------
   *           /              \
   *   left child eb     right child eb
   * ----------------- ------------------
   * | preserved-row | | null-supplying |
   * ----------------- ------------------
   *
   * 2) right outer join
   *
   *               parent eb
   *         --------------------
   *         | right outer join |
   *         --------------------
   *           /              \
   *   left child eb     right child eb
   * ------------------ -----------------
   * | null-supplying | | preserved-row |
   * ------------------ -----------------
   *
   * 3) full outer join
   *
   *               parent eb
   *         -------------------
   *         | full outer join |
   *         -------------------
   *           /              \
   *   left child eb      right child eb
   * ------------------ ------------------
   * | null-supplying | | preserved-row  |
   * | preserved-row  | | null-supplying |
   * ------------------ ------------------
   *
   * The null-supplying and preserved-row properties are used to find which relations will be broadcasted.
   */
    protected boolean nullSuppllying = false;

    protected boolean preservedRow = false;

    public ExecutionBlock(ExecutionBlockId executionBlockId) {
        this.executionBlockId = executionBlockId;
    }

    public ExecutionBlockId getId() {
        return executionBlockId;
    }

    public void setPlan(LogicalNode plan) throws TajoException {
        this.plan = plan;
        if (plan == null) {
            return;
        }
        final PlanVisitor visitor = new PlanVisitor();
        planContext = new PlanContext();
        visitor.visit(planContext, null, null, plan, new Stack<>());
    }

    public void addUnionScan(ExecutionBlockId realScanEbId, ExecutionBlockId delegatedScanEbId) {
        unionScanMap.put(realScanEbId, delegatedScanEbId);
    }

    public Map<ExecutionBlockId, ExecutionBlockId> getUnionScanMap() {
        return unionScanMap;
    }

    public LogicalNode getPlan() {
        return plan;
    }

    public Enforcer getEnforcer() {
        return enforcer;
    }

    public StoreTableNode getStoreTableNode() {
        return planContext.store;
    }

    public int getNonBroadcastRelNum() {
        int nonBroadcastRelNum = 0;
        for (ScanNode scanNode : planContext.scanlist) {
            if (!broadcastRelations.containsKey(scanNode.getCanonicalName())) {
                nonBroadcastRelNum++;
            }
        }
        return nonBroadcastRelNum;
    }

    public ScanNode[] getScanNodes() {
        return planContext.scanlist.toArray(new ScanNode[planContext.scanlist.size()]);
    }

    public boolean hasJoin() {
        return planContext.hasJoinPlan;
    }

    public boolean hasUnion() {
        return planContext.hasUnionPlan;
    }

    public boolean hasAgg() {
        return planContext.hasAggPlan;
    }

    public boolean isUnionOnly() {
        return planContext.isUnionOnly();
    }

    public void addBroadcastRelation(ScanNode relationNode) {
        if (!broadcastRelations.containsKey(relationNode.getCanonicalName())) {
            enforcer.addBroadcast(relationNode.getCanonicalName());
        }
        broadcastRelations.put(relationNode.getCanonicalName(), relationNode);
    }

    public void removeBroadcastRelation(ScanNode relationNode) {
        broadcastRelations.remove(relationNode.getCanonicalName());
        enforcer.removeBroadcast(relationNode.getCanonicalName());
    }

    public boolean isBroadcastRelation(ScanNode relationNode) {
        return broadcastRelations.containsKey(relationNode.getCanonicalName());
    }

    public boolean hasBroadcastRelation() {
        return broadcastRelations.size() > 0;
    }

    public Collection<ScanNode> getBroadcastRelations() {
        return broadcastRelations.values();
    }

    public String toString() {
        return executionBlockId.toString();
    }

    public void setNullSuppllying() {
        nullSuppllying = true;
    }

    public void setPreservedRow() {
        preservedRow = true;
    }

    public boolean isNullSuppllying() {
        return nullSuppllying;
    }

    public boolean isPreservedRow() {
        return preservedRow;
    }

    private clreplaced PlanContext {

        StoreTableNode store = null;

        List<ScanNode> scanlist = new ArrayList<>();

        boolean hasJoinPlan = false;

        boolean hasUnionPlan = false;

        boolean hasAggPlan = false;

        boolean hreplacedortPlan = false;

        boolean isUnionOnly() {
            return hasUnionPlan && !hasJoinPlan && !hasAggPlan && !hreplacedortPlan;
        }
    }

    private clreplaced PlanVisitor extends BasicLogicalPlanVisitor<PlanContext, LogicalNode> {

        @Override
        public LogicalNode visitJoin(PlanContext context, LogicalPlan plan, LogicalPlan.QueryBlock block, JoinNode node, Stack<LogicalNode> stack) throws TajoException {
            context.hasJoinPlan = true;
            return super.visitJoin(context, plan, block, node, stack);
        }

        @Override
        public LogicalNode visitGroupBy(PlanContext context, LogicalPlan plan, LogicalPlan.QueryBlock block, GroupbyNode node, Stack<LogicalNode> stack) throws TajoException {
            context.hasAggPlan = true;
            return super.visitGroupBy(context, plan, block, node, stack);
        }

        @Override
        public LogicalNode visitWindowAgg(PlanContext context, LogicalPlan plan, LogicalPlan.QueryBlock block, WindowAggNode node, Stack<LogicalNode> stack) throws TajoException {
            context.hasAggPlan = true;
            return super.visitWindowAgg(context, plan, block, node, stack);
        }

        @Override
        public LogicalNode visitDistinctGroupby(PlanContext context, LogicalPlan plan, LogicalPlan.QueryBlock block, DistinctGroupbyNode node, Stack<LogicalNode> stack) throws TajoException {
            context.hasAggPlan = true;
            return super.visitDistinctGroupby(context, plan, block, node, stack);
        }

        @Override
        public LogicalNode visitSort(PlanContext context, LogicalPlan plan, LogicalPlan.QueryBlock block, SortNode node, Stack<LogicalNode> stack) throws TajoException {
            context.hreplacedortPlan = true;
            return super.visitSort(context, plan, block, node, stack);
        }

        @Override
        public LogicalNode visitUnion(PlanContext context, LogicalPlan plan, LogicalPlan.QueryBlock block, UnionNode node, Stack<LogicalNode> stack) throws TajoException {
            context.hasUnionPlan = true;
            return super.visitUnion(context, plan, block, node, stack);
        }

        @Override
        public LogicalNode visitStoreTable(PlanContext context, LogicalPlan plan, LogicalPlan.QueryBlock block, StoreTableNode node, Stack<LogicalNode> stack) throws TajoException {
            context.store = node;
            return super.visitStoreTable(context, plan, block, node, stack);
        }

        @Override
        public LogicalNode visitScan(PlanContext context, LogicalPlan plan, LogicalPlan.QueryBlock block, ScanNode node, Stack<LogicalNode> stack) throws TajoException {
            context.scanlist.add(node);
            return super.visitScan(context, plan, block, node, stack);
        }

        @Override
        public LogicalNode visitParreplacedionedTableScan(PlanContext context, LogicalPlan plan, LogicalPlan.QueryBlock block, ParreplacedionedTableScanNode node, Stack<LogicalNode> stack) throws TajoException {
            context.scanlist.add(node);
            return super.visitParreplacedionedTableScan(context, plan, block, node, stack);
        }

        @Override
        public LogicalNode visitIndexScan(PlanContext context, LogicalPlan plan, LogicalPlan.QueryBlock block, IndexScanNode node, Stack<LogicalNode> stack) throws TajoException {
            context.scanlist.add(node);
            return super.visitIndexScan(context, plan, block, node, stack);
        }
    }
}

19 View Complete Implementation : MasterPlan.java
Copyright Apache License 2.0
Author : apache
public boolean isReverseConnected(ExecutionBlockId target, ExecutionBlockId src) {
    return execBlockGraph.hasReversedEdge(target, src);
}

19 View Complete Implementation : MasterPlan.java
Copyright Apache License 2.0
Author : apache
public List<DataChannel> getIncomingChannels(ExecutionBlockId target) {
    return execBlockGraph.getIncomingEdges(target);
}

19 View Complete Implementation : MasterPlan.java
Copyright Apache License 2.0
Author : apache
public boolean containsExecBlock(ExecutionBlockId execBlockId) {
    return execBlockMap.containsKey(execBlockId);
}

19 View Complete Implementation : TaskRequestEvent.java
Copyright Apache License 2.0
Author : apache
public clreplaced TaskRequestEvent extends AbstractEvent<TaskRequestEventType> {

    public enum TaskRequestEventType {

        TASK_REQ
    }

    private final ContainerId workerId;

    private final ExecutionBlockId executionBlockId;

    private final RpcCallback<QueryUnitRequestProto> callback;

    public TaskRequestEvent(ContainerId workerId, ExecutionBlockId executionBlockId, RpcCallback<QueryUnitRequestProto> callback) {
        super(TaskRequestEventType.TASK_REQ);
        this.workerId = workerId;
        this.executionBlockId = executionBlockId;
        this.callback = callback;
    }

    public ContainerId getContainerId() {
        return this.workerId;
    }

    public ExecutionBlockId getExecutionBlockId() {
        return executionBlockId;
    }

    public RpcCallback<QueryUnitRequestProto> getCallback() {
        return this.callback;
    }
}

19 View Complete Implementation : ExecutionBlockErrorEvent.java
Copyright Apache License 2.0
Author : apache
public clreplaced ExecutionBlockErrorEvent extends TaskManagerEvent {

    private ExecutionBlockId executionBlockId;

    private Throwable error;

    public ExecutionBlockErrorEvent(ExecutionBlockId executionBlockId, Throwable e) {
        super(EventType.EB_FAIL);
        this.executionBlockId = executionBlockId;
        this.error = e;
    }

    public ExecutionBlockId getExecutionBlockId() {
        return executionBlockId;
    }

    public Throwable getError() {
        return error;
    }
}

19 View Complete Implementation : MasterPlan.java
Copyright Apache License 2.0
Author : apache
public int getChildCount(ExecutionBlockId blockId) {
    return execBlockGraph.getChildCount(blockId);
}

19 View Complete Implementation : TestNodeResourceManager.java
Copyright Apache License 2.0
Author : apache
@Test
public void testNodeResourceCancellation() throws Exception {
    int requestSize = conf.getIntVar(TajoConf.ConfVars.WORKER_RESOURCE_AVAILABLE_CPU_CORES);
    int overSize = 10;
    // skip task execution
    resourceManager.setTaskHandlerEvent(false);
    CallFuture<BatchAllocationResponse> callFuture = new CallFuture<>();
    BatchAllocationRequest.Builder requestProto = BatchAllocationRequest.newBuilder();
    ExecutionBlockId ebId = new ExecutionBlockId(LocalTajoTestingUtility.newQueryId(), 0);
    requestProto.setExecutionBlockId(ebId.getProto());
    replacedertEquals(resourceManager.getTotalResource(), resourceManager.getAvailableResource());
    requestProto.addAllTaskRequest(MockNodeResourceManager.createTaskRequests(ebId, taskMemory, requestSize + overSize));
    dispatcher.getEventHandler().handle(new NodeResourceAllocateEvent(requestProto.build(), callFuture));
    BatchAllocationResponse responseProto = callFuture.get();
    replacedertEquals(overSize, responseProto.getCancellationTaskCount());
}

19 View Complete Implementation : TaskSchedulerContext.java
Copyright Apache License 2.0
Author : apache
public clreplaced TaskSchedulerContext {

    private QueryMasterTask.QueryMasterTaskContext masterContext;

    private boolean isLeafQuery;

    private ExecutionBlockId blockId;

    private int taskSize;

    private int estimatedTaskNum;

    public TaskSchedulerContext(QueryMasterTask.QueryMasterTaskContext masterContext, boolean isLeafQuery, ExecutionBlockId blockId) {
        this.masterContext = masterContext;
        this.isLeafQuery = isLeafQuery;
        this.blockId = blockId;
    }

    public QueryMasterTask.QueryMasterTaskContext getMasterContext() {
        return masterContext;
    }

    public boolean isLeafQuery() {
        return isLeafQuery;
    }

    public ExecutionBlockId getBlockId() {
        return blockId;
    }

    public int getTaskSize() {
        return taskSize;
    }

    public int getEstimatedTaskNum() {
        return estimatedTaskNum;
    }

    public void setTaskSize(int taskSize) {
        this.taskSize = taskSize;
    }

    public void setEstimatedTaskNum(int estimatedTaskNum) {
        this.estimatedTaskNum = estimatedTaskNum;
    }
}

19 View Complete Implementation : YarnTaskRunnerLauncherImpl.java
Copyright Apache License 2.0
Author : apache
private void launchTaskRunners(ExecutionBlockId executionBlockId, Collection<Container> containers) {
    commonContainerSpec = YarnContainerProxy.createCommonContainerLaunchContext(getConfig(), executionBlockId.getQueryId().toString(), false);
    for (Container container : containers) {
        final ContainerProxy proxy = new YarnContainerProxy(context, getConfig(), yarnRPC, container, executionBlockId);
        executorService.submit(new LaunchRunner(container.getId(), proxy));
    }
}

19 View Complete Implementation : MasterPlan.java
Copyright Apache License 2.0
Author : apache
public List<DataChannel> getOutgoingChannels(ExecutionBlockId src) {
    return execBlockGraph.getOutgoingEdges(src);
}

19 View Complete Implementation : ExecutionBlock.java
Copyright Apache License 2.0
Author : apache
public void addUnionScan(ExecutionBlockId realScanEbId, ExecutionBlockId delegatedScanEbId) {
    unionScanMap.put(realScanEbId, delegatedScanEbId);
}

19 View Complete Implementation : MasterPlan.java
Copyright Apache License 2.0
Author : apache
public ExecutionBlock getChild(ExecutionBlockId execBlockId, int idx) {
    return execBlockMap.get(execBlockGraph.getChild(execBlockId, idx));
}

19 View Complete Implementation : MasterPlan.java
Copyright Apache License 2.0
Author : apache
public List<ExecutionBlock> getChilds(ExecutionBlockId id) {
    List<ExecutionBlock> childBlocks = new ArrayList<>();
    for (ExecutionBlockId cid : execBlockGraph.getChilds(id)) {
        childBlocks.add(execBlockMap.get(cid));
    }
    return childBlocks;
}

19 View Complete Implementation : DataChannel.java
Copyright Apache License 2.0
Author : apache
public clreplaced DataChannel {

    private ExecutionBlockId srcId;

    private ExecutionBlockId targetId;

    private TransmitType transmitType = TransmitType.PULL_TRANSMIT;

    private ShuffleType shuffleType;

    private Integer numOutputs = 1;

    private Column[] shuffleKeys;

    private Schema schema;

    private StoreType storeType = StoreType.RAW;

    public DataChannel(ExecutionBlockId srcId, ExecutionBlockId targetId) {
        this.srcId = srcId;
        this.targetId = targetId;
    }

    public DataChannel(ExecutionBlockId srcId, ExecutionBlockId targetId, ShuffleType shuffleType) {
        this(srcId, targetId);
        this.shuffleType = shuffleType;
    }

    public DataChannel(ExecutionBlock src, ExecutionBlock target, ShuffleType shuffleType, int numOutput) {
        this(src.getId(), target.getId(), shuffleType, numOutput);
        setSchema(src.getPlan().getOutSchema());
    }

    public DataChannel(ExecutionBlockId srcId, ExecutionBlockId targetId, ShuffleType shuffleType, int numOutputs) {
        this(srcId, targetId, shuffleType);
        this.numOutputs = numOutputs;
    }

    public DataChannel(DataChannelProto proto) {
        this.srcId = new ExecutionBlockId(proto.getSrcId());
        this.targetId = new ExecutionBlockId(proto.getTargetId());
        this.transmitType = proto.getTransmitType();
        this.shuffleType = proto.getShuffleType();
        if (proto.hreplacedchema()) {
            this.setSchema(new Schema(proto.getSchema()));
        }
        if (proto.getShuffleKeysCount() > 0) {
            shuffleKeys = new Column[proto.getShuffleKeysCount()];
            for (int i = 0; i < proto.getShuffleKeysCount(); i++) {
                shuffleKeys[i] = new Column(proto.getShuffleKeys(i));
            }
        } else {
            shuffleKeys = new Column[] {};
        }
        if (proto.hasNumOutputs()) {
            this.numOutputs = proto.getNumOutputs();
        }
        if (proto.hreplacedtoreType()) {
            this.storeType = proto.getStoreType();
        }
    }

    public ExecutionBlockId getSrcId() {
        return srcId;
    }

    public ExecutionBlockId getTargetId() {
        return targetId;
    }

    public ShuffleType getShuffleType() {
        return shuffleType;
    }

    public TransmitType getTransmitType() {
        return this.transmitType;
    }

    public void setTransmitType(TransmitType transmitType) {
        this.transmitType = transmitType;
    }

    public void setShuffle(ShuffleType shuffleType, Column[] keys, int numOutputs) {
        Preconditions.checkArgument(keys.length >= 0, "At least one shuffle key must be specified.");
        Preconditions.checkArgument(numOutputs > 0, "The number of outputs must be positive: %s", numOutputs);
        this.shuffleType = shuffleType;
        this.shuffleKeys = keys;
        this.numOutputs = numOutputs;
    }

    public void setShuffleType(ShuffleType shuffleType) {
        this.shuffleType = shuffleType;
    }

    public boolean hreplacedhuffleKeys() {
        return shuffleKeys != null;
    }

    public void setShuffleKeys(Column[] key) {
        this.shuffleKeys = key;
    }

    public Column[] getShuffleKeys() {
        return this.shuffleKeys;
    }

    public void setShuffleOutputNum(int partNum) {
        this.numOutputs = partNum;
    }

    public int getShuffleOutputNum() {
        return numOutputs;
    }

    public boolean hreplacedtoreType() {
        return this.storeType != null;
    }

    public void setStoreType(StoreType storeType) {
        this.storeType = storeType;
    }

    public StoreType getStoreType() {
        return storeType;
    }

    public DataChannelProto getProto() {
        DataChannelProto.Builder builder = DataChannelProto.newBuilder();
        builder.setSrcId(srcId.getProto());
        builder.setTargetId(targetId.getProto());
        if (transmitType != null) {
            builder.setTransmitType(transmitType);
        }
        builder.setShuffleType(shuffleType);
        if (schema != null) {
            builder.setSchema(schema.getProto());
        }
        if (shuffleKeys != null) {
            for (Column column : shuffleKeys) {
                builder.addShuffleKeys(column.getProto());
            }
        }
        if (numOutputs != null) {
            builder.setNumOutputs(numOutputs);
        }
        if (storeType != null) {
            builder.setStoreType(storeType);
        }
        return builder.build();
    }

    public void setSchema(Schema schema) {
        this.schema = SchemaUtil.clone(schema);
    }

    public Schema getSchema() {
        return schema;
    }

    public String toString() {
        StringBuilder sb = new StringBuilder();
        sb.append("[").append(srcId.getQueryId()).append("] ");
        sb.append(srcId.getId()).append(" => ").append(targetId.getId());
        sb.append(" (type=").append(shuffleType);
        if (hreplacedhuffleKeys()) {
            sb.append(", key=");
            sb.append(TUtil.arrayToString(shuffleKeys));
            sb.append(", num=").append(numOutputs);
        }
        sb.append(")");
        return sb.toString();
    }
}

19 View Complete Implementation : MasterPlan.java
Copyright Apache License 2.0
Author : apache
public Optional<ShuffleContext> getShuffleInfo(ExecutionBlockId ebId) {
    ExecutionBlockId parentId = getParent(getExecBlock(ebId)).getId();
    return shuffleInfo.containsKey(parentId) ? Optional.of(shuffleInfo.get(parentId)) : Optional.empty();
}

19 View Complete Implementation : ContainerProxy.java
Copyright Apache License 2.0
Author : apache
public abstract clreplaced ContainerProxy {

    protected static final Log LOG = LogFactory.getLog(ContainerProxy.clreplaced);

    final public static FsPermission QUERYCONF_FILE_PERMISSION = // rw-r--r--
    FsPermission.createImmutable((short) 0644);

    protected static enum ContainerState {

        PREP, FAILED, RUNNING, DONE, KILLED_BEFORE_LAUNCH
    }

    protected final ExecutionBlockId executionBlockId;

    protected Configuration conf;

    protected QueryMasterTask.QueryMasterTaskContext context;

    protected ContainerState state;

    // store enough information to be able to cleanup the container
    protected Container container;

    protected ContainerId containerID;

    protected String hostName;

    protected int port = -1;

    public abstract void launch(ContainerLaunchContext containerLaunchContext);

    public abstract void stopContainer();

    public ContainerProxy(QueryMasterTask.QueryMasterTaskContext context, Configuration conf, ExecutionBlockId executionBlockId, Container container) {
        this.context = context;
        this.conf = conf;
        this.state = ContainerState.PREP;
        this.container = container;
        this.executionBlockId = executionBlockId;
        this.containerID = container.getId();
    }

    public synchronized boolean isCompletelyDone() {
        return state == ContainerState.DONE || state == ContainerState.FAILED;
    }

    public String getTaskHostName() {
        return this.hostName;
    }

    public int getTaskPort() {
        return this.port;
    }

    public String getId() {
        return executionBlockId.toString();
    }
}

19 View Complete Implementation : MasterPlan.java
Copyright Apache License 2.0
Author : apache
public <CONTEXT> void accept(CONTEXT context, ExecutionBlockId v, DirectedGraphVisitor<CONTEXT, ExecutionBlockId> visitor) throws TajoException {
    execBlockGraph.accept(context, v, visitor);
}

19 View Complete Implementation : MasterPlan.java
Copyright Apache License 2.0
Author : apache
/**
 * @param ebId
 * @param parreplacedionNum
 */
public void addShuffleInfo(ExecutionBlockId ebId, int parreplacedionNum) {
    ExecutionBlockId parentId = getParent(getExecBlock(ebId)).getId();
    shuffleInfo.put(parentId, new ShuffleContext(ebId, parreplacedionNum));
}

19 View Complete Implementation : SubQueryEvent.java
Copyright Apache License 2.0
Author : apache
public clreplaced SubQueryEvent extends AbstractEvent<SubQueryEventType> {

    private final ExecutionBlockId id;

    public SubQueryEvent(ExecutionBlockId id, SubQueryEventType subQueryEventType) {
        super(subQueryEventType);
        this.id = id;
    }

    public ExecutionBlockId getSubQueryId() {
        return id;
    }
}

19 View Complete Implementation : StageEvent.java
Copyright Apache License 2.0
Author : apache
public clreplaced StageEvent extends AbstractEvent<StageEventType> {

    private final ExecutionBlockId id;

    public StageEvent(ExecutionBlockId id, StageEventType stageEventType) {
        super(stageEventType);
        this.id = id;
    }

    public ExecutionBlockId getStageId() {
        return id;
    }
}

19 View Complete Implementation : MasterPlan.java
Copyright Apache License 2.0
Author : apache
public ExecutionBlock getExecBlock(ExecutionBlockId execBlockId) {
    return execBlockMap.get(execBlockId);
}

19 View Complete Implementation : Query.java
Copyright Apache License 2.0
Author : apache
public Stage getStage(ExecutionBlockId id) {
    return this.stages.get(id);
}

19 View Complete Implementation : TaskSchedulerEvent.java
Copyright Apache License 2.0
Author : apache
public abstract clreplaced TaskSchedulerEvent extends AbstractEvent<EventType> {

    public enum EventType {

        T_SCHEDULE, T_SCHEDULE_CANCEL
    }

    protected final ExecutionBlockId executionBlockId;

    public TaskSchedulerEvent(EventType eventType, ExecutionBlockId executionBlockId) {
        super(eventType);
        this.executionBlockId = executionBlockId;
    }

    public ExecutionBlockId getExecutionBlockId() {
        return this.executionBlockId;
    }
}

19 View Complete Implementation : ExecutionBlockSharedResource.java
Copyright Apache License 2.0
Author : apache
public void releaseBroadcastCache(ExecutionBlockId id) {
    TableCache.getInstance().releaseCache(id);
}

19 View Complete Implementation : MasterPlan.java
Copyright Apache License 2.0
Author : apache
public void addConnect(ExecutionBlockId src, ExecutionBlockId target, TajoWorkerProtocol.ShuffleType type) {
    addConnect(new DataChannel(src, target, type));
}

19 View Complete Implementation : TestNodeResourceManager.java
Copyright Apache License 2.0
Author : apache
@Test
public void testNodeResourceAllocateEvent() throws Exception {
    int requestSize = 4;
    // skip task execution
    resourceManager.setTaskHandlerEvent(false);
    CallFuture<BatchAllocationResponse> callFuture = new CallFuture<>();
    BatchAllocationRequest.Builder requestProto = BatchAllocationRequest.newBuilder();
    ExecutionBlockId ebId = new ExecutionBlockId(LocalTajoTestingUtility.newQueryId(), 0);
    requestProto.setExecutionBlockId(ebId.getProto());
    replacedertEquals(resourceManager.getTotalResource(), resourceManager.getAvailableResource());
    requestProto.addAllTaskRequest(MockNodeResourceManager.createTaskRequests(ebId, taskMemory, requestSize));
    dispatcher.getEventHandler().handle(new NodeResourceAllocateEvent(requestProto.build(), callFuture));
    BatchAllocationResponse responseProto = callFuture.get();
    replacedertNotEquals(resourceManager.getTotalResource(), resourceManager.getAvailableResource());
    // allocated all
    replacedertEquals(0, responseProto.getCancellationTaskCount());
}

19 View Complete Implementation : TaskRunnerGroupEvent.java
Copyright Apache License 2.0
Author : apache
public clreplaced TaskRunnerGroupEvent extends AbstractEvent<EventType> {

    public enum EventType {

        CONTAINER_REMOTE_LAUNCH, CONTAINER_REMOTE_CLEANUP
    }

    protected final ExecutionBlockId executionBlockId;

    protected final Collection<Container> containers;

    public TaskRunnerGroupEvent(EventType eventType, ExecutionBlockId executionBlockId, Collection<Container> containers) {
        super(eventType);
        this.executionBlockId = executionBlockId;
        this.containers = containers;
    }

    public Collection<Container> getContainers() {
        return containers;
    }

    public ExecutionBlockId getExecutionBlockId() {
        return executionBlockId;
    }
}

19 View Complete Implementation : TestNodeResourceManager.java
Copyright Apache License 2.0
Author : apache
@Test
public void testNodeResourceDeallocateEvent() throws Exception {
    int requestSize = 4;
    // skip task execution
    resourceManager.setTaskHandlerEvent(false);
    CallFuture<BatchAllocationResponse> callFuture = new CallFuture<>();
    BatchAllocationRequest.Builder requestProto = BatchAllocationRequest.newBuilder();
    ExecutionBlockId ebId = new ExecutionBlockId(LocalTajoTestingUtility.newQueryId(), 0);
    requestProto.setExecutionBlockId(ebId.getProto());
    replacedertEquals(resourceManager.getTotalResource(), resourceManager.getAvailableResource());
    requestProto.addAllTaskRequest(MockNodeResourceManager.createTaskRequests(ebId, taskMemory, requestSize));
    dispatcher.getEventHandler().handle(new NodeResourceAllocateEvent(requestProto.build(), callFuture));
    BatchAllocationResponse responseProto = callFuture.get();
    replacedertNotEquals(resourceManager.getTotalResource(), resourceManager.getAvailableResource());
    replacedertEquals(0, responseProto.getCancellationTaskCount());
    // deallocate
    for (TaskAllocationProto allocationRequestProto : requestProto.getTaskRequestList()) {
        // direct invoke handler for testing
        resourceManager.handle(new NodeResourceDeallocateEvent(allocationRequestProto.getResource(), NodeResourceEvent.ResourceType.TASK));
    }
    replacedertEquals(resourceManager.getTotalResource(), resourceManager.getAvailableResource());
}

19 View Complete Implementation : MasterPlan.java
Copyright Apache License 2.0
Author : apache
public boolean isConnected(ExecutionBlockId src, ExecutionBlockId target) {
    return execBlockGraph.hasEdge(src, target);
}

19 View Complete Implementation : TestNodeResourceManager.java
Copyright Apache License 2.0
Author : apache
@Test
public void testResourceDeallocateWithEbCreateFailure() throws Exception {
    final int taskSize = 10;
    resourceManager.setTaskHandlerEvent(true);
    final ExecutionBlockId ebId = new ExecutionBlockId(LocalTajoTestingUtility.newQueryId(), 0);
    final Queue<TaskAllocationProto> totalTasks = MockNodeResourceManager.createTaskRequests(ebId, taskMemory, taskSize);
    TaskAllocationProto task = totalTasks.poll();
    BatchAllocationRequest.Builder requestProto = BatchAllocationRequest.newBuilder();
    requestProto.addTaskRequest(task);
    requestProto.setExecutionBlockId(ebId.getProto());
    CallFuture<BatchAllocationResponse> callFuture = new CallFuture<>();
    dispatcher.getEventHandler().handle(new NodeResourceAllocateEvent(requestProto.build(), callFuture));
    replacedertTrue(callFuture.get().getCancellationTaskCount() == 0);
    // wait for resource deallocation
    Thread.sleep(2000);
    replacedertEquals(resourceManager.getTotalResource(), resourceManager.getAvailableResource());
}

19 View Complete Implementation : ExecutionBlock.java
Copyright Apache License 2.0
Author : apache
/**
 * A distributed execution plan (DEP) is a direct acyclic graph (DAG) of ExecutionBlocks.
 * An ExecutionBlock is a basic execution unit that could be distributed across a number of nodes.
 * An ExecutionBlock clreplaced contains input information (e.g., child execution blocks or input
 * tables), and output information (e.g., parreplacedion type, parreplacedion key, and parreplacedion number).
 * In addition, it includes a logical plan to be executed in each node.
 */
public clreplaced ExecutionBlock {

    private ExecutionBlockId executionBlockId;

    private LogicalNode plan = null;

    private StoreTableNode store = null;

    private List<ScanNode> scanlist = new ArrayList<ScanNode>();

    private Enforcer enforcer = new Enforcer();

    private boolean hasJoinPlan;

    private boolean hasUnionPlan;

    private Set<String> broadcasted = new HashSet<String>();

    public ExecutionBlock(ExecutionBlockId executionBlockId) {
        this.executionBlockId = executionBlockId;
    }

    public ExecutionBlockId getId() {
        return executionBlockId;
    }

    public void setPlan(LogicalNode plan) {
        hasJoinPlan = false;
        hasUnionPlan = false;
        this.scanlist.clear();
        this.plan = plan;
        if (plan == null) {
            return;
        }
        LogicalNode node = plan;
        ArrayList<LogicalNode> s = new ArrayList<LogicalNode>();
        s.add(node);
        while (!s.isEmpty()) {
            node = s.remove(s.size() - 1);
            if (node instanceof UnaryNode) {
                UnaryNode unary = (UnaryNode) node;
                s.add(s.size(), unary.getChild());
            } else if (node instanceof BinaryNode) {
                BinaryNode binary = (BinaryNode) node;
                if (binary.getType() == NodeType.JOIN) {
                    hasJoinPlan = true;
                } else if (binary.getType() == NodeType.UNION) {
                    hasUnionPlan = true;
                }
                s.add(s.size(), binary.getLeftChild());
                s.add(s.size(), binary.getRightChild());
            } else if (node instanceof ScanNode) {
                scanlist.add((ScanNode) node);
            } else if (node instanceof TableSubQueryNode) {
                TableSubQueryNode subQuery = (TableSubQueryNode) node;
                s.add(s.size(), subQuery.getSubQuery());
            }
        }
    }

    public LogicalNode getPlan() {
        return plan;
    }

    public Enforcer getEnforcer() {
        return enforcer;
    }

    public StoreTableNode getStoreTableNode() {
        return store;
    }

    public ScanNode[] getScanNodes() {
        return this.scanlist.toArray(new ScanNode[scanlist.size()]);
    }

    public boolean hasJoin() {
        return hasJoinPlan;
    }

    public boolean hasUnion() {
        return hasUnionPlan;
    }

    public void addBroadcastTable(String tableName) {
        broadcasted.add(tableName);
        enforcer.addBroadcast(tableName);
    }

    public boolean isBroadcastTable(String tableName) {
        return broadcasted.contains(tableName);
    }

    public Collection<String> getBroadcastTables() {
        return broadcasted;
    }

    public String toString() {
        return executionBlockId.toString();
    }
}

19 View Complete Implementation : MasterPlan.java
Copyright Apache License 2.0
Author : apache
public DataChannel getChannel(ExecutionBlockId src, ExecutionBlockId target) {
    return execBlockGraph.getEdge(src, target);
}

19 View Complete Implementation : MasterPlan.java
Copyright Apache License 2.0
Author : apache
public List<ExecutionBlock> getChilds(ExecutionBlockId id) {
    List<ExecutionBlock> childBlocks = new ArrayList<ExecutionBlock>();
    for (ExecutionBlockId cid : execBlockGraph.getChilds(id)) {
        childBlocks.add(execBlockMap.get(cid));
    }
    return childBlocks;
}

19 View Complete Implementation : QuerySubQueryEvent.java
Copyright Apache License 2.0
Author : apache
public clreplaced QuerySubQueryEvent extends QueryEvent {

    private ExecutionBlockId executionBlockId;

    public QuerySubQueryEvent(final ExecutionBlockId id, final QueryEventType queryEvent) {
        super(id.getQueryId(), queryEvent);
        this.executionBlockId = id;
    }

    public ExecutionBlockId getExecutionBlockId() {
        return this.executionBlockId;
    }
}

19 View Complete Implementation : MasterPlan.java
Copyright Apache License 2.0
Author : apache
public void addConnect(ExecutionBlockId src, ExecutionBlockId target, ShuffleType type) {
    addConnect(new DataChannel(src, target, type));
}

18 View Complete Implementation : ContainerAllocationEvent.java
Copyright Apache License 2.0
Author : apache
public clreplaced ContainerAllocationEvent extends AbstractEvent<ContainerAllocatorEventType> {

    private final ExecutionBlockId executionBlockId;

    private final Priority priority;

    private final Resource resource;

    private final boolean isLeafQuery;

    private final int requiredNum;

    private final float progress;

    public ContainerAllocationEvent(ContainerAllocatorEventType eventType, ExecutionBlockId executionBlockId, Priority priority, Resource resource, int requiredNum, boolean isLeafQuery, float progress) {
        super(eventType);
        this.executionBlockId = executionBlockId;
        this.priority = priority;
        this.resource = resource;
        this.requiredNum = requiredNum;
        this.isLeafQuery = isLeafQuery;
        this.progress = progress;
    }

    public ExecutionBlockId getExecutionBlockId() {
        return executionBlockId;
    }

    public Priority getPriority() {
        return priority;
    }

    public int getRequiredNum() {
        return requiredNum;
    }

    public boolean isLeafQuery() {
        return isLeafQuery;
    }

    public Resource getCapability() {
        return resource;
    }

    public float getProgress() {
        return progress;
    }

    public Resource getResource() {
        return resource;
    }
}

18 View Complete Implementation : QueryCompletedEvent.java
Copyright Apache License 2.0
Author : apache
public clreplaced QueryCompletedEvent extends QueryEvent {

    private final ExecutionBlockId executionBlockId;

    private final SubQueryState finalState;

    public QueryCompletedEvent(final ExecutionBlockId executionBlockId, SubQueryState finalState) {
        super(executionBlockId.getQueryId(), QueryEventType.QUERY_COMPLETED);
        this.executionBlockId = executionBlockId;
        this.finalState = finalState;
    }

    public ExecutionBlockId getExecutionBlockId() {
        return executionBlockId;
    }

    public SubQueryState getState() {
        return finalState;
    }
}

18 View Complete Implementation : SubQueryCompletedEvent.java
Copyright Apache License 2.0
Author : apache
public clreplaced SubQueryCompletedEvent extends QueryEvent {

    private final ExecutionBlockId executionBlockId;

    private final SubQueryState finalState;

    public SubQueryCompletedEvent(final ExecutionBlockId executionBlockId, SubQueryState finalState) {
        super(executionBlockId.getQueryId(), QueryEventType.SUBQUERY_COMPLETED);
        this.executionBlockId = executionBlockId;
        this.finalState = finalState;
    }

    public ExecutionBlockId getExecutionBlockId() {
        return executionBlockId;
    }

    public SubQueryState getState() {
        return finalState;
    }
}

18 View Complete Implementation : Repartitioner.java
Copyright Apache License 2.0
Author : apache
public static String createBasicFetchUri(String hostName, int port, ExecutionBlockId childSid, int taskId, int attemptId) {
    String scheme = "http://";
    StringBuilder sb = new StringBuilder(scheme);
    sb.append(hostName).append(":").append(port).append("/?").append("qid=").append(childSid.getQueryId().toString()).append("&sid=").append(childSid.getId()).append("&").append("ta=").append(taskId).append("_").append(attemptId).append("&").append("p=0").append("&").append("type=r");
    return sb.toString();
}

18 View Complete Implementation : TajoContainerProxy.java
Copyright Apache License 2.0
Author : apache
public static void releaseWorkerResource(QueryMasterTask.QueryMasterTaskContext context, ExecutionBlockId executionBlockId, ContainerId containerId) throws Exception {
    List<ContainerId> containerIds = new ArrayList<ContainerId>();
    containerIds.add(containerId);
    releaseWorkerResource(context, executionBlockId, containerIds);
}

18 View Complete Implementation : TaskRunnerManager.java
Copyright Apache License 2.0
Author : apache
public Task findTaskByQueryUnitAttemptId(QueryUnitAttemptId quAttemptId) {
    ExecutionBlockId ebid = quAttemptId.getQueryUnitId().getExecutionBlockId();
    synchronized (taskRunnerMap) {
        for (TaskRunner eachTaskRunner : taskRunnerMap.values()) {
            if (eachTaskRunner.getExecutionBlockId().equals(ebid)) {
                Task task = eachTaskRunner.getContext().getTask(quAttemptId);
                if (task != null) {
                    return task;
                }
            }
        }
    }
    synchronized (finishedTaskRunnerMap) {
        for (TaskRunner eachTaskRunner : finishedTaskRunnerMap.values()) {
            if (eachTaskRunner.getExecutionBlockId().equals(ebid)) {
                Task task = eachTaskRunner.getContext().getTask(quAttemptId);
                if (task != null) {
                    return task;
                }
            }
        }
    }
    return null;
}