org.apache.hadoop.yarn.api.records.Container - java examples

Here are the examples of the java api org.apache.hadoop.yarn.api.records.Container taken from open source projects. By voting up you can indicate which examples are most useful and appropriate.

155 Examples 7

18 View Complete Implementation : NMClientAsyncImpl.java
Copyright Apache License 2.0
Author : apache
public void startContainerAsync(Container container, ContainerLaunchContext containerLaunchContext) {
    if (containers.putIfAbsent(container.getId(), new StatefulContainer(this, container.getId())) != null) {
        callbackHandler.onStartContainerError(container.getId(), RPCUtil.getRemoteException("Container " + container.getId() + " is already started or scheduled to start"));
    }
    try {
        events.put(new StartContainerEvent(container, containerLaunchContext));
    } catch (InterruptedException e) {
        LOG.warn("Exception when scheduling the event of starting Container " + container.getId());
        callbackHandler.onStartContainerError(container.getId(), e);
    }
}

18 View Complete Implementation : ApplicationMasterServiceUtils.java
Copyright Apache License 2.0
Author : apache
/**
 * Add updated containers to {@link AllocateResponse}.
 * @param allocateResponse Allocate Response.
 * @param updateType Update Type.
 * @param updatedContainers Updated Containers.
 */
public static void addToUpdatedContainers(AllocateResponse allocateResponse, ContainerUpdateType updateType, List<Container> updatedContainers) {
    if (updatedContainers != null && updatedContainers.size() > 0) {
        ArrayList<UpdatedContainer> containersToSet = new ArrayList<>();
        if (allocateResponse.getUpdatedContainers() != null && !allocateResponse.getUpdatedContainers().isEmpty()) {
            containersToSet.addAll(allocateResponse.getUpdatedContainers());
        }
        for (Container updatedContainer : updatedContainers) {
            containersToSet.add(UpdatedContainer.newInstance(updateType, updatedContainer));
        }
        allocateResponse.setUpdatedContainers(containersToSet);
    }
}

18 View Complete Implementation : NMClient.java
Copyright Apache License 2.0
Author : apache
/**
 * <p>Increase the resource of a container.</p>
 *
 * <p>The <code>ApplicationMaster</code> or other applications that use the
 * client must provide the details of the container, including the Id and
 * the target resource encapsulated in the updated container token via
 * {@link Container}.
 * </p>
 *
 * @param container the container with updated token.
 *
 * @throws YarnException YarnException.
 * @throws IOException IOException.
 */
@Deprecated
public abstract void increaseContainerResource(Container container) throws YarnException, IOException;

18 View Complete Implementation : NMClientAsyncImpl.java
Copyright Apache License 2.0
Author : apache
@Override
public void updateContainerResourceAsync(Container container) {
    if (!(callbackHandler instanceof AbstractCallbackHandler)) {
        LOG.error("Callback handler does not implement container resource " + "increase callback methods");
        return;
    }
    AbstractCallbackHandler handler = (AbstractCallbackHandler) callbackHandler;
    if (containers.get(container.getId()) == null) {
        handler.onUpdateContainerResourceError(container.getId(), RPCUtil.getRemoteException("Container " + container.getId() + " is neither started nor scheduled to start"));
    }
    try {
        events.put(new UpdateContainerResourceEvent(container, false));
    } catch (InterruptedException e) {
        LOG.warn("Exception when scheduling the event of increasing resource of " + "Container " + container.getId());
        handler.onUpdateContainerResourceError(container.getId(), e);
    }
}

18 View Complete Implementation : TaskAttemptContainerAssignedEvent.java
Copyright Apache License 2.0
Author : apache
public clreplaced TaskAttemptContainerreplacedignedEvent extends TaskAttemptEvent {

    private final Container container;

    private final Map<ApplicationAccessType, String> applicationACLs;

    public TaskAttemptContainerreplacedignedEvent(TaskAttemptId id, Container container, Map<ApplicationAccessType, String> applicationACLs) {
        super(id, TaskAttemptEventType.TA_replacedIGNED);
        this.container = container;
        this.applicationACLs = applicationACLs;
    }

    public Container getContainer() {
        return this.container;
    }

    public Map<ApplicationAccessType, String> getApplicationACLs() {
        return this.applicationACLs;
    }
}

17 View Complete Implementation : AppSchedulingInfo.java
Copyright Apache License 2.0
Author : apache
public ContainerRequest allocate(NodeType type, SchedulerNode node, SchedulerRequestKey schedulerKey, Container containerAllocated) {
    writeLock.lock();
    try {
        if (null != containerAllocated) {
            updateMetricsForAllocatedContainer(type, node, containerAllocated);
        }
        return schedulerKeyToAppPlacementAllocator.get(schedulerKey).allocate(schedulerKey, type, node);
    } finally {
        writeLock.unlock();
    }
}

17 View Complete Implementation : AMRMClient.java
Copyright Apache License 2.0
Author : apache
/**
 * Request container resource change before calling <code>allocate</code>.
 * Any previous pending resource change request of the same container will be
 * removed.
 *
 * Application that calls this method is expected to maintain the
 * <code>Container</code>s that are returned from previous successful
 * allocations or resource changes. By preplaceding in the existing container and a
 * target resource capability to this method, the application requests the
 * ResourceManager to change the existing resource allocation to the target
 * resource allocation.
 *
 * @deprecated use
 * {@link #requestContainerUpdate(Container, UpdateContainerRequest)}
 *
 * @param container The container returned from the last successful resource
 *                  allocation or resource change
 * @param capability  The target resource capability of the container
 */
@Deprecated
public void requestContainerResourceChange(Container container, Resource capability) {
    Preconditions.checkNotNull(container, "Container cannot be null!!");
    Preconditions.checkNotNull(capability, "UpdateContainerRequest cannot be null!!");
    requestContainerUpdate(container, UpdateContainerRequest.newInstance(container.getVersion(), container.getId(), Resources.fitsIn(capability, container.getResource()) ? ContainerUpdateType.DECREASE_RESOURCE : ContainerUpdateType.INCREASE_RESOURCE, capability, null));
}

17 View Complete Implementation : ContainerRemoteLaunchEvent.java
Copyright Apache License 2.0
Author : apache
public clreplaced ContainerRemoteLaunchEvent extends ContainerLauncherEvent {

    private final Container allocatedContainer;

    private final ContainerLaunchContext containerLaunchContext;

    private final Task task;

    public ContainerRemoteLaunchEvent(TaskAttemptId taskAttemptID, ContainerLaunchContext containerLaunchContext, Container allocatedContainer, Task remoteTask) {
        super(taskAttemptID, allocatedContainer.getId(), StringInterner.weakIntern(allocatedContainer.getNodeId().toString()), allocatedContainer.getContainerToken(), ContainerLauncher.EventType.CONTAINER_REMOTE_LAUNCH);
        this.allocatedContainer = allocatedContainer;
        this.containerLaunchContext = containerLaunchContext;
        this.task = remoteTask;
    }

    public ContainerLaunchContext getContainerLaunchContext() {
        return this.containerLaunchContext;
    }

    public Container getAllocatedContainer() {
        return this.allocatedContainer;
    }

    public Task getRemoteTask() {
        return this.task;
    }

    @Override
    public int hashCode() {
        return super.hashCode();
    }

    @Override
    public boolean equals(Object obj) {
        return super.equals(obj);
    }
}

17 View Complete Implementation : FiCaSchedulerApp.java
Copyright Apache License 2.0
Author : apache
public RMContainer allocate(FiCaSchedulerNode node, SchedulerRequestKey schedulerKey, Container container) {
    readLock.lock();
    try {
        if (isStopped) {
            return null;
        }
        // Required sanity check - AM can call 'allocate' to update resource
        // request without locking the scheduler, hence we need to check
        if (getOutstandingAsksCount(schedulerKey) <= 0) {
            return null;
        }
        AppPlacementAllocator<FiCaSchedulerNode> ps = appSchedulingInfo.getAppPlacementAllocator(schedulerKey);
        if (null == ps) {
            LOG.warn("Failed to get " + AppPlacementAllocator.clreplaced.getName() + " for application=" + getApplicationId() + " schedulerRequestKey=" + schedulerKey);
            return null;
        }
        // Create RMContainer
        RMContainer rmContainer = new RMContainerImpl(container, schedulerKey, this.getApplicationAttemptId(), node.getNodeID(), appSchedulingInfo.getUser(), this.rmContext, ps.getPrimaryRequestedNodeParreplacedion());
        ((RMContainerImpl) rmContainer).setQueueName(this.getQueueName());
        // FIXME, should set when confirmed
        updateAMContainerDiagnostics(AMState.replacedIGNED, null);
        return rmContainer;
    } finally {
        readLock.unlock();
    }
}

17 View Complete Implementation : ContainerPBImpl.java
Copyright Apache License 2.0
Author : apache
// TODO Comparator
@Override
public int compareTo(Container other) {
    if (this.getId().compareTo(other.getId()) == 0) {
        if (this.getNodeId().compareTo(other.getNodeId()) == 0) {
            return this.getResource().compareTo(other.getResource());
        } else {
            return this.getNodeId().compareTo(other.getNodeId());
        }
    } else {
        return this.getId().compareTo(other.getId());
    }
}

17 View Complete Implementation : AMRMClientAsync.java
Copyright Apache License 2.0
Author : apache
/**
 * Request container resource change before calling <code>allocate</code>.
 * Any previous pending resource change request of the same container will be
 * removed.
 *
 * Application that calls this method is expected to maintain the
 * <code>Container</code>s that are returned from previous successful
 * allocations or resource changes. By preplaceding in the existing container and a
 * target resource capability to this method, the application requests the
 * ResourceManager to change the existing resource allocation to the target
 * resource allocation.
 *
 * @deprecated use
 * {@link #requestContainerUpdate(Container, UpdateContainerRequest)}
 *
 * @param container The container returned from the last successful resource
 *                  allocation or resource change
 * @param capability  The target resource capability of the container
 */
@Deprecated
public void requestContainerResourceChange(Container container, Resource capability) {
    Preconditions.checkNotNull(container, "Container cannot be null!!");
    Preconditions.checkNotNull(capability, "UpdateContainerRequest cannot be null!!");
    requestContainerUpdate(container, UpdateContainerRequest.newInstance(container.getVersion(), container.getId(), Resources.fitsIn(capability, container.getResource()) ? ContainerUpdateType.DECREASE_RESOURCE : ContainerUpdateType.INCREASE_RESOURCE, capability, null));
}

16 View Complete Implementation : ContainerLaunchService.java
Copyright Apache License 2.0
Author : apache
public Future<ProviderService.ResolvedLaunchParams> launchCompInstance(Service service, ComponentInstance instance, Container container, ComponentLaunchContext componentLaunchContext) {
    ContainerLauncher launcher = new ContainerLauncher(service, instance, container, componentLaunchContext, false);
    return executorService.submit(launcher);
}

16 View Complete Implementation : DockerProviderService.java
Copyright Apache License 2.0
Author : apache
@Override
public void buildContainerLaunchCommand(AbstractLauncher launcher, Service service, ComponentInstance instance, SliderFileSystem fileSystem, Configuration yarnConf, Container container, ContainerLaunchService.ComponentLaunchContext compLaunchContext, Map<String, String> tokensForSubsreplacedution) throws IOException, SliderException {
    boolean useEntryPoint = checkUseEntryPoint(compLaunchContext);
    if (useEntryPoint) {
        String launchCommand = compLaunchContext.getLaunchCommand();
        if (!StringUtils.isEmpty(launchCommand)) {
            if (launchCommand.contains(" ")) {
                // convert space delimiter command to exec format
                launchCommand = ProviderUtils.replaceSpacesWithDelimiter(launchCommand, ",");
            }
            launcher.addCommand(launchCommand);
        }
    } else {
        // subsreplacedute launch command
        String launchCommand = compLaunchContext.getLaunchCommand();
        // docker container may have empty commands
        if (!StringUtils.isEmpty(launchCommand)) {
            launchCommand = ProviderUtils.subsreplaceduteStrWithTokens(launchCommand, tokensForSubsreplacedution);
            CommandLineBuilder operation = new CommandLineBuilder();
            operation.add(launchCommand);
            operation.addOutAndErrFiles(OUT_FILE, ERR_FILE);
            launcher.addCommand(operation.build());
        }
    }
}

16 View Complete Implementation : MockServiceAM.java
Copyright Apache License 2.0
Author : apache
public Container updateContainerStatus(Service service, int id, String compName, String host) {
    ContainerId containerId = createContainerId(id);
    Container container = createContainer(containerId, compName);
    addContainerStatus(container, ContainerState.RUNNING, host);
    return container;
}

16 View Complete Implementation : TestRMApplicationHistoryWriter.java
Copyright Apache License 2.0
Author : apache
private static RMAppAttempt createRMAppAttempt(ApplicationAttemptId appAttemptId) {
    RMAppAttempt appAttempt = mock(RMAppAttempt.clreplaced);
    when(appAttempt.getAppAttemptId()).thenReturn(appAttemptId);
    when(appAttempt.getHost()).thenReturn("test host");
    when(appAttempt.getRpcPort()).thenReturn(-100);
    Container container = mock(Container.clreplaced);
    when(container.getId()).thenReturn(ContainerId.newContainerId(appAttemptId, 1));
    when(appAttempt.getMasterContainer()).thenReturn(container);
    when(appAttempt.getDiagnostics()).thenReturn("test diagnostics info");
    when(appAttempt.getTrackingUrl()).thenReturn("test url");
    when(appAttempt.getFinalApplicationStatus()).thenReturn(FinalApplicationStatus.UNDEFINED);
    return appAttempt;
}

16 View Complete Implementation : FiCaSchedulerApp.java
Copyright Apache License 2.0
Author : apache
public void reserve(SchedulerRequestKey schedulerKey, FiCaSchedulerNode node, RMContainer rmContainer, Container container, boolean reReservation) {
    // Update reserved metrics if this is the first reservation
    // rmContainer will be moved to reserved in the super.reserve
    if (!reReservation) {
        queue.getMetrics().reserveResource(node.getParreplacedion(), getUser(), container.getResource());
    }
    // Inform the application
    rmContainer = super.reserve(node, schedulerKey, rmContainer, container);
    // Update the node
    node.reserveResource(this, schedulerKey, rmContainer);
}

16 View Complete Implementation : MockAMLauncher.java
Copyright Apache License 2.0
Author : apache
@Override
@SuppressWarnings("unchecked")
public void handle(AMLauncherEvent event) {
    if (AMLauncherEventType.LAUNCH == event.getType()) {
        ApplicationId appId = event.getAppAttempt().getAppAttemptId().getApplicationId();
        // find AMSimulator
        AMSimulator ams = appIdAMSim.get(appId);
        if (ams != null) {
            try {
                Container amContainer = event.getAppAttempt().getMasterContainer();
                setupAMRMToken(event.getAppAttempt());
                // Notify RMAppAttempt to change state
                super.context.getDispatcher().getEventHandler().handle(new RMAppAttemptEvent(event.getAppAttempt().getAppAttemptId(), RMAppAttemptEventType.LAUNCHED));
                ams.notifyAMContainerLaunched(event.getAppAttempt().getMasterContainer());
                LOG.info("Notify AM launcher launched:" + amContainer.getId());
                se.getNmMap().get(amContainer.getNodeId()).addNewContainer(amContainer, 100000000L);
                return;
            } catch (Exception e) {
                throw new YarnRuntimeException(e);
            }
        }
        throw new YarnRuntimeException("Didn't find any AMSimulator for applicationId=" + appId);
    }
}

16 View Complete Implementation : MockServiceAM.java
Copyright Apache License 2.0
Author : apache
/**
 * Simulates a recovered container that is sent to the AM in the heartbeat
 * response.
 *
 * @param containerId The ID for the container
 * @param compName    The component to which the recovered container is fed.
 */
public void feedRecoveredContainer(ContainerId containerId, String compName) {
    Container container = createContainer(containerId, compName);
    recoveredContainers.add(container);
    addContainerStatus(container, ContainerState.RUNNING);
}

16 View Complete Implementation : TestSystemMetricsPublisher.java
Copyright Apache License 2.0
Author : apache
private static RMAppAttempt createRMAppAttempt(ApplicationAttemptId appAttemptId, boolean unmanagedAMAttempt) {
    RMAppAttempt appAttempt = mock(RMAppAttempt.clreplaced);
    when(appAttempt.getAppAttemptId()).thenReturn(appAttemptId);
    when(appAttempt.getHost()).thenReturn("test host");
    when(appAttempt.getRpcPort()).thenReturn(-100);
    if (!unmanagedAMAttempt) {
        Container container = mock(Container.clreplaced);
        when(container.getId()).thenReturn(ContainerId.newContainerId(appAttemptId, 1));
        when(appAttempt.getMasterContainer()).thenReturn(container);
    }
    when(appAttempt.getDiagnostics()).thenReturn("test diagnostics info");
    when(appAttempt.getTrackingUrl()).thenReturn("test tracking url");
    when(appAttempt.getOriginalTrackingUrl()).thenReturn("test original tracking url");
    return appAttempt;
}

16 View Complete Implementation : NMTokenSecretManagerInNM.java
Copyright Apache License 2.0
Author : apache
/**
 * Used by the Distributed Scheduler framework to generate NMTokens
 * @param applicationSubmitter
 * @param container
 * @return NMToken
 */
public NMToken generateNMToken(String applicationSubmitter, Container container) {
    this.readLock.lock();
    try {
        Token token = createNMToken(container.getId().getApplicationAttemptId(), container.getNodeId(), applicationSubmitter);
        return NMToken.newInstance(container.getNodeId(), token);
    } finally {
        this.readLock.unlock();
    }
}

15 View Complete Implementation : NodeManager.java
Copyright Apache License 2.0
Author : apache
private List<ContainerStatus> getContainerStatuses(Map<ApplicationId, List<Container>> containers) {
    List<ContainerStatus> containerStatuses = new ArrayList<ContainerStatus>();
    for (List<Container> appContainers : containers.values()) {
        for (Container container : appContainers) {
            containerStatuses.add(containerStatusMap.get(container));
        }
    }
    return containerStatuses;
}

15 View Complete Implementation : TestSchedulingWithAllocationRequestId.java
Copyright Apache License 2.0
Author : apache
private void checkAllocatedContainer(Container allocated, int memory, NodeId nodeId, long allocationRequestId) {
    replacedert.replacedertEquals(memory, allocated.getResource().getMemorySize());
    replacedert.replacedertEquals(nodeId, allocated.getNodeId());
    replacedert.replacedertEquals(allocationRequestId, allocated.getAllocationRequestId());
}

15 View Complete Implementation : FiCaSchedulerNode.java
Copyright Apache License 2.0
Author : apache
@Override
protected synchronized void updateResourceForReleasedContainer(Container container) {
    super.updateResourceForReleasedContainer(container);
    if (killableContainers.containsKey(container.getId())) {
        Resources.subtractFrom(totalKillableResources, container.getResource());
        killableContainers.remove(container.getId());
    }
}

15 View Complete Implementation : SchedulerNode.java
Copyright Apache License 2.0
Author : apache
/**
 * The Scheduler has allocated containers on this node to the given
 * application.
 * @param rmContainer Allocated container
 * @param launchedOnNode True if the container has been launched
 */
protected synchronized void allocateContainer(RMContainer rmContainer, boolean launchedOnNode) {
    Container container = rmContainer.getContainer();
    if (rmContainer.getExecutionType() == ExecutionType.GUARANTEED) {
        deductUnallocatedResource(container.getResource());
        ++numContainers;
    }
    launchedContainers.put(container.getId(), new ContainerInfo(rmContainer, launchedOnNode));
}

15 View Complete Implementation : MockServiceAM.java
Copyright Apache License 2.0
Author : apache
private Container createContainer(ContainerId containerId, String compName) {
    NodeId nodeId = NodeId.newInstance("localhost", 1234);
    Container container = Container.newInstance(containerId, nodeId, "localhost", Resource.newInstance(100, 1), Priority.newInstance(0), null);
    long allocateId = context.scheduler.getAllComponents().get(compName).getAllocateId();
    container.setAllocationRequestId(allocateId);
    return container;
}

15 View Complete Implementation : NMTokenSecretManagerInRM.java
Copyright Apache License 2.0
Author : apache
public NMToken createAndGetNMToken(String applicationSubmitter, ApplicationAttemptId appAttemptId, Container container) {
    this.writeLock.lock();
    try {
        HashSet<NodeId> nodeSet = this.appAttemptToNodeKeyMap.get(appAttemptId);
        NMToken nmToken = null;
        if (nodeSet != null) {
            if (!nodeSet.contains(container.getNodeId())) {
                LOG.info("Sending NMToken for nodeId : " + container.getNodeId() + " for container : " + container.getId());
                Token token = createNMToken(container.getId().getApplicationAttemptId(), container.getNodeId(), applicationSubmitter);
                nmToken = NMToken.newInstance(container.getNodeId(), token);
                nodeSet.add(container.getNodeId());
            }
        }
        return nmToken;
    } finally {
        this.writeLock.unlock();
    }
}

15 View Complete Implementation : SchedulerUtils.java
Copyright Apache License 2.0
Author : apache
public static RMContainer createOpportunisticRmContainer(RMContext rmContext, Container container, boolean isRemotelyAllocated) {
    SchedulerNode node = ((AbstractYarnScheduler) rmContext.getScheduler()).getNode(container.getNodeId());
    if (node == null) {
        return null;
    }
    SchedulerApplicationAttempt appAttempt = ((AbstractYarnScheduler) rmContext.getScheduler()).getCurrentAttemptForContainer(container.getId());
    RMContainer rmContainer = new RMContainerImpl(container, SchedulerRequestKey.extractFrom(container), appAttempt.getApplicationAttemptId(), container.getNodeId(), appAttempt.getUser(), rmContext, isRemotelyAllocated);
    appAttempt.addRMContainer(container.getId(), rmContainer);
    node.allocateContainer(rmContainer);
    return rmContainer;
}

15 View Complete Implementation : MockRunningServiceContext.java
Copyright Apache License 2.0
Author : apache
public void replacedignNewContainer(ApplicationAttemptId attemptId, long containerNum, Component component) {
    Container container = org.apache.hadoop.yarn.api.records.Container.newInstance(ContainerId.newContainerId(attemptId, containerNum), NODE_ID, "localhost", null, null, null);
    component.handle(new ComponentEvent(component.getName(), ComponentEventType.CONTAINER_ALLOCATED).setContainer(container).setContainerId(container.getId()));
    ComponentInstance instance = this.scheduler.getLiveInstances().get(container.getId());
    ComponentInstanceEvent startEvent = new ComponentInstanceEvent(container.getId(), ComponentInstanceEventType.START);
    instance.handle(startEvent);
    ComponentInstanceEvent readyEvent = new ComponentInstanceEvent(container.getId(), ComponentInstanceEventType.BECOME_READY);
    instance.handle(readyEvent);
}

15 View Complete Implementation : TestAMRMClientAsync.java
Copyright Apache License 2.0
Author : apache
private AllocateResponse createAllocateResponse(List<ContainerStatus> completed, List<Container> allocated, List<Container> increased, List<Container> decreased, List<NMToken> nmTokens) {
    List<UpdatedContainer> updatedContainers = new ArrayList<>();
    for (Container c : increased) {
        updatedContainers.add(UpdatedContainer.newInstance(ContainerUpdateType.INCREASE_RESOURCE, c));
    }
    for (Container c : decreased) {
        updatedContainers.add(UpdatedContainer.newInstance(ContainerUpdateType.DECREASE_RESOURCE, c));
    }
    AllocateResponse response = AllocateResponse.newInstance(0, completed, allocated, new ArrayList<NodeReport>(), null, null, 1, null, nmTokens, null, updatedContainers);
    return response;
}

15 View Complete Implementation : MockServiceAM.java
Copyright Apache License 2.0
Author : apache
/**
 * Feeds the container to the component.
 * @param service The service for the component
 * @param containerId container id
 * @param compName The component to which the container is fed
 * @return
 */
public Container feedContainerToComp(Service service, ContainerId containerId, String compName) {
    Container container = createContainer(containerId, compName);
    synchronized (feedContainers) {
        feedContainers.add(container);
    }
    addContainerStatus(container, ContainerState.RUNNING);
    return container;
}

14 View Complete Implementation : RMStateStoreTestBase.java
Copyright Apache License 2.0
Author : apache
protected RMAppAttempt storeAttempt(RMStateStore store, ApplicationAttemptId attemptId, String containerIdStr, Token<AMRMTokenIdentifier> appToken, SecretKey clientTokenMasterKey, TestDispatcher dispatcher) throws Exception {
    RMAppAttemptMetrics mockRmAppAttemptMetrics = mock(RMAppAttemptMetrics.clreplaced);
    Container container = new ContainerPBImpl();
    container.setId(ContainerId.fromString(containerIdStr));
    RMAppAttempt mockAttempt = mock(RMAppAttempt.clreplaced);
    when(mockAttempt.getAppAttemptId()).thenReturn(attemptId);
    when(mockAttempt.getMasterContainer()).thenReturn(container);
    when(mockAttempt.getAMRMToken()).thenReturn(appToken);
    when(mockAttempt.getClientTokenMasterKey()).thenReturn(clientTokenMasterKey);
    when(mockAttempt.getRMAppAttemptMetrics()).thenReturn(mockRmAppAttemptMetrics);
    when(mockRmAppAttemptMetrics.getAggregateAppResourceUsage()).thenReturn(new AggregateAppResourceUsage(new HashMap<>()));
    dispatcher.attemptId = attemptId;
    store.storeNewApplicationAttempt(mockAttempt);
    waitNotify(dispatcher);
    return mockAttempt;
}

14 View Complete Implementation : ContainerUpdateContext.java
Copyright Apache License 2.0
Author : apache
/**
 * Returns the resource that the container will finally be replacedigned with
 * at the end of the update operation.
 * @param tempContainer Temporary Container created for the operation.
 * @param existingContainer Existing Container.
 * @param updateType Update Type.
 * @return Final Resource.
 */
private Resource createUpdatedResource(Container tempContainer, Container existingContainer, ContainerUpdateType updateType) {
    if (ContainerUpdateType.INCREASE_RESOURCE == updateType) {
        return Resources.add(existingContainer.getResource(), tempContainer.getResource());
    } else if (ContainerUpdateType.DECREASE_RESOURCE == updateType) {
        return outstandingDecreases.get(existingContainer.getId());
    } else {
        return existingContainer.getResource();
    }
}

14 View Complete Implementation : TestNMSimulator.java
Copyright Apache License 2.0
Author : apache
@Test
public void testNMSimulator() throws Exception {
    // Register one node
    NMSimulator node1 = new NMSimulator();
    node1.init("/rack1/node1", Resources.createResource(GB * 10, 10), 0, 1000, rm, -1f);
    node1.middleStep();
    int numClusterNodes = rm.getResourceScheduler().getNumClusterNodes();
    int replacedulativeSleepTime = 0;
    int sleepInterval = 100;
    while (numClusterNodes != 1 && replacedulativeSleepTime < 5000) {
        Thread.sleep(sleepInterval);
        replacedulativeSleepTime = replacedulativeSleepTime + sleepInterval;
        numClusterNodes = rm.getResourceScheduler().getNumClusterNodes();
    }
    GenericTestUtils.waitFor(new Supplier<Boolean>() {

        @Override
        public Boolean get() {
            return rm.getResourceScheduler().getRootQueueMetrics().getAvailableMB() > 0;
        }
    }, 500, 10000);
    replacedert.replacedertEquals(1, rm.getResourceScheduler().getNumClusterNodes());
    replacedert.replacedertEquals(GB * 10, rm.getResourceScheduler().getRootQueueMetrics().getAvailableMB());
    replacedert.replacedertEquals(10, rm.getResourceScheduler().getRootQueueMetrics().getAvailableVirtualCores());
    // Allocate one container on node1
    ContainerId cId1 = newContainerId(1, 1, 1);
    Container container1 = Container.newInstance(cId1, null, null, Resources.createResource(GB, 1), null, null);
    node1.addNewContainer(container1, 100000l);
    replacedert.replacedertTrue("Node1 should have one running container.", node1.getRunningContainers().containsKey(cId1));
    // Allocate one AM container on node1
    ContainerId cId2 = newContainerId(2, 1, 1);
    Container container2 = Container.newInstance(cId2, null, null, Resources.createResource(GB, 1), null, null);
    node1.addNewContainer(container2, -1l);
    replacedert.replacedertTrue("Node1 should have one running AM container", node1.getAMContainers().contains(cId2));
    // Remove containers
    node1.cleanupContainer(cId1);
    replacedert.replacedertTrue("Container1 should be removed from Node1.", node1.getCompletedContainers().contains(cId1));
    node1.cleanupContainer(cId2);
    replacedert.replacedertFalse("Container2 should be removed from Node1.", node1.getAMContainers().contains(cId2));
}

14 View Complete Implementation : OpportunisticContainerAllocator.java
Copyright Apache License 2.0
Author : apache
@SuppressWarnings("checkstyle:parameternumber")
protected Container createContainer(long rmIdentifier, AllocationParams appParams, ContainerIdGenerator idCounter, ApplicationAttemptId id, String userName, Map<Resource, List<Allocation>> allocations, String location, ResourceRequest anyAsk, RemoteNode rNode) throws YarnException {
    Container container = buildContainer(rmIdentifier, appParams, idCounter, anyAsk, id, userName, rNode);
    List<Allocation> allocList = allocations.get(anyAsk.getCapability());
    if (allocList == null) {
        allocList = new ArrayList<>();
        allocations.put(anyAsk.getCapability(), allocList);
    }
    allocList.add(new Allocation(container, location));
    return container;
}

14 View Complete Implementation : DistributedScheduler.java
Copyright Apache License 2.0
Author : apache
/**
 * Adds all the newly allocated Containers to the allocate Response.
 * Additionally, in case the NMToken for one of the nodes does not exist, it
 * generates one and adds it to the response.
 */
private void updateAllocateResponse(AllocateResponse response, List<NMToken> nmTokens, List<Container> allocatedContainers) {
    List<NMToken> newTokens = new ArrayList<>();
    if (allocatedContainers.size() > 0) {
        response.getAllocatedContainers().addAll(allocatedContainers);
        for (Container alloc : allocatedContainers) {
            if (!nodeTokens.containsKey(alloc.getNodeId())) {
                newTokens.add(nmSecretManager.generateNMToken(appSubmitter, alloc));
            }
        }
        List<NMToken> retTokens = new ArrayList<>(nmTokens);
        retTokens.addAll(newTokens);
        response.setNMTokens(retTokens);
    }
}

14 View Complete Implementation : TestNMClientAsync.java
Copyright Apache License 2.0
Author : apache
@Test(timeout = 10000)
public void testOutOfOrder() throws Exception {
    CyclicBarrier barrierA = new CyclicBarrier(2);
    CyclicBarrier barrierB = new CyclicBarrier(2);
    CyclicBarrier barrierC = new CyclicBarrier(2);
    asyncClient = new MockNMClientAsync2(barrierA, barrierB, barrierC);
    asyncClient.init(new Configuration());
    asyncClient.start();
    final Container container = mockContainer(1);
    final ContainerLaunchContext clc = recordFactory.newRecordInstance(ContainerLaunchContext.clreplaced);
    // start container from another thread
    Thread t = new Thread() {

        @Override
        public void run() {
            asyncClient.startContainerAsync(container, clc);
        }
    };
    t.start();
    barrierA.await();
    asyncClient.stopContainerAsync(container.getId(), container.getNodeId());
    barrierC.await();
    replacedert.replacedertFalse("Starting and stopping should be out of order", ((TestCallbackHandler2) asyncClient.getCallbackHandler()).exceptionOccurred.get());
}

14 View Complete Implementation : TestSchedulerOvercommit.java
Copyright Apache License 2.0
Author : apache
/**
 * Reducing the resources with 0 time out kills the container right away.
 */
@Test
public void testReduceKill() throws Exception {
    Container container = createContainer(am, 2 * GB);
    replacedertMemory(scheduler, nmId, 4 * GB, 0);
    // Reducing to 2GB should kill the container
    long t0 = Time.now();
    updateNodeResource(rm, nmId, 2 * GB, 2, 0);
    waitMemory(scheduler, nm, 2 * GB, 0 * GB, INTERVAL, 2 * INTERVAL);
    // Check that the new container was killed
    List<ContainerStatus> completedContainers = am.schedule().getCompletedContainersStatuses();
    replacedertEquals(1, completedContainers.size());
    ContainerStatus containerStatus = completedContainers.get(0);
    replacedertContainerKilled(container.getId(), containerStatus);
    // It should kill the containers right away
    replacedertTime(0, Time.now() - t0);
}

14 View Complete Implementation : AppSchedulingInfo.java
Copyright Apache License 2.0
Author : apache
public static void updateMetrics(ApplicationId applicationId, NodeType type, SchedulerNode node, Container containerAllocated, String user, Queue queue) {
    LOG.debug("allocate: applicationId={} container={} host={} user={}" + " resource={} type={}", applicationId, containerAllocated.getId(), containerAllocated.getNodeId(), user, containerAllocated.getResource(), type);
    if (node != null) {
        queue.getMetrics().allocateResources(node.getParreplacedion(), user, 1, containerAllocated.getResource(), true);
    }
    queue.getMetrics().incrNodeTypeAggregations(user, type);
}

13 View Complete Implementation : BuilderUtils.java
Copyright Apache License 2.0
Author : apache
public static Container newContainer(ContainerId containerId, NodeId nodeId, String nodeHttpAddress, Resource resource, Priority priority, Token containerToken, ExecutionType executionType, long allocationRequestId) {
    Container container = recordFactory.newRecordInstance(Container.clreplaced);
    container.setId(containerId);
    container.setNodeId(nodeId);
    container.setNodeHttpAddress(nodeHttpAddress);
    container.setResource(resource);
    container.setPriority(priority);
    container.setContainerToken(containerToken);
    container.setExecutionType(executionType);
    container.setAllocationRequestId(allocationRequestId);
    return container;
}

13 View Complete Implementation : CentralizedOpportunisticContainerAllocator.java
Copyright Apache License 2.0
Author : apache
@SuppressWarnings("checkstyle:parameternumber")
private List<Container> allocateNodeLocal(EnrichedResourceRequest enrichedAsk, String nodeLocation, int toAllocate, long rmIdentifier, AllocationParams appParams, ContainerIdGenerator idCounter, Set<String> blacklist, ApplicationAttemptId id, String userName, Map<Resource, List<Allocation>> allocations) throws YarnException {
    List<Container> allocatedContainers = new ArrayList<>();
    while (toAllocate > 0) {
        RMNode node = nodeQueueLoadMonitor.selectLocalNode(nodeLocation, blacklist);
        if (node != null) {
            toAllocate--;
            Container container = createContainer(rmIdentifier, appParams, idCounter, id, userName, allocations, nodeLocation, enrichedAsk.getRequest(), convertToRemoteNode(node));
            allocatedContainers.add(container);
            LOG.info("Allocated [{}] as opportunistic at location [{}]", container.getId(), nodeLocation);
            metrics.incrNodeLocalOppContainers();
        } else {
            // we couldn't allocate any - break the loop.
            break;
        }
    }
    return allocatedContainers;
}

13 View Complete Implementation : TestSystemMetricsPublisherForV2.java
Copyright Apache License 2.0
Author : apache
private static RMAppAttempt createRMAppAttempt(ApplicationAttemptId appAttemptId) {
    RMAppAttempt appAttempt = mock(RMAppAttempt.clreplaced);
    when(appAttempt.getAppAttemptId()).thenReturn(appAttemptId);
    when(appAttempt.getHost()).thenReturn("test host");
    when(appAttempt.getRpcPort()).thenReturn(-100);
    Container container = mock(Container.clreplaced);
    when(container.getId()).thenReturn(ContainerId.newContainerId(appAttemptId, 1));
    when(container.getNodeId()).thenReturn(NodeId.newInstance("testhost", 8042));
    when(container.getNodeHttpAddress()).thenReturn("testhost:25050");
    when(appAttempt.getMasterContainer()).thenReturn(container);
    when(appAttempt.getDiagnostics()).thenReturn("test diagnostics info");
    when(appAttempt.getTrackingUrl()).thenReturn("test tracking url");
    when(appAttempt.getOriginalTrackingUrl()).thenReturn("test original tracking url");
    when(appAttempt.getStartTime()).thenReturn(200L);
    return appAttempt;
}

13 View Complete Implementation : TestApplicationMasterServiceInterceptor.java
Copyright Apache License 2.0
Author : apache
@Test(timeout = 300000)
public void testApplicationMasterInterceptor() throws Exception {
    conf.set(YarnConfiguration.RM_APPLICATION_MASTER_SERVICE_PROCESSORS, TestInterceptor1.clreplaced.getName() + "," + TestInterceptor2.clreplaced.getName());
    MockRM rm = new MockRM(conf);
    rm.start();
    // Register node1
    MockNM nm1 = rm.registerNode("127.0.0.1:1234", 6 * GB);
    // Submit an application
    RMApp app1 = MockRMAppSubmitter.submitWithMemory(2048, rm);
    // kick the scheduling
    nm1.nodeHeartbeat(true);
    RMAppAttempt attempt1 = app1.getCurrentAppAttempt();
    MockAM am1 = rm.sendAMLaunched(attempt1.getAppAttemptId());
    am1.registerAppAttempt();
    int allocCount = 0;
    am1.addRequests(new String[] { "127.0.0.1" }, GB, 1, 1);
    // send the request
    AllocateResponse alloc1Response = am1.schedule();
    allocCount++;
    // kick the scheduler
    nm1.nodeHeartbeat(true);
    while (alloc1Response.getAllocatedContainers().size() < 1) {
        LOG.info("Waiting for containers to be created for app 1...");
        sleep(1000);
        alloc1Response = am1.schedule();
        allocCount++;
    }
    // replacedert RMIdentifier is set properly in allocated containers
    Container allocatedContainer = alloc1Response.getAllocatedContainers().get(0);
    ContainerTokenIdentifier tokenId = BuilderUtils.newContainerTokenIdentifier(allocatedContainer.getContainerToken());
    am1.unregisterAppAttempt();
    replacedert.replacedertEquals(1, beforeRegCount.get());
    replacedert.replacedertEquals(1, afterRegCount.get());
    // The allocate calls should be incremented twice
    replacedert.replacedertEquals(allocCount * 2, beforeAllocCount.get());
    replacedert.replacedertEquals(allocCount * 2, afterAllocCount.get());
    // Finish should only be called once, since the FirstInterceptor
    // does not forward the call.
    replacedert.replacedertEquals(1, beforeFinishCount.get());
    replacedert.replacedertEquals(1, afterFinishCount.get());
    rm.stop();
}

13 View Complete Implementation : ContainerUpdateContext.java
Copyright Apache License 2.0
Author : apache
/**
 * Add the container to outstanding decreases.
 * @param updateReq UpdateContainerRequest.
 * @param schedulerNode SchedulerNode.
 * @param container Container.
 * @return If it was possible to decrease the container.
 */
public synchronized boolean checkAndAddToOutstandingDecreases(UpdateContainerRequest updateReq, SchedulerNode schedulerNode, Container container) {
    if (outstandingDecreases.containsKey(container.getId())) {
        return false;
    }
    if (ContainerUpdateType.DECREASE_RESOURCE == updateReq.getContainerUpdateType()) {
        SchedulerRequestKey updateKey = new SchedulerRequestKey(container.getPriority(), container.getAllocationRequestId(), container.getId());
        cancelPreviousRequest(schedulerNode, updateKey);
        outstandingDecreases.put(container.getId(), updateReq.getCapability());
    } else {
        outstandingDecreases.put(container.getId(), container.getResource());
    }
    return true;
}

13 View Complete Implementation : MockNM.java
Copyright Apache License 2.0
Author : apache
public void containerIncreaseStatus(Container container) throws Exception {
    ContainerStatus containerStatus = BuilderUtils.newContainerStatus(container.getId(), ContainerState.RUNNING, "Success", 0, container.getResource());
    List<Container> increasedConts = Collections.singletonList(container);
    nodeHeartbeat(Collections.singletonList(containerStatus), increasedConts, true, responseId);
}

13 View Complete Implementation : TestNMClient.java
Copyright Apache License 2.0
Author : apache
private void testGetContainerStatus(Container container, int index, ContainerState state, String diagnostics, List<Integer> exitStatuses) throws YarnException, IOException {
    while (true) {
        sleep(250);
        ContainerStatus status = nmClient.getContainerStatus(container.getId(), container.getNodeId());
        // NodeManager may still need some time to get the stable
        // container status
        if (status.getState() == state) {
            replacedertEquals(container.getId(), status.getContainerId());
            replacedertTrue("" + index + ": " + status.getDiagnostics(), status.getDiagnostics().contains(diagnostics));
            replacedertTrue("Exit Statuses are supposed to be in: " + exitStatuses + ", but the actual exit status code is: " + status.getExitStatus(), exitStatuses.contains(status.getExitStatus()));
            break;
        }
    }
}

13 View Complete Implementation : NMSimulator.java
Copyright Apache License 2.0
Author : apache
/**
 * launch a new container with the given life time
 */
public void addNewContainer(Container container, long lifeTimeMS) {
    LOG.debug("NodeManager {} launches a new container ({}).", node.getNodeID(), container.getId());
    if (lifeTimeMS != -1) {
        // normal container
        ContainerSimulator cs = new ContainerSimulator(container.getId(), container.getResource(), lifeTimeMS + System.currentTimeMillis(), lifeTimeMS, container.getAllocationRequestId());
        containerQueue.add(cs);
        runningContainers.put(cs.getId(), cs);
    } else {
        // AM container
        // -1 means AMContainer
        synchronized (amContainerList) {
            amContainerList.add(container.getId());
        }
    }
}

12 View Complete Implementation : AbstractProviderService.java
Copyright Apache License 2.0
Author : apache
public void buildContainerEnvironment(AbstractLauncher launcher, Service service, ComponentInstance instance, SliderFileSystem fileSystem, Configuration yarnConf, Container container, ContainerLaunchService.ComponentLaunchContext compLaunchContext, Map<String, String> tokensForSubsreplacedution) throws IOException, SliderException {
    // Set the environment variables in launcher
    launcher.putEnv(ServiceUtils.buildEnvMap(compLaunchContext.getConfiguration(), tokensForSubsreplacedution));
    launcher.setEnv("WORK_DIR", ApplicationConstants.Environment.PWD.$());
    launcher.setEnv("LOG_DIR", ApplicationConstants.LOG_DIR_EXPANSION_VAR);
    if (System.getenv(HADOOP_USER_NAME) != null) {
        launcher.setEnv(HADOOP_USER_NAME, System.getenv(HADOOP_USER_NAME));
    }
    launcher.setEnv("LANG", "en_US.UTF-8");
    launcher.setEnv("LC_ALL", "en_US.UTF-8");
    launcher.setEnv("LANGUAGE", "en_US.UTF-8");
    for (Entry<String, String> entry : launcher.getEnv().entrySet()) {
        tokensForSubsreplacedution.put($(entry.getKey()), entry.getValue());
    }
}

12 View Complete Implementation : RegularContainerAllocator.java
Copyright Apache License 2.0
Author : apache
private ContainerAllocation handleNewContainerAllocation(ContainerAllocation allocationResult, FiCaSchedulerNode node, SchedulerRequestKey schedulerKey, Container container) {
    // Inform the application
    RMContainer allocatedContainer = application.allocate(node, schedulerKey, container);
    allocationResult.updatedContainer = allocatedContainer;
    // Does the application need this resource?
    if (allocatedContainer == null) {
        // Skip this app if we failed to allocate.
        ContainerAllocation ret = new ContainerAllocation(allocationResult.containerToBeUnreserved, null, AllocationState.APP_SKIPPED);
        ActivitiesLogger.APP.recordAppActivityWithoutAllocation(activitiesManager, node, application, schedulerKey, ActivityDiagnosticConstant.APPLICATION_FAIL_TO_ALLOCATE, ActivityState.REJECTED, ActivityLevel.APP);
        return ret;
    }
    return allocationResult;
}

12 View Complete Implementation : AbstractProviderService.java
Copyright Apache License 2.0
Author : apache
public ResolvedLaunchParams buildContainerLaunchContext(AbstractLauncher launcher, Service service, ComponentInstance instance, SliderFileSystem fileSystem, Configuration yarnConf, Container container, ContainerLaunchService.ComponentLaunchContext compLaunchContext) throws IOException, SliderException {
    ResolvedLaunchParams resolved = new ResolvedLaunchParams();
    processArtifact(launcher, instance, fileSystem, service, compLaunchContext);
    ServiceContext context = instance.getComponent().getScheduler().getContext();
    // Generate tokens (key-value pair) for config subsreplacedution.
    Map<String, String> tokensForSubsreplacedution = buildContainerTokens(instance, container, compLaunchContext);
    // Setup launch context environment
    buildContainerEnvironment(launcher, service, instance, fileSystem, yarnConf, container, compLaunchContext, tokensForSubsreplacedution);
    // create config file on hdfs and addResolvedRsrcPath local resource
    ProviderUtils.createConfigFileAndAddLocalResource(launcher, fileSystem, compLaunchContext, tokensForSubsreplacedution, instance, context, resolved);
    // handles static files (like normal file / archive file) for localization.
    ProviderUtils.handleStaticFilesForLocalization(launcher, fileSystem, compLaunchContext, resolved);
    // replace launch command with token specific information
    buildContainerLaunchCommand(launcher, service, instance, fileSystem, yarnConf, container, compLaunchContext, tokensForSubsreplacedution);
    // Setup container retry settings
    buildContainerRetry(launcher, yarnConf, compLaunchContext, instance);
    return resolved;
}

12 View Complete Implementation : ApplicationAttemptStateData.java
Copyright Apache License 2.0
Author : apache
public static ApplicationAttemptStateData newInstance(ApplicationAttemptId attemptId, Container container, Credentials attemptTokens, long startTime, RMAppAttemptState finalState, String finalTrackingUrl, String diagnostics, FinalApplicationStatus amUnregisteredFinalStatus, int exitStatus, long finishTime, Map<String, Long> resourceSecondsMap, Map<String, Long> preemptedResourceSecondsMap) {
    ApplicationAttemptStateData attemptStateData = Records.newRecord(ApplicationAttemptStateData.clreplaced);
    attemptStateData.setAttemptId(attemptId);
    attemptStateData.setMasterContainer(container);
    attemptStateData.setAppAttemptTokens(attemptTokens);
    attemptStateData.setState(finalState);
    attemptStateData.setFinalTrackingUrl(finalTrackingUrl);
    attemptStateData.setDiagnostics(diagnostics == null ? "" : diagnostics);
    attemptStateData.setStartTime(startTime);
    attemptStateData.setFinalApplicationStatus(amUnregisteredFinalStatus);
    attemptStateData.setAMContainerExitStatus(exitStatus);
    attemptStateData.setFinishTime(finishTime);
    attemptStateData.setMemorySeconds(RMServerUtils.getOrDefault(resourceSecondsMap, ResourceInformation.MEMORY_MB.getName(), 0L));
    attemptStateData.setVcoreSeconds(RMServerUtils.getOrDefault(resourceSecondsMap, ResourceInformation.VCORES.getName(), 0L));
    attemptStateData.setPreemptedMemorySeconds(RMServerUtils.getOrDefault(preemptedResourceSecondsMap, ResourceInformation.MEMORY_MB.getName(), 0L));
    attemptStateData.setPreemptedVcoreSeconds(RMServerUtils.getOrDefault(preemptedResourceSecondsMap, ResourceInformation.VCORES.getName(), 0L));
    attemptStateData.setResourceSecondsMap(resourceSecondsMap);
    attemptStateData.setPreemptedResourceSecondsMap(preemptedResourceSecondsMap);
    return attemptStateData;
}