org.apache.hadoop.yarn.api.records.ApplicationAttemptId.newInstance() - java examples

Here are the examples of the java api org.apache.hadoop.yarn.api.records.ApplicationAttemptId.newInstance() taken from open source projects. By voting up you can indicate which examples are most useful and appropriate.

155 Examples 7

19 View Complete Implementation : TestDecommissioningNodesWatcher.java
Copyright Apache License 2.0
Author : apache
// Get mocked ContainerStatus for bunch of containers,
// where numRunningContainers are RUNNING.
private List<ContainerStatus> getContainerStatuses(RMApp app, int numRunningContainers) {
    // Total 3 containers
    final int total = 3;
    numRunningContainers = Math.min(total, numRunningContainers);
    List<ContainerStatus> output = new ArrayList<ContainerStatus>();
    for (int i = 0; i < total; i++) {
        ContainerState cstate = (i >= numRunningContainers) ? ContainerState.COMPLETE : ContainerState.RUNNING;
        output.add(ContainerStatus.newInstance(ContainerId.newContainerId(ApplicationAttemptId.newInstance(app.getApplicationId(), 0), i), cstate, "", 0));
    }
    return output;
}

19 View Complete Implementation : TestAHSWebApp.java
Copyright Apache License 2.0
Author : apache
@Test
public void testAppAttemptPage() throws Exception {
    Injector injector = WebAppTests.createMockInjector(ApplicationBaseProtocol.clreplaced, mockApplicationHistoryClientService(1, 1, 5));
    AppAttemptPage appAttemptPageInstance = injector.getInstance(AppAttemptPage.clreplaced);
    appAttemptPageInstance.render();
    WebAppTests.flushOutput(injector);
    appAttemptPageInstance.set(YarnWebParams.APPLICATION_ATTEMPT_ID, ApplicationAttemptId.newInstance(ApplicationId.newInstance(0, 1), 1).toString());
    appAttemptPageInstance.render();
    WebAppTests.flushOutput(injector);
}

19 View Complete Implementation : TestDistributedOpportunisticContainerAllocator.java
Copyright Apache License 2.0
Author : apache
@Test
public void testRoundRobinRackLocalAllocationSameSchedKey() throws Exception {
    ResourceBlacklistRequest blacklistRequest = ResourceBlacklistRequest.newInstance(new ArrayList<>(), new ArrayList<>());
    List<ResourceRequest> reqs = Arrays.asList(ResourceRequest.newInstance(PRIORITY_NORMAL, "*", CAPABILITY_1GB, 2, true, null, OPPORTUNISTIC_REQ), ResourceRequest.newInstance(PRIORITY_NORMAL, "h1", CAPABILITY_1GB, 2, true, null, OPPORTUNISTIC_REQ), ResourceRequest.newInstance(PRIORITY_NORMAL, "/r1", CAPABILITY_1GB, 2, true, null, OPPORTUNISTIC_REQ));
    ApplicationAttemptId appAttId = ApplicationAttemptId.newInstance(ApplicationId.newInstance(0L, 1), 1);
    oppCntxt.updateNodeList(Arrays.asList(RemoteNode.newInstance(NodeId.newInstance("h3", 1234), "h3:1234", "/r2"), RemoteNode.newInstance(NodeId.newInstance("h2", 1234), "h2:1234", "/r1"), RemoteNode.newInstance(NodeId.newInstance("h5", 1234), "h5:1234", "/r1"), RemoteNode.newInstance(NodeId.newInstance("h4", 1234), "h4:1234", "/r2")));
    List<Container> containers = allocator.allocateContainers(blacklistRequest, reqs, appAttId, oppCntxt, 1L, "luser");
    Set<String> allocatedHosts = new HashSet<>();
    for (Container c : containers) {
        allocatedHosts.add(c.getNodeHttpAddress());
    }
    LOG.info("Containers: {}", containers);
    replacedert.replacedertTrue(allocatedHosts.contains("h2:1234"));
    replacedert.replacedertTrue(allocatedHosts.contains("h5:1234"));
    replacedert.replacedertFalse(allocatedHosts.contains("h3:1234"));
    replacedert.replacedertFalse(allocatedHosts.contains("h4:1234"));
    replacedert.replacedertEquals(2, containers.size());
}

19 View Complete Implementation : TestApplicationHistoryClientService.java
Copyright Apache License 2.0
Author : apache
@Test
public void testContainers() throws IOException, YarnException {
    ApplicationId appId = ApplicationId.newInstance(0, 1);
    ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(appId, 1);
    ContainerId containerId = ContainerId.newContainerId(appAttemptId, 1);
    ContainerId containerId1 = ContainerId.newContainerId(appAttemptId, 2);
    GetContainersRequest request = GetContainersRequest.newInstance(appAttemptId);
    GetContainersResponse response = clientService.getContainers(request);
    List<ContainerReport> containers = response.getContainerList();
    replacedert.replacedertNotNull(containers);
    replacedert.replacedertEquals(containerId, containers.get(0).getContainerId());
    replacedert.replacedertEquals(containerId1, containers.get(1).getContainerId());
}

19 View Complete Implementation : TestFileSystemApplicationHistoryStore.java
Copyright Apache License 2.0
Author : apache
@Test
public void testWriteAfterApplicationFinish() throws IOException {
    LOG.info("Starting testWriteAfterApplicationFinish");
    ApplicationId appId = ApplicationId.newInstance(0, 1);
    writeApplicationStartData(appId);
    writeApplicationFinishData(appId);
    // write application attempt history data
    ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(appId, 1);
    try {
        writeApplicationAttemptStartData(appAttemptId);
        replacedert.fail();
    } catch (IOException e) {
        replacedert.replacedertTrue(e.getMessage().contains("is not opened"));
    }
    try {
        writeApplicationAttemptFinishData(appAttemptId);
        replacedert.fail();
    } catch (IOException e) {
        replacedert.replacedertTrue(e.getMessage().contains("is not opened"));
    }
    // write container history data
    ContainerId containerId = ContainerId.newContainerId(appAttemptId, 1);
    try {
        writeContainerStartData(containerId);
        replacedert.fail();
    } catch (IOException e) {
        replacedert.replacedertTrue(e.getMessage().contains("is not opened"));
    }
    try {
        writeContainerFinishData(containerId);
        replacedert.fail();
    } catch (IOException e) {
        replacedert.replacedertTrue(e.getMessage().contains("is not opened"));
    }
}

19 View Complete Implementation : TestApplicationHistoryClientService.java
Copyright Apache License 2.0
Author : apache
@Test
public void testApplicationAttemptReport() throws IOException, YarnException {
    ApplicationId appId = ApplicationId.newInstance(0, 1);
    ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(appId, 1);
    GetApplicationAttemptReportRequest request = GetApplicationAttemptReportRequest.newInstance(appAttemptId);
    GetApplicationAttemptReportResponse response = clientService.getApplicationAttemptReport(request);
    ApplicationAttemptReport attemptReport = response.getApplicationAttemptReport();
    replacedert.replacedertNotNull(attemptReport);
    replacedert.replacedertEquals("appattempt_0_0001_000001", attemptReport.getApplicationAttemptId().toString());
}

19 View Complete Implementation : TestCentralizedOpportunisticContainerAllocator.java
Copyright Apache License 2.0
Author : apache
/**
 * Tests Opportunistic container should not be allocated on blacklisted
 * nodes.
 * @throws Exception
 */
@Test
public void testBlacklistRejection() throws Exception {
    ResourceBlacklistRequest blacklistRequest = ResourceBlacklistRequest.newInstance(Arrays.asList("h1", "h2"), new ArrayList<>());
    List<ResourceRequest> reqs = Collections.singletonList(createResourceRequest(1, "*", 1));
    ApplicationAttemptId appAttId = ApplicationAttemptId.newInstance(ApplicationId.newInstance(0L, 1), 1);
    allocator.setNodeQueueLoadMonitor(createNodeQueueLoadMonitor(2, 2, 100));
    List<Container> containers = allocator.allocateContainers(blacklistRequest, reqs, appAttId, oppCntxt, 1L, "user");
    replacedertEquals(0, containers.size());
    replacedertEquals(1, oppCntxt.getOutstandingOpReqs().size());
}

19 View Complete Implementation : TestContainerLaunchRPC.java
Copyright Apache License 2.0
Author : apache
private void testRPCTimeout(String rpcClreplaced) throws Exception {
    Configuration conf = new Configuration();
    // set timeout low for the test
    conf.setInt("yarn.rpc.nm-command-timeout", 3000);
    conf.set(YarnConfiguration.IPC_RPC_IMPL, rpcClreplaced);
    YarnRPC rpc = YarnRPC.create(conf);
    String bindAddr = "localhost:0";
    InetSocketAddress addr = NetUtils.createSocketAddr(bindAddr);
    Server server = rpc.getServer(ContainerManagementProtocol.clreplaced, new DummyContainerManager(), addr, conf, null, 1);
    server.start();
    try {
        ContainerManagementProtocol proxy = (ContainerManagementProtocol) rpc.getProxy(ContainerManagementProtocol.clreplaced, server.getListenerAddress(), conf);
        ContainerLaunchContext containerLaunchContext = recordFactory.newRecordInstance(ContainerLaunchContext.clreplaced);
        ApplicationId applicationId = ApplicationId.newInstance(0, 0);
        ApplicationAttemptId applicationAttemptId = ApplicationAttemptId.newInstance(applicationId, 0);
        ContainerId containerId = ContainerId.newContainerId(applicationAttemptId, 100);
        NodeId nodeId = NodeId.newInstance("localhost", 1234);
        Resource resource = Resource.newInstance(1234, 2);
        ContainerTokenIdentifier containerTokenIdentifier = new ContainerTokenIdentifier(containerId, "localhost", "user", resource, System.currentTimeMillis() + 10000, 42, 42, Priority.newInstance(0), 0);
        Token containerToken = newContainerToken(nodeId, "preplacedword".getBytes(), containerTokenIdentifier);
        StartContainerRequest scRequest = StartContainerRequest.newInstance(containerLaunchContext, containerToken);
        List<StartContainerRequest> list = new ArrayList<StartContainerRequest>();
        list.add(scRequest);
        StartContainersRequest allRequests = StartContainersRequest.newInstance(list);
        try {
            proxy.startContainers(allRequests);
        } catch (Exception e) {
            LOG.info(StringUtils.stringifyException(e));
            replacedert.replacedertEquals("Error, exception is not: " + SocketTimeoutException.clreplaced.getName(), SocketTimeoutException.clreplaced.getName(), e.getClreplaced().getName());
            return;
        }
    } finally {
        server.stop();
    }
    replacedert.fail("timeout exception should have occurred!");
}

19 View Complete Implementation : TestAppLogAggregatorImpl.java
Copyright Apache License 2.0
Author : apache
@Test
public void testAggregatorWithRetentionPolicyDisabledShouldUploadAllFiles() throws Exception {
    final ApplicationId applicationId = ApplicationId.newInstance(System.currentTimeMillis(), 0);
    final ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(applicationId, 0);
    final ContainerId containerId = ContainerId.newContainerId(attemptId, 0);
    // create artificial log files
    final File appLogDir = new File(LOCAL_LOG_DIR, applicationId.toString());
    final File containerLogDir = new File(appLogDir, containerId.toString());
    containerLogDir.mkdirs();
    final Set<File> logFiles = createContainerLogFiles(containerLogDir, 3);
    final long logRetentionSecs = 10000;
    final long recoveredLogInitedTime = -1;
    verifyLogAggregationWithExpectedFiles2DeleteAndUpload(applicationId, containerId, logRetentionSecs, recoveredLogInitedTime, logFiles, logFiles);
}

19 View Complete Implementation : TestCentralizedOpportunisticContainerAllocator.java
Copyright Apache License 2.0
Author : apache
/**
 * Tests that allocation of Opportunistic containers should be spread out.
 * @throws Exception
 */
@Test
public void testRoundRobinSimpleAllocation() throws Exception {
    List<ResourceRequest> reqs = Arrays.asList(createResourceRequest(1, ResourceRequest.ANY, 1), createResourceRequest(2, ResourceRequest.ANY, 1), createResourceRequest(3, ResourceRequest.ANY, 1));
    ApplicationAttemptId appAttId = ApplicationAttemptId.newInstance(ApplicationId.newInstance(0L, 1), 1);
    allocator.setNodeQueueLoadMonitor(createNodeQueueLoadMonitor(3, 2, 3));
    List<Container> containers = allocator.allocateContainers(EMPTY_BLACKLIST_REQUEST, reqs, appAttId, oppCntxt, 1L, "user");
    LOG.info("Containers: {}", containers);
    Set<String> allocatedNodes = new HashSet<>();
    for (Container c : containers) {
        allocatedNodes.add(c.getNodeId().toString());
    }
    replacedertTrue(allocatedNodes.contains("h1:1234"));
    replacedertTrue(allocatedNodes.contains("h2:1234"));
    replacedertTrue(allocatedNodes.contains("h3:1234"));
    replacedertEquals(3, containers.size());
}

19 View Complete Implementation : TestPlacementConstraintsUtil.java
Copyright Apache License 2.0
Author : apache
@Test
public void testNodeAntiAffinityreplacedignment() throws InvalidAllocationTagsQueryException {
    PlacementConstraintManagerService pcm = new MemoryPlacementConstraintManager();
    AllocationTagsManager tm = new AllocationTagsManager(rmContext);
    // Register App1 with anti-affinity constraint map
    pcm.registerApplication(appId1, constraintMap2);
    /**
     * place container:
     * Node0:123 (Rack1):
     *    container_app1_1 (hbase-m)
     */
    RMNode n0_r1 = rmNodes.get(0);
    RMNode n1_r1 = rmNodes.get(1);
    RMNode n2_r2 = rmNodes.get(2);
    RMNode n3_r2 = rmNodes.get(3);
    SchedulerNode schedulerNode0 = newSchedulerNode(n0_r1.getHostName(), n0_r1.getRackName(), n0_r1.getNodeID());
    SchedulerNode schedulerNode1 = newSchedulerNode(n1_r1.getHostName(), n1_r1.getRackName(), n1_r1.getNodeID());
    SchedulerNode schedulerNode2 = newSchedulerNode(n2_r2.getHostName(), n2_r2.getRackName(), n2_r2.getNodeID());
    SchedulerNode schedulerNode3 = newSchedulerNode(n3_r2.getHostName(), n3_r2.getRackName(), n3_r2.getNodeID());
    // 1 Containers on node 0 with allocationTag 'hbase-m'
    ContainerId hbase_m = ContainerId.newContainerId(ApplicationAttemptId.newInstance(appId1, 0), 0);
    tm.addContainer(n0_r1.getNodeID(), hbase_m, ImmutableSet.of("hbase-m"));
    // 'spark' placement on Node0 should now FAIL
    replacedert.replacedertFalse(PlacementConstraintsUtil.canSatisfyConstraints(appId1, createSchedulingRequest(sourceTag1), schedulerNode0, pcm, tm));
    // SUCCEED on the rest of the nodes
    replacedert.replacedertTrue(PlacementConstraintsUtil.canSatisfyConstraints(appId1, createSchedulingRequest(sourceTag1), schedulerNode1, pcm, tm));
    replacedert.replacedertTrue(PlacementConstraintsUtil.canSatisfyConstraints(appId1, createSchedulingRequest(sourceTag1), schedulerNode2, pcm, tm));
    replacedert.replacedertTrue(PlacementConstraintsUtil.canSatisfyConstraints(appId1, createSchedulingRequest(sourceTag1), schedulerNode3, pcm, tm));
}

19 View Complete Implementation : TestYARNTokenIdentifier.java
Copyright Apache License 2.0
Author : apache
@Test
public void testAMContainerTokenIdentifier() throws IOException {
    ContainerId containerID = ContainerId.newContainerId(ApplicationAttemptId.newInstance(ApplicationId.newInstance(1, 1), 1), 1);
    String hostName = "host0";
    String appSubmitter = "usr0";
    Resource r = Resource.newInstance(1024, 1);
    long expiryTimeStamp = 1000;
    int masterKeyId = 1;
    long rmIdentifier = 1;
    Priority priority = Priority.newInstance(1);
    long creationTime = 1000;
    ContainerTokenIdentifier token = new ContainerTokenIdentifier(containerID, hostName, appSubmitter, r, expiryTimeStamp, masterKeyId, rmIdentifier, priority, creationTime, null, CommonNodeLabelsManager.NO_LABEL, ContainerType.APPLICATION_MASTER);
    ContainerTokenIdentifier anotherToken = new ContainerTokenIdentifier();
    byte[] tokenContent = token.getBytes();
    DataInputBuffer dib = new DataInputBuffer();
    dib.reset(tokenContent, tokenContent.length);
    anotherToken.readFields(dib);
    replacedert.replacedertEquals(ContainerType.APPLICATION_MASTER, anotherToken.getContainerType());
    replacedert.replacedertEquals(ExecutionType.GUARANTEED, anotherToken.getExecutionType());
    token = new ContainerTokenIdentifier(containerID, 0, hostName, appSubmitter, r, expiryTimeStamp, masterKeyId, rmIdentifier, priority, creationTime, null, CommonNodeLabelsManager.NO_LABEL, ContainerType.TASK, ExecutionType.OPPORTUNISTIC);
    anotherToken = new ContainerTokenIdentifier();
    tokenContent = token.getBytes();
    dib = new DataInputBuffer();
    dib.reset(tokenContent, tokenContent.length);
    anotherToken.readFields(dib);
    replacedert.replacedertEquals(ContainerType.TASK, anotherToken.getContainerType());
    replacedert.replacedertEquals(ExecutionType.OPPORTUNISTIC, anotherToken.getExecutionType());
}

19 View Complete Implementation : BaseAMRMProxyTest.java
Copyright Apache License 2.0
Author : apache
/**
 * Return an instance of ApplicationAttemptId using specified identifier. This
 * identifier will be used for the ApplicationId too.
 *
 * @param testAppId
 * @return
 */
protected ApplicationAttemptId getApplicationAttemptId(int testAppId) {
    return ApplicationAttemptId.newInstance(getApplicationId(testAppId), testAppId);
}

19 View Complete Implementation : TestCentralizedOpportunisticContainerAllocator.java
Copyright Apache License 2.0
Author : apache
/**
 * Tests that allocation of rack local Opportunistic container requests
 * with same allocation request id should be spread out.
 * @throws Exception
 */
@Test
public void testRoundRobinRackLocalAllocationSameSchedulerKey() throws Exception {
    List<ResourceRequest> reqs = Arrays.asList(createResourceRequest(2, "/r1", 2), createResourceRequest(2, "h5", 2), createResourceRequest(2, ResourceRequest.ANY, 2));
    ApplicationAttemptId appAttId = ApplicationAttemptId.newInstance(ApplicationId.newInstance(0L, 1), 1);
    NodeQueueLoadMonitor selector = createNodeQueueLoadMonitor(Arrays.asList("h1", "h2", "h3", "h4"), Arrays.asList("/r2", "/r1", "/r3", "/r1"), Arrays.asList(4, 4, 4, 4), Arrays.asList(5, 5, 5, 5));
    allocator.setNodeQueueLoadMonitor(selector);
    List<Container> containers = allocator.allocateContainers(EMPTY_BLACKLIST_REQUEST, reqs, appAttId, oppCntxt, 1L, "user");
    Set<String> allocatedHosts = new HashSet<>();
    for (Container c : containers) {
        allocatedHosts.add(c.getNodeId().toString());
    }
    LOG.info("Containers: {}", containers);
    replacedertTrue(allocatedHosts.contains("h2:1234"));
    replacedertTrue(allocatedHosts.contains("h4:1234"));
    replacedertFalse(allocatedHosts.contains("h1:1234"));
    replacedertFalse(allocatedHosts.contains("h3:1234"));
    replacedertEquals(2, containers.size());
}

19 View Complete Implementation : TestCentralizedOpportunisticContainerAllocator.java
Copyright Apache License 2.0
Author : apache
/**
 * Tests maximum opportunistic container allocation per AM heartbeat when
 * limit is set to -1.
 * @throws Exception
 */
@Test
public void testMaxAllocationsPerAMHeartbeatWithNoLimit() throws Exception {
    allocator.setMaxAllocationsPerAMHeartbeat(-1);
    List<ResourceRequest> reqs = new ArrayList<>();
    final int numContainers = 20;
    for (int i = 0; i < numContainers; i++) {
        reqs.add(createResourceRequest(i + 1, "h1", 1));
    }
    ApplicationAttemptId appAttId = ApplicationAttemptId.newInstance(ApplicationId.newInstance(0L, 1), 1);
    allocator.setNodeQueueLoadMonitor(createNodeQueueLoadMonitor(3, 2, 500));
    List<Container> containers = allocator.allocateContainers(EMPTY_BLACKLIST_REQUEST, reqs, appAttId, oppCntxt, 1L, "user");
    // all containers should be allocated in single heartbeat.
    replacedertEquals(numContainers, containers.size());
}

19 View Complete Implementation : TestDistributedOpportunisticContainerAllocator.java
Copyright Apache License 2.0
Author : apache
@Test
public void testSimpleAllocation() throws Exception {
    ResourceBlacklistRequest blacklistRequest = ResourceBlacklistRequest.newInstance(new ArrayList<>(), new ArrayList<>());
    List<ResourceRequest> reqs = Arrays.asList(ResourceRequest.newInstance(PRIORITY_NORMAL, "*", CAPABILITY_1GB, 1, true, null, OPPORTUNISTIC_REQ));
    ApplicationAttemptId appAttId = ApplicationAttemptId.newInstance(ApplicationId.newInstance(0L, 1), 1);
    oppCntxt.updateNodeList(Arrays.asList(RemoteNode.newInstance(NodeId.newInstance("h1", 1234), "h1:1234", "/r1")));
    List<Container> containers = allocator.allocateContainers(blacklistRequest, reqs, appAttId, oppCntxt, 1L, "luser");
    replacedert.replacedertEquals(1, containers.size());
    replacedert.replacedertEquals(0, oppCntxt.getOutstandingOpReqs().size());
}

19 View Complete Implementation : TestCentralizedOpportunisticContainerAllocator.java
Copyright Apache License 2.0
Author : apache
/**
 * Tests allocation of an Opportunistic container from single application.
 * @throws Exception
 */
@Test
public void testSimpleAllocation() throws Exception {
    List<ResourceRequest> reqs = Collections.singletonList(createResourceRequest(1, "*", 1));
    ApplicationAttemptId appAttId = ApplicationAttemptId.newInstance(ApplicationId.newInstance(0L, 1), 1);
    allocator.setNodeQueueLoadMonitor(createNodeQueueLoadMonitor(1, 2, 100));
    List<Container> containers = allocator.allocateContainers(EMPTY_BLACKLIST_REQUEST, reqs, appAttId, oppCntxt, 1L, "user");
    replacedertEquals(1, containers.size());
    replacedertEquals(0, oppCntxt.getOutstandingOpReqs().size());
}

19 View Complete Implementation : TestApplicationHistoryClientService.java
Copyright Apache License 2.0
Author : apache
@Test
public void testContainerReport() throws IOException, YarnException {
    ApplicationId appId = ApplicationId.newInstance(0, 1);
    ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(appId, 1);
    ContainerId containerId = ContainerId.newContainerId(appAttemptId, 1);
    GetContainerReportRequest request = GetContainerReportRequest.newInstance(containerId);
    GetContainerReportResponse response = clientService.getContainerReport(request);
    ContainerReport container = response.getContainerReport();
    replacedert.replacedertNotNull(container);
    replacedert.replacedertEquals(containerId, container.getContainerId());
    replacedert.replacedertEquals("http://0.0.0.0:8188/applicationhistory/logs/" + "test host:100/container_0_0001_01_000001/" + "container_0_0001_01_000001/user1", container.getLogUrl());
}

19 View Complete Implementation : TestCentralizedOpportunisticContainerAllocator.java
Copyright Apache License 2.0
Author : apache
/**
 * Tests that allocation of rack local Opportunistic container requests
 * should be spread out.
 * @throws Exception
 */
@Test
public void testRoundRobinRackLocalAllocation() throws Exception {
    List<ResourceRequest> reqs = Arrays.asList(createResourceRequest(1, "/r1", 1), createResourceRequest(1, "h5", 1), createResourceRequest(1, ResourceRequest.ANY, 1), createResourceRequest(2, "/r1", 1), createResourceRequest(2, "h5", 1), createResourceRequest(2, ResourceRequest.ANY, 1));
    ApplicationAttemptId appAttId = ApplicationAttemptId.newInstance(ApplicationId.newInstance(0L, 1), 1);
    NodeQueueLoadMonitor selector = createNodeQueueLoadMonitor(Arrays.asList("h1", "h2", "h3", "h4"), Arrays.asList("/r2", "/r1", "/r3", "/r1"), Arrays.asList(4, 4, 4, 4), Arrays.asList(5, 5, 5, 5));
    allocator.setNodeQueueLoadMonitor(selector);
    List<Container> containers = allocator.allocateContainers(EMPTY_BLACKLIST_REQUEST, reqs, appAttId, oppCntxt, 1L, "user");
    Set<String> allocatedHosts = new HashSet<>();
    for (Container c : containers) {
        allocatedHosts.add(c.getNodeId().toString());
    }
    LOG.info("Containers: {}", containers);
    replacedertTrue(allocatedHosts.contains("h2:1234"));
    replacedertTrue(allocatedHosts.contains("h4:1234"));
    replacedertFalse(allocatedHosts.contains("h1:1234"));
    replacedertFalse(allocatedHosts.contains("h3:1234"));
    replacedertEquals(2, containers.size());
}

19 View Complete Implementation : TestAHSClient.java
Copyright Apache License 2.0
Author : apache
@Test(timeout = 10000)
public void testGetApplicationAttempts() throws YarnException, IOException {
    Configuration conf = new Configuration();
    final AHSClient client = new MockAHSClient();
    client.init(conf);
    client.start();
    ApplicationId applicationId = ApplicationId.newInstance(1234, 5);
    List<ApplicationAttemptReport> reports = client.getApplicationAttempts(applicationId);
    replacedert.replacedertNotNull(reports);
    replacedert.replacedertEquals(reports.get(0).getApplicationAttemptId(), ApplicationAttemptId.newInstance(applicationId, 1));
    replacedert.replacedertEquals(reports.get(1).getApplicationAttemptId(), ApplicationAttemptId.newInstance(applicationId, 2));
    client.stop();
}

19 View Complete Implementation : TestNMTimelinePublisher.java
Copyright Apache License 2.0
Author : apache
@Test
public void testPublishContainerPausedEvent() {
    ApplicationId appId = ApplicationId.newInstance(0, 1);
    ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(appId, 1);
    ContainerId cId = ContainerId.newContainerId(appAttemptId, 1);
    ContainerEvent containerEvent = new ContainerPauseEvent(cId, "test pause");
    publisher.createTimelineClient(appId);
    publisher.publishContainerEvent(containerEvent);
    publisher.stopTimelineClient(appId);
    dispatcher.await();
    ContainerEnreplacedy cEnreplacedy = new ContainerEnreplacedy();
    cEnreplacedy.setId(cId.toString());
    TimelineEnreplacedy[] lastPublishedEnreplacedies = timelineClient.getLastPublishedEnreplacedies();
    replacedert.replacedertNotNull(lastPublishedEnreplacedies);
    replacedert.replacedertEquals(1, lastPublishedEnreplacedies.length);
    TimelineEnreplacedy enreplacedy = lastPublishedEnreplacedies[0];
    replacedert.replacedertEquals(cEnreplacedy, enreplacedy);
    NavigableSet<TimelineEvent> events = enreplacedy.getEvents();
    replacedert.replacedertEquals(1, events.size());
    replacedert.replacedertEquals(ContainerMetricsConstants.PAUSED_EVENT_TYPE, events.iterator().next().getId());
    Map<String, Object> info = enreplacedy.getInfo();
    replacedert.replacedertTrue(info.containsKey(ContainerMetricsConstants.DIAGNOSTICS_INFO));
    replacedert.replacedertEquals("test pause", info.get(ContainerMetricsConstants.DIAGNOSTICS_INFO));
}

19 View Complete Implementation : TestFileSystemApplicationHistoryStore.java
Copyright Apache License 2.0
Author : apache
private void testWriteHistoryData(int num, boolean missingContainer, boolean missingApplicationAttempt) throws IOException {
    // write application history data
    for (int i = 1; i <= num; ++i) {
        ApplicationId appId = ApplicationId.newInstance(0, i);
        writeApplicationStartData(appId);
        // write application attempt history data
        for (int j = 1; j <= num; ++j) {
            ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(appId, j);
            writeApplicationAttemptStartData(appAttemptId);
            if (missingApplicationAttempt && j == num) {
                continue;
            }
            // write container history data
            for (int k = 1; k <= num; ++k) {
                ContainerId containerId = ContainerId.newContainerId(appAttemptId, k);
                writeContainerStartData(containerId);
                if (missingContainer && k == num) {
                    continue;
                }
                writeContainerFinishData(containerId);
            }
            writeApplicationAttemptFinishData(appAttemptId);
        }
        writeApplicationFinishData(appId);
    }
}

19 View Complete Implementation : TestMRJobsWithHistoryService.java
Copyright Apache License 2.0
Author : apache
private void verifyJobReport(JobReport jobReport, JobId jobId) {
    List<AMInfo> amInfos = jobReport.getAMInfos();
    replacedert.replacedertEquals(1, amInfos.size());
    AMInfo amInfo = amInfos.get(0);
    ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(jobId.getAppId(), 1);
    ContainerId amContainerId = ContainerId.newContainerId(appAttemptId, 1);
    replacedert.replacedertEquals(appAttemptId, amInfo.getAppAttemptId());
    replacedert.replacedertEquals(amContainerId, amInfo.getContainerId());
    replacedert.replacedertTrue(jobReport.getSubmitTime() > 0);
    replacedert.replacedertTrue(jobReport.getStartTime() > 0 && jobReport.getStartTime() >= jobReport.getSubmitTime());
    replacedert.replacedertTrue(jobReport.getFinishTime() > 0 && jobReport.getFinishTime() >= jobReport.getStartTime());
}

19 View Complete Implementation : TestQueueState.java
Copyright Apache License 2.0
Author : apache
private FiCaSchedulerApp getMockApplication(ApplicationId appId, String user, Resource amResource) {
    FiCaSchedulerApp application = mock(FiCaSchedulerApp.clreplaced);
    ApplicationAttemptId applicationAttemptId = ApplicationAttemptId.newInstance(appId, 0);
    doReturn(applicationAttemptId.getApplicationId()).when(application).getApplicationId();
    doReturn(applicationAttemptId).when(application).getApplicationAttemptId();
    doReturn(user).when(application).getUser();
    doReturn(amResource).when(application).getAMResource();
    doReturn(Priority.newInstance(0)).when(application).getPriority();
    doReturn(CommonNodeLabelsManager.NO_LABEL).when(application).getAppAMNodeParreplacedionName();
    doReturn(amResource).when(application).getAMResource(CommonNodeLabelsManager.NO_LABEL);
    when(application.compareInputOrderTo(any(FiCaSchedulerApp.clreplaced))).thenCallRealMethod();
    return application;
}

19 View Complete Implementation : TestAHSClient.java
Copyright Apache License 2.0
Author : apache
@Test(timeout = 10000)
public void testGetApplicationAttempt() throws YarnException, IOException {
    Configuration conf = new Configuration();
    final AHSClient client = new MockAHSClient();
    client.init(conf);
    client.start();
    List<ApplicationReport> expectedReports = ((MockAHSClient) client).getReports();
    ApplicationId applicationId = ApplicationId.newInstance(1234, 5);
    ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(applicationId, 1);
    ApplicationAttemptReport report = client.getApplicationAttemptReport(appAttemptId);
    replacedert.replacedertNotNull(report);
    replacedert.replacedertEquals(report.getApplicationAttemptId().toString(), expectedReports.get(0).getCurrentApplicationAttemptId().toString());
    client.stop();
}

19 View Complete Implementation : TestDistributedOpportunisticContainerAllocator.java
Copyright Apache License 2.0
Author : apache
@Test
public void testRoundRobinRackLocalAllocation() throws Exception {
    ResourceBlacklistRequest blacklistRequest = ResourceBlacklistRequest.newInstance(new ArrayList<>(), new ArrayList<>());
    List<ResourceRequest> reqs = Arrays.asList(ResourceRequest.newBuilder().allocationRequestId(1).priority(PRIORITY_NORMAL).resourceName("/r1").capability(CAPABILITY_1GB).relaxLocality(true).executionType(ExecutionType.OPPORTUNISTIC).build(), ResourceRequest.newBuilder().allocationRequestId(1).priority(PRIORITY_NORMAL).resourceName("h1").capability(CAPABILITY_1GB).relaxLocality(true).executionType(ExecutionType.OPPORTUNISTIC).build(), ResourceRequest.newBuilder().allocationRequestId(1).priority(PRIORITY_NORMAL).resourceName(ResourceRequest.ANY).capability(CAPABILITY_1GB).relaxLocality(true).executionType(ExecutionType.OPPORTUNISTIC).build(), ResourceRequest.newBuilder().allocationRequestId(2).priority(PRIORITY_NORMAL).resourceName("/r1").capability(CAPABILITY_1GB).relaxLocality(true).executionType(ExecutionType.OPPORTUNISTIC).build(), ResourceRequest.newBuilder().allocationRequestId(2).priority(PRIORITY_NORMAL).resourceName("h1").capability(CAPABILITY_1GB).relaxLocality(true).executionType(ExecutionType.OPPORTUNISTIC).build(), ResourceRequest.newBuilder().allocationRequestId(2).priority(PRIORITY_NORMAL).resourceName(ResourceRequest.ANY).capability(CAPABILITY_1GB).relaxLocality(true).executionType(ExecutionType.OPPORTUNISTIC).build());
    ApplicationAttemptId appAttId = ApplicationAttemptId.newInstance(ApplicationId.newInstance(0L, 1), 1);
    oppCntxt.updateNodeList(Arrays.asList(RemoteNode.newInstance(NodeId.newInstance("h3", 1234), "h3:1234", "/r2"), RemoteNode.newInstance(NodeId.newInstance("h2", 1234), "h2:1234", "/r1"), RemoteNode.newInstance(NodeId.newInstance("h5", 1234), "h5:1234", "/r1"), RemoteNode.newInstance(NodeId.newInstance("h4", 1234), "h4:1234", "/r2")));
    List<Container> containers = allocator.allocateContainers(blacklistRequest, reqs, appAttId, oppCntxt, 1L, "luser");
    Set<String> allocatedHosts = new HashSet<>();
    for (Container c : containers) {
        allocatedHosts.add(c.getNodeHttpAddress());
    }
    LOG.info("Containers: {}", containers);
    replacedert.replacedertTrue(allocatedHosts.contains("h2:1234"));
    replacedert.replacedertTrue(allocatedHosts.contains("h5:1234"));
    replacedert.replacedertFalse(allocatedHosts.contains("h3:1234"));
    replacedert.replacedertFalse(allocatedHosts.contains("h4:1234"));
    replacedert.replacedertEquals(2, containers.size());
}

19 View Complete Implementation : TestCentralizedOpportunisticContainerAllocator.java
Copyright Apache License 2.0
Author : apache
/**
 * Tests scheduling of many rack local Opportunistic container requests.
 * @throws Exception
 */
@Test
public void testLotsOfContainersRackLocalAllocation() throws Exception {
    List<ResourceRequest> reqs = new ArrayList<>();
    // add 100 container requests.
    for (int i = 0; i < 100; i++) {
        reqs.add(createResourceRequest(i + 1, ResourceRequest.ANY, 1));
        reqs.add(createResourceRequest(i + 1, "h5", 1));
        reqs.add(createResourceRequest(i + 1, "/r1", 1));
    }
    ApplicationAttemptId appAttId = ApplicationAttemptId.newInstance(ApplicationId.newInstance(0L, 1), 1);
    NodeQueueLoadMonitor selector = createNodeQueueLoadMonitor(Arrays.asList("h1", "h2", "h3", "h4"), Arrays.asList("/r1", "/r1", "/r1", "/r2"), Arrays.asList(0, 0, 0, 0), Arrays.asList(500, 500, 500, 300));
    allocator.setNodeQueueLoadMonitor(selector);
    List<Container> containers = new ArrayList<>();
    containers = allocator.allocateContainers(EMPTY_BLACKLIST_REQUEST, reqs, appAttId, oppCntxt, 1L, "user");
    replacedertEquals(100, containers.size());
}

19 View Complete Implementation : TestQueueStateManager.java
Copyright Apache License 2.0
Author : apache
private FiCaSchedulerApp getMockApplication(ApplicationId appId, String user, Resource amResource) {
    FiCaSchedulerApp application = mock(FiCaSchedulerApp.clreplaced);
    ApplicationAttemptId applicationAttemptId = ApplicationAttemptId.newInstance(appId, 0);
    doReturn(applicationAttemptId.getApplicationId()).when(application).getApplicationId();
    doReturn(applicationAttemptId).when(application).getApplicationAttemptId();
    doReturn(user).when(application).getUser();
    doReturn(amResource).when(application).getAMResource();
    doReturn(Priority.newInstance(0)).when(application).getPriority();
    doReturn(CommonNodeLabelsManager.NO_LABEL).when(application).getAppAMNodeParreplacedionName();
    doReturn(amResource).when(application).getAMResource(CommonNodeLabelsManager.NO_LABEL);
    when(application.compareInputOrderTo(any(FiCaSchedulerApp.clreplaced))).thenCallRealMethod();
    return application;
}

19 View Complete Implementation : TestDistributedOpportunisticContainerAllocator.java
Copyright Apache License 2.0
Author : apache
/**
 * Tests maximum opportunistic container allocation per AM heartbeat for
 * allocation requests with different scheduler key.
 * @throws Exception
 */
@Test
public void testMaxAllocationsPerAMHeartbeatDifferentSchedKey() throws Exception {
    ResourceBlacklistRequest blacklistRequest = ResourceBlacklistRequest.newInstance(new ArrayList<>(), new ArrayList<>());
    allocator.setMaxAllocationsPerAMHeartbeat(2);
    final ExecutionTypeRequest oppRequest = ExecutionTypeRequest.newInstance(ExecutionType.OPPORTUNISTIC, true);
    List<ResourceRequest> reqs = Arrays.asList(ResourceRequest.newInstance(Priority.newInstance(1), "*", CAPABILITY_1GB, 1, true, null, OPPORTUNISTIC_REQ), ResourceRequest.newInstance(Priority.newInstance(2), "h6", CAPABILITY_1GB, 2, true, null, OPPORTUNISTIC_REQ), ResourceRequest.newInstance(Priority.newInstance(3), "/r3", CAPABILITY_1GB, 2, true, null, OPPORTUNISTIC_REQ));
    ApplicationAttemptId appAttId = ApplicationAttemptId.newInstance(ApplicationId.newInstance(0L, 1), 1);
    oppCntxt.updateNodeList(Arrays.asList(RemoteNode.newInstance(NodeId.newInstance("h3", 1234), "h3:1234", "/r2"), RemoteNode.newInstance(NodeId.newInstance("h2", 1234), "h2:1234", "/r1"), RemoteNode.newInstance(NodeId.newInstance("h5", 1234), "h5:1234", "/r1"), RemoteNode.newInstance(NodeId.newInstance("h4", 1234), "h4:1234", "/r2")));
    List<Container> containers = allocator.allocateContainers(blacklistRequest, reqs, appAttId, oppCntxt, 1L, "user1");
    LOG.info("Containers: {}", containers);
    // Although capacity is present, but only 2 containers should be allocated
    // as max allocation per AM heartbeat is set to 2.
    replacedert.replacedertEquals(2, containers.size());
    containers = allocator.allocateContainers(blacklistRequest, new ArrayList<>(), appAttId, oppCntxt, 1L, "user1");
    LOG.info("Containers: {}", containers);
    // 2 more containers should be allocated from pending allocation requests.
    replacedert.replacedertEquals(2, containers.size());
    containers = allocator.allocateContainers(blacklistRequest, new ArrayList<>(), appAttId, oppCntxt, 1L, "user1");
    LOG.info("Containers: {}", containers);
    // Remaining 1 container should be allocated.
    replacedert.replacedertEquals(1, containers.size());
}

19 View Complete Implementation : TestDistributedOpportunisticContainerAllocator.java
Copyright Apache License 2.0
Author : apache
@Test
public void testLotsOfContainersRackLocalAllocationSameSchedKey() throws Exception {
    ResourceBlacklistRequest blacklistRequest = ResourceBlacklistRequest.newInstance(new ArrayList<>(), new ArrayList<>());
    List<ResourceRequest> reqs = Arrays.asList(ResourceRequest.newInstance(PRIORITY_NORMAL, "*", CAPABILITY_1GB, 1000, true, null, OPPORTUNISTIC_REQ), ResourceRequest.newInstance(PRIORITY_NORMAL, "h1", CAPABILITY_1GB, 1000, true, null, OPPORTUNISTIC_REQ), ResourceRequest.newInstance(PRIORITY_NORMAL, "/r1", CAPABILITY_1GB, 1000, true, null, OPPORTUNISTIC_REQ));
    ApplicationAttemptId appAttId = ApplicationAttemptId.newInstance(ApplicationId.newInstance(0L, 1), 1);
    oppCntxt.updateNodeList(Arrays.asList(RemoteNode.newInstance(NodeId.newInstance("h3", 1234), "h3:1234", "/r2"), RemoteNode.newInstance(NodeId.newInstance("h2", 1234), "h2:1234", "/r1"), RemoteNode.newInstance(NodeId.newInstance("h5", 1234), "h5:1234", "/r1"), RemoteNode.newInstance(NodeId.newInstance("h4", 1234), "h4:1234", "/r2")));
    List<Container> containers = new ArrayList<>();
    for (int i = 0; i < 250; i++) {
        containers.addAll(allocator.allocateContainers(blacklistRequest, reqs, appAttId, oppCntxt, 1L, "luser"));
    }
    replacedert.replacedertEquals(1000, containers.size());
}

19 View Complete Implementation : TestNMProxy.java
Copyright Apache License 2.0
Author : apache
private ContainerManagementProtocol getNMProxy(Configuration conf) {
    ApplicationId appId = ApplicationId.newInstance(1, 1);
    ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(appId, 1);
    org.apache.hadoop.yarn.api.records.Token nmToken = context.getNMTokenSecretManager().createNMToken(attemptId, context.getNodeId(), user);
    final InetSocketAddress address = conf.getSocketAddr(YarnConfiguration.NM_BIND_HOST, YarnConfiguration.NM_ADDRESS, YarnConfiguration.DEFAULT_NM_ADDRESS, YarnConfiguration.DEFAULT_NM_PORT);
    Token<NMTokenIdentifier> token = ConverterUtils.convertFromYarn(nmToken, SecurityUtil.buildTokenService(address));
    UserGroupInformation ugi = UserGroupInformation.createRemoteUser(user);
    ugi.addToken(token);
    return NMProxy.createNMProxy(conf, ContainerManagementProtocol.clreplaced, ugi, YarnRPC.create(conf), address);
}

19 View Complete Implementation : TestLocalContainerAllocator.java
Copyright Apache License 2.0
Author : apache
@Test
public void testAMRMTokenUpdate() throws Exception {
    Configuration conf = new Configuration();
    ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(ApplicationId.newInstance(1, 1), 1);
    AMRMTokenIdentifier oldTokenId = new AMRMTokenIdentifier(attemptId, 1);
    AMRMTokenIdentifier newTokenId = new AMRMTokenIdentifier(attemptId, 2);
    Token<AMRMTokenIdentifier> oldToken = new Token<AMRMTokenIdentifier>(oldTokenId.getBytes(), "oldpreplacedword".getBytes(), oldTokenId.getKind(), new Text());
    Token<AMRMTokenIdentifier> newToken = new Token<AMRMTokenIdentifier>(newTokenId.getBytes(), "newpreplacedword".getBytes(), newTokenId.getKind(), new Text());
    MockScheduler scheduler = new MockScheduler();
    scheduler.amToken = newToken;
    final LocalContainerAllocator lca = new StubbedLocalContainerAllocator(scheduler);
    lca.init(conf);
    lca.start();
    UserGroupInformation testUgi = UserGroupInformation.createUserForTesting("someuser", new String[0]);
    testUgi.addToken(oldToken);
    testUgi.doAs(new PrivilegedExceptionAction<Void>() {

        @Override
        public Void run() throws Exception {
            lca.heartbeat();
            return null;
        }
    });
    lca.close();
    // verify there is only one AMRM token in the UGI and it matches the
    // updated token from the RM
    int tokenCount = 0;
    Token<? extends TokenIdentifier> ugiToken = null;
    for (Token<? extends TokenIdentifier> token : testUgi.getTokens()) {
        if (AMRMTokenIdentifier.KIND_NAME.equals(token.getKind())) {
            ugiToken = token;
            ++tokenCount;
        }
    }
    replacedert.replacedertEquals("too many AMRM tokens", 1, tokenCount);
    replacedert.replacedertArrayEquals("token identifier not updated", newToken.getIdentifier(), ugiToken.getIdentifier());
    replacedert.replacedertArrayEquals("token preplacedword not updated", newToken.getPreplacedword(), ugiToken.getPreplacedword());
    replacedert.replacedertEquals("AMRM token service not updated", new Text(ClientRMProxy.getAMRMTokenService(conf)), ugiToken.getService());
}

19 View Complete Implementation : TestDistributedScheduler.java
Copyright Apache License 2.0
Author : apache
private RequestInterceptor setup(Configuration conf, DistributedScheduler distributedScheduler) {
    NodeStatusUpdater nodeStatusUpdater = Mockito.mock(NodeStatusUpdater.clreplaced);
    Mockito.when(nodeStatusUpdater.getRMIdentifier()).thenReturn(12345l);
    NMContainerTokenSecretManager nmContainerTokenSecretManager = new NMContainerTokenSecretManager(conf);
    MasterKey mKey = new MasterKey() {

        @Override
        public int getKeyId() {
            return 1;
        }

        @Override
        public void setKeyId(int keyId) {
        }

        @Override
        public ByteBuffer getBytes() {
            return ByteBuffer.allocate(8);
        }

        @Override
        public void setBytes(ByteBuffer bytes) {
        }
    };
    nmContainerTokenSecretManager.setMasterKey(mKey);
    OpportunisticContainerAllocator containerAllocator = new DistributedOpportunisticContainerAllocator(nmContainerTokenSecretManager);
    NMTokenSecretManagerInNM nmTokenSecretManagerInNM = new NMTokenSecretManagerInNM();
    nmTokenSecretManagerInNM.setMasterKey(mKey);
    distributedScheduler.initLocal(1234, ApplicationAttemptId.newInstance(ApplicationId.newInstance(1, 1), 1), containerAllocator, nmTokenSecretManagerInNM, "test");
    RequestInterceptor finalReqIntcptr = Mockito.mock(RequestInterceptor.clreplaced);
    distributedScheduler.setNextInterceptor(finalReqIntcptr);
    return finalReqIntcptr;
}

19 View Complete Implementation : TestCentralizedOpportunisticContainerAllocator.java
Copyright Apache License 2.0
Author : apache
/**
 * Tests allocation of node local Opportunistic container requests.
 * @throws Exception
 */
@Test
public void testNodeLocalAllocation() throws Exception {
    List<ResourceRequest> reqs = Arrays.asList(createResourceRequest(1, ResourceRequest.ANY, 1), createResourceRequest(2, "/r1", 1), createResourceRequest(2, "h1", 1), createResourceRequest(2, ResourceRequest.ANY, 1), createResourceRequest(3, "/r1", 1), createResourceRequest(3, "h1", 1), createResourceRequest(3, ResourceRequest.ANY, 1));
    ApplicationAttemptId appAttId = ApplicationAttemptId.newInstance(ApplicationId.newInstance(0L, 1), 1);
    allocator.setNodeQueueLoadMonitor(createNodeQueueLoadMonitor(3, 2, 5));
    List<Container> containers = allocator.allocateContainers(EMPTY_BLACKLIST_REQUEST, reqs, appAttId, oppCntxt, 1L, "user");
    LOG.info("Containers: {}", containers);
    // all 3 containers should be allocated.
    replacedertEquals(3, containers.size());
    // container with allocation id 2 and 3 should be allocated on node h1
    for (Container c : containers) {
        if (c.getAllocationRequestId() == 2 || c.getAllocationRequestId() == 3) {
            replacedertEquals("h1:1234", c.getNodeId().toString());
        }
    }
}

19 View Complete Implementation : TestCentralizedOpportunisticContainerAllocator.java
Copyright Apache License 2.0
Author : apache
/**
 * Tests off switch allocation of Opportunistic containers.
 * @throws Exception
 */
@Test
public void testOffSwitchAllocationWhenNoNodeOrRack() throws Exception {
    List<ResourceRequest> reqs = Arrays.asList(createResourceRequest(2, "/r3", 2), createResourceRequest(2, "h6", 2), createResourceRequest(2, ResourceRequest.ANY, 2));
    ApplicationAttemptId appAttId = ApplicationAttemptId.newInstance(ApplicationId.newInstance(0L, 1), 1);
    NodeQueueLoadMonitor selector = createNodeQueueLoadMonitor(Arrays.asList("h1", "h2", "h3", "h4"), Arrays.asList("/r2", "/r1", "/r2", "/r1"), Arrays.asList(4, 4, 4, 4), Arrays.asList(5, 5, 5, 5));
    allocator.setNodeQueueLoadMonitor(selector);
    List<Container> containers = allocator.allocateContainers(EMPTY_BLACKLIST_REQUEST, reqs, appAttId, oppCntxt, 1L, "user");
    LOG.info("Containers: {}", containers);
    replacedertEquals(2, containers.size());
}

19 View Complete Implementation : TestProtocolRecords.java
Copyright Apache License 2.0
Author : apache
@Test
public void testRegisterNodeManagerRequest() {
    ApplicationId appId = ApplicationId.newInstance(123456789, 1);
    ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(appId, 1);
    ContainerId containerId = ContainerId.newContainerId(attemptId, 1);
    NMContainerStatus containerReport = NMContainerStatus.newInstance(containerId, 0, ContainerState.RUNNING, Resource.newInstance(1024, 1), "diagnostics", 0, Priority.newInstance(10), 1234);
    List<NMContainerStatus> reports = Arrays.asList(containerReport);
    RegisterNodeManagerRequest request = RegisterNodeManagerRequest.newInstance(NodeId.newInstance("1.1.1.1", 1000), 8080, Resource.newInstance(1024, 1), "NM-version-id", reports, Arrays.asList(appId));
    RegisterNodeManagerRequest requestProto = new RegisterNodeManagerRequestPBImpl(((RegisterNodeManagerRequestPBImpl) request).getProto());
    replacedert.replacedertEquals(containerReport, requestProto.getNMContainerStatuses().get(0));
    replacedert.replacedertEquals(8080, requestProto.getHttpPort());
    replacedert.replacedertEquals("NM-version-id", requestProto.getNMVersion());
    replacedert.replacedertEquals(NodeId.newInstance("1.1.1.1", 1000), requestProto.getNodeId());
    replacedert.replacedertEquals(Resource.newInstance(1024, 1), requestProto.getResource());
    replacedert.replacedertEquals(1, requestProto.getRunningApplications().size());
    replacedert.replacedertEquals(appId, requestProto.getRunningApplications().get(0));
}

19 View Complete Implementation : TestProtocolRecords.java
Copyright Apache License 2.0
Author : apache
@Test
public void testNMContainerStatus() {
    ApplicationId appId = ApplicationId.newInstance(123456789, 1);
    ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(appId, 1);
    ContainerId containerId = ContainerId.newContainerId(attemptId, 1);
    Resource resource = Resource.newInstance(1000, 200);
    NMContainerStatus report = NMContainerStatus.newInstance(containerId, 0, ContainerState.COMPLETE, resource, "diagnostics", ContainerExitStatus.ABORTED, Priority.newInstance(10), 1234);
    NMContainerStatus reportProto = new NMContainerStatusPBImpl(((NMContainerStatusPBImpl) report).getProto());
    replacedert.replacedertEquals("diagnostics", reportProto.getDiagnostics());
    replacedert.replacedertEquals(resource, reportProto.getAllocatedResource());
    replacedert.replacedertEquals(ContainerExitStatus.ABORTED, reportProto.getContainerExitStatus());
    replacedert.replacedertEquals(ContainerState.COMPLETE, reportProto.getContainerState());
    replacedert.replacedertEquals(containerId, reportProto.getContainerId());
    replacedert.replacedertEquals(Priority.newInstance(10), reportProto.getPriority());
    replacedert.replacedertEquals(1234, reportProto.getCreationTime());
}

19 View Complete Implementation : TestDistributedOpportunisticContainerAllocator.java
Copyright Apache License 2.0
Author : apache
@Test
public void testBlacklistRejection() throws Exception {
    ResourceBlacklistRequest blacklistRequest = ResourceBlacklistRequest.newInstance(Arrays.asList("h1", "h2"), new ArrayList<>());
    List<ResourceRequest> reqs = Arrays.asList(ResourceRequest.newInstance(PRIORITY_NORMAL, "*", CAPABILITY_1GB, 1, true, null, OPPORTUNISTIC_REQ));
    ApplicationAttemptId appAttId = ApplicationAttemptId.newInstance(ApplicationId.newInstance(0L, 1), 1);
    oppCntxt.updateNodeList(Arrays.asList(RemoteNode.newInstance(NodeId.newInstance("h1", 1234), "h1:1234", "/r1"), RemoteNode.newInstance(NodeId.newInstance("h2", 1234), "h2:1234", "/r2")));
    List<Container> containers = allocator.allocateContainers(blacklistRequest, reqs, appAttId, oppCntxt, 1L, "luser");
    replacedert.replacedertEquals(0, containers.size());
    replacedert.replacedertEquals(1, oppCntxt.getOutstandingOpReqs().size());
}

19 View Complete Implementation : TestApplicationHistoryManagerImpl.java
Copyright Apache License 2.0
Author : apache
@Test
public void testApplicationReport() throws IOException, YarnException {
    ApplicationId appId = null;
    appId = ApplicationId.newInstance(0, 1);
    writeApplicationStartData(appId);
    writeApplicationFinishData(appId);
    ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(appId, 1);
    writeApplicationAttemptStartData(appAttemptId);
    writeApplicationAttemptFinishData(appAttemptId);
    ApplicationReport appReport = applicationHistoryManagerImpl.getApplication(appId);
    replacedert.replacedertNotNull(appReport);
    replacedert.replacedertEquals(appId, appReport.getApplicationId());
    replacedert.replacedertEquals(appAttemptId, appReport.getCurrentApplicationAttemptId());
    replacedert.replacedertEquals(appAttemptId.toString(), appReport.getHost());
    replacedert.replacedertEquals("test type", appReport.getApplicationType().toString());
    replacedert.replacedertEquals("test queue", appReport.getQueue().toString());
}

19 View Complete Implementation : TestNMTimelinePublisher.java
Copyright Apache License 2.0
Author : apache
@Test
public void testContainerResourceUsage() {
    ApplicationId appId = ApplicationId.newInstance(0, 1);
    publisher.createTimelineClient(appId);
    Container aContainer = mock(Container.clreplaced);
    when(aContainer.getContainerId()).thenReturn(ContainerId.newContainerId(ApplicationAttemptId.newInstance(appId, 1), 0L));
    long idPrefix = TimelineServiceHelper.invertLong(aContainer.getContainerId().getContainerId());
    publisher.reportContainerResourceUsage(aContainer, 1024L, 8F);
    verifyPublishedResourceUsageMetrics(timelineClient, 1024L, 8, idPrefix);
    timelineClient.reset();
    publisher.reportContainerResourceUsage(aContainer, 1024L, 0.8F);
    verifyPublishedResourceUsageMetrics(timelineClient, 1024L, 1, idPrefix);
    timelineClient.reset();
    publisher.reportContainerResourceUsage(aContainer, 1024L, 0.49F);
    verifyPublishedResourceUsageMetrics(timelineClient, 1024L, 0, idPrefix);
    timelineClient.reset();
    publisher.reportContainerResourceUsage(aContainer, 1024L, (float) ResourceCalculatorProcessTree.UNAVAILABLE);
    verifyPublishedResourceUsageMetrics(timelineClient, 1024L, ResourceCalculatorProcessTree.UNAVAILABLE, idPrefix);
}

19 View Complete Implementation : TestAHSWebApp.java
Copyright Apache License 2.0
Author : apache
ApplicationHistoryClientService mockApplicationHistoryClientService(int numApps, int numAppAttempts, int numContainers) throws Exception {
    ApplicationHistoryManager ahManager = new MockApplicationHistoryManagerImpl(store);
    ApplicationHistoryClientService historyClientService = new ApplicationHistoryClientService(ahManager);
    for (int i = 1; i <= numApps; ++i) {
        ApplicationId appId = ApplicationId.newInstance(0, i);
        writeApplicationStartData(appId);
        for (int j = 1; j <= numAppAttempts; ++j) {
            ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(appId, j);
            writeApplicationAttemptStartData(appAttemptId);
            for (int k = 1; k <= numContainers; ++k) {
                ContainerId containerId = ContainerId.newContainerId(appAttemptId, k);
                writeContainerStartData(containerId);
                writeContainerFinishData(containerId);
            }
            writeApplicationAttemptFinishData(appAttemptId);
        }
        writeApplicationFinishData(appId);
    }
    return historyClientService;
}

19 View Complete Implementation : TestNMTimelinePublisher.java
Copyright Apache License 2.0
Author : apache
@Test
public void testPublishContainerKilledEvent() {
    ApplicationId appId = ApplicationId.newInstance(0, 1);
    ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(appId, 1);
    ContainerId cId = ContainerId.newContainerId(appAttemptId, 1);
    ContainerEvent containerEvent = new ContainerKillEvent(cId, 1, "test kill");
    publisher.createTimelineClient(appId);
    publisher.publishContainerEvent(containerEvent);
    publisher.stopTimelineClient(appId);
    dispatcher.await();
    ContainerEnreplacedy cEnreplacedy = new ContainerEnreplacedy();
    cEnreplacedy.setId(cId.toString());
    TimelineEnreplacedy[] lastPublishedEnreplacedies = timelineClient.getLastPublishedEnreplacedies();
    replacedert.replacedertNotNull(lastPublishedEnreplacedies);
    replacedert.replacedertEquals(1, lastPublishedEnreplacedies.length);
    TimelineEnreplacedy enreplacedy = lastPublishedEnreplacedies[0];
    replacedert.replacedertEquals(cEnreplacedy, enreplacedy);
    NavigableSet<TimelineEvent> events = enreplacedy.getEvents();
    replacedert.replacedertEquals(1, events.size());
    replacedert.replacedertEquals(ContainerMetricsConstants.KILLED_EVENT_TYPE, events.iterator().next().getId());
    Map<String, Object> info = enreplacedy.getInfo();
    replacedert.replacedertTrue(info.containsKey(ContainerMetricsConstants.DIAGNOSTICS_INFO));
    replacedert.replacedertEquals("test kill", info.get(ContainerMetricsConstants.DIAGNOSTICS_INFO));
    replacedert.replacedertTrue(info.containsKey(ContainerMetricsConstants.EXIT_STATUS_INFO));
    replacedert.replacedertEquals(1, info.get(ContainerMetricsConstants.EXIT_STATUS_INFO));
}

19 View Complete Implementation : TestCentralizedOpportunisticContainerAllocator.java
Copyright Apache License 2.0
Author : apache
/**
 * Tests maximum opportunistic container allocation per AM heartbeat for
 * allocation requests with different scheduler key.
 * @throws Exception
 */
@Test
public void testMaxAllocationsPerAMHeartbeatDifferentSchedKey() throws Exception {
    allocator.setMaxAllocationsPerAMHeartbeat(2);
    List<ResourceRequest> reqs = Arrays.asList(createResourceRequest(1, ResourceRequest.ANY, 1), createResourceRequest(2, "h6", 2), createResourceRequest(3, "/r3", 2));
    ApplicationAttemptId appAttId = ApplicationAttemptId.newInstance(ApplicationId.newInstance(0L, 1), 1);
    allocator.setNodeQueueLoadMonitor(createNodeQueueLoadMonitor(3, 2, 5));
    List<Container> containers = allocator.allocateContainers(EMPTY_BLACKLIST_REQUEST, reqs, appAttId, oppCntxt, 1L, "user");
    LOG.info("Containers: {}", containers);
    // Although capacity is present, but only 2 containers should be allocated
    // as max allocation per AM heartbeat is set to 2.
    replacedertEquals(2, containers.size());
    containers = allocator.allocateContainers(EMPTY_BLACKLIST_REQUEST, new ArrayList<>(), appAttId, oppCntxt, 1L, "user");
    LOG.info("Containers: {}", containers);
    // 2 more containers should be allocated from pending allocation requests.
    replacedertEquals(2, containers.size());
    containers = allocator.allocateContainers(EMPTY_BLACKLIST_REQUEST, new ArrayList<>(), appAttId, oppCntxt, 1L, "user");
    LOG.info("Containers: {}", containers);
    // Remaining 1 container should be allocated.
    replacedertEquals(1, containers.size());
}

19 View Complete Implementation : TestAHSv2ClientImpl.java
Copyright Apache License 2.0
Author : apache
@Test
public void testGetAppAttemptReport() throws IOException, YarnException {
    final ApplicationId appId = ApplicationId.newInstance(0, 1);
    final ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(appId, 1);
    when(spyTimelineReaderClient.getApplicationAttemptEnreplacedy(appAttemptId, "ALL", null)).thenReturn(createAppAttemptTimelineEnreplacedy(appAttemptId));
    ApplicationAttemptReport report = client.getApplicationAttemptReport(appAttemptId);
    replacedertThat(report.getApplicationAttemptId()).isEqualTo(appAttemptId);
    replacedertThat(report.getFinishTime()).isEqualTo(Integer.MAX_VALUE + 2L);
    replacedertThat(report.getOriginalTrackingUrl()).isEqualTo("test original tracking url");
}

19 View Complete Implementation : TestDistributedOpportunisticContainerAllocator.java
Copyright Apache License 2.0
Author : apache
@Test
public void testSimpleRackLocalAllocation() throws Exception {
    ResourceBlacklistRequest blacklistRequest = ResourceBlacklistRequest.newInstance(new ArrayList<>(), new ArrayList<>());
    List<ResourceRequest> reqs = Arrays.asList(ResourceRequest.newInstance(PRIORITY_NORMAL, "*", CAPABILITY_1GB, 1, true, null, OPPORTUNISTIC_REQ), ResourceRequest.newInstance(PRIORITY_NORMAL, "h1", CAPABILITY_1GB, 1, true, null, OPPORTUNISTIC_REQ), ResourceRequest.newInstance(PRIORITY_NORMAL, "/r1", CAPABILITY_1GB, 1, true, null, OPPORTUNISTIC_REQ));
    ApplicationAttemptId appAttId = ApplicationAttemptId.newInstance(ApplicationId.newInstance(0L, 1), 1);
    oppCntxt.updateNodeList(Arrays.asList(RemoteNode.newInstance(NodeId.newInstance("h3", 1234), "h3:1234", "/r2"), RemoteNode.newInstance(NodeId.newInstance("h2", 1234), "h2:1234", "/r1"), RemoteNode.newInstance(NodeId.newInstance("h4", 1234), "h4:1234", "/r2")));
    List<Container> containers = allocator.allocateContainers(blacklistRequest, reqs, appAttId, oppCntxt, 1L, "luser");
    Set<String> allocatedHosts = new HashSet<>();
    for (Container c : containers) {
        allocatedHosts.add(c.getNodeHttpAddress());
    }
    replacedert.replacedertTrue(allocatedHosts.contains("h2:1234"));
    replacedert.replacedertFalse(allocatedHosts.contains("h3:1234"));
    replacedert.replacedertFalse(allocatedHosts.contains("h4:1234"));
    replacedert.replacedertEquals(1, containers.size());
}

19 View Complete Implementation : TestFileSystemApplicationHistoryStore.java
Copyright Apache License 2.0
Author : apache
@Test
public void testMreplacediveWriteContainerHistoryData() throws IOException {
    LOG.info("Starting testMreplacediveWriteContainerHistoryData");
    long mb = 1024 * 1024;
    long usedDiskBefore = fs.getContentSummary(fsWorkingPath).getLength() / mb;
    ApplicationId appId = ApplicationId.newInstance(0, 1);
    writeApplicationStartData(appId);
    ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(appId, 1);
    for (int i = 1; i <= 100000; ++i) {
        ContainerId containerId = ContainerId.newContainerId(appAttemptId, i);
        writeContainerStartData(containerId);
        writeContainerFinishData(containerId);
    }
    writeApplicationFinishData(appId);
    long usedDiskAfter = fs.getContentSummary(fsWorkingPath).getLength() / mb;
    replacedert.replacedertTrue((usedDiskAfter - usedDiskBefore) < 20);
}

19 View Complete Implementation : TestRegisterNodeManagerRequest.java
Copyright Apache License 2.0
Author : apache
@Test
public void testRegisterNodeManagerRequest() {
    RegisterNodeManagerRequest request = RegisterNodeManagerRequest.newInstance(NodeId.newInstance("host", 1234), 1234, Resource.newInstance(0, 0), "version", Arrays.asList(NMContainerStatus.newInstance(ContainerId.newContainerId(ApplicationAttemptId.newInstance(ApplicationId.newInstance(1234L, 1), 1), 1), 0, ContainerState.RUNNING, Resource.newInstance(1024, 1), "good", -1, Priority.newInstance(0), 1234)), Arrays.asList(ApplicationId.newInstance(1234L, 1), ApplicationId.newInstance(1234L, 2)));
    // serialze to proto, and get request from proto
    RegisterNodeManagerRequest request1 = new RegisterNodeManagerRequestPBImpl(((RegisterNodeManagerRequestPBImpl) request).getProto());
    // check values
    replacedert.replacedertEquals(request1.getNMContainerStatuses().size(), request.getNMContainerStatuses().size());
    replacedert.replacedertEquals(request1.getNMContainerStatuses().get(0).getContainerId(), request.getNMContainerStatuses().get(0).getContainerId());
    replacedert.replacedertEquals(request1.getRunningApplications().size(), request.getRunningApplications().size());
    replacedert.replacedertEquals(request1.getRunningApplications().get(0), request.getRunningApplications().get(0));
    replacedert.replacedertEquals(request1.getRunningApplications().get(1), request.getRunningApplications().get(1));
}

19 View Complete Implementation : TestPlacementConstraintsUtil.java
Copyright Apache License 2.0
Author : apache
@Test
public void testRackAntiAffinityreplacedignment() throws InvalidAllocationTagsQueryException {
    AllocationTagsManager tm = new AllocationTagsManager(rmContext);
    PlacementConstraintManagerService pcm = new MemoryPlacementConstraintManager();
    // Register App1 with anti-affinity constraint map
    pcm.registerApplication(appId1, constraintMap2);
    /**
     * Place container:
     * Node0:123 (Rack1):
     *    container_app1_1 (hbase-rs)
     */
    RMNode n0_r1 = rmNodes.get(0);
    RMNode n1_r1 = rmNodes.get(1);
    RMNode n2_r2 = rmNodes.get(2);
    RMNode n3_r2 = rmNodes.get(3);
    // 1 Containers on Node0-Rack1 with allocationTag 'hbase-rs'
    ContainerId hbase_m = ContainerId.newContainerId(ApplicationAttemptId.newInstance(appId1, 0), 0);
    tm.addContainer(n0_r1.getNodeID(), hbase_m, ImmutableSet.of("hbase-rs"));
    SchedulerNode schedulerNode0 = newSchedulerNode(n0_r1.getHostName(), n0_r1.getRackName(), n0_r1.getNodeID());
    SchedulerNode schedulerNode1 = newSchedulerNode(n1_r1.getHostName(), n1_r1.getRackName(), n1_r1.getNodeID());
    SchedulerNode schedulerNode2 = newSchedulerNode(n2_r2.getHostName(), n2_r2.getRackName(), n2_r2.getNodeID());
    SchedulerNode schedulerNode3 = newSchedulerNode(n3_r2.getHostName(), n3_r2.getRackName(), n3_r2.getNodeID());
    // 'zk' placement on Rack1 should FAIL
    replacedert.replacedertFalse(PlacementConstraintsUtil.canSatisfyConstraints(appId1, createSchedulingRequest(sourceTag2), schedulerNode0, pcm, tm));
    replacedert.replacedertFalse(PlacementConstraintsUtil.canSatisfyConstraints(appId1, createSchedulingRequest(sourceTag2), schedulerNode1, pcm, tm));
    // SUCCEED on the rest of the RACKs
    replacedert.replacedertTrue(PlacementConstraintsUtil.canSatisfyConstraints(appId1, createSchedulingRequest(sourceTag2), schedulerNode2, pcm, tm));
    replacedert.replacedertTrue(PlacementConstraintsUtil.canSatisfyConstraints(appId1, createSchedulingRequest(sourceTag2), schedulerNode3, pcm, tm));
}

19 View Complete Implementation : TestLogAggregationIndexedFileController.java
Copyright Apache License 2.0
Author : apache
@Before
public void setUp() throws IOException {
    setConf(getTestConf());
    appId = ApplicationId.newInstance(123456, 1);
    ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(appId, 1);
    containerId = ContainerId.newContainerId(attemptId, 1);
    nodeId = NodeId.newInstance("localhost", 9999);
    fs = FileSystem.get(getConf());
    sysOutStream = new ByteArrayOutputStream();
    PrintStream sysOut = new PrintStream(sysOutStream);
    System.setOut(sysOut);
    ByteArrayOutputStream sysErrStream = new ByteArrayOutputStream();
    PrintStream sysErr = new PrintStream(sysErrStream);
    System.setErr(sysErr);
}

19 View Complete Implementation : TestDistributedOpportunisticContainerAllocator.java
Copyright Apache License 2.0
Author : apache
@Test
public void testNodeLocalAllocationSameSchedKey() throws Exception {
    ResourceBlacklistRequest blacklistRequest = ResourceBlacklistRequest.newInstance(new ArrayList<>(), new ArrayList<>());
    List<ResourceRequest> reqs = Arrays.asList(ResourceRequest.newBuilder().allocationRequestId(2).numContainers(2).priority(PRIORITY_NORMAL).resourceName("/r1").capability(CAPABILITY_1GB).relaxLocality(true).executionType(ExecutionType.OPPORTUNISTIC).build(), ResourceRequest.newBuilder().allocationRequestId(2).numContainers(2).priority(PRIORITY_NORMAL).resourceName("h1").capability(CAPABILITY_1GB).relaxLocality(true).executionType(ExecutionType.OPPORTUNISTIC).build(), ResourceRequest.newBuilder().allocationRequestId(2).numContainers(2).priority(PRIORITY_NORMAL).resourceName(ResourceRequest.ANY).capability(CAPABILITY_1GB).relaxLocality(true).executionType(ExecutionType.OPPORTUNISTIC).build());
    ApplicationAttemptId appAttId = ApplicationAttemptId.newInstance(ApplicationId.newInstance(0L, 1), 1);
    oppCntxt.updateNodeList(Arrays.asList(RemoteNode.newInstance(NodeId.newInstance("h1", 1234), "h1:1234", "/r1"), RemoteNode.newInstance(NodeId.newInstance("h2", 1234), "h2:1234", "/r1"), RemoteNode.newInstance(NodeId.newInstance("h3", 1234), "h3:1234", "/r1")));
    List<Container> containers = allocator.allocateContainers(blacklistRequest, reqs, appAttId, oppCntxt, 1L, "luser");
    LOG.info("Containers: {}", containers);
    Set<String> allocatedHosts = new HashSet<>();
    for (Container c : containers) {
        allocatedHosts.add(c.getNodeHttpAddress());
    }
    replacedert.replacedertEquals(2, containers.size());
    replacedert.replacedertTrue(allocatedHosts.contains("h1:1234"));
    replacedert.replacedertFalse(allocatedHosts.contains("h2:1234"));
    replacedert.replacedertFalse(allocatedHosts.contains("h3:1234"));
}