Here are the examples of the java api org.apache.hadoop.yarn.api.records.ApplicationAttemptId.newInstance() taken from open source projects. By voting up you can indicate which examples are most useful and appropriate.
155 Examples
19
View Complete Implementation : TestDecommissioningNodesWatcher.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
// Get mocked ContainerStatus for bunch of containers,
// where numRunningContainers are RUNNING.
private List<ContainerStatus> getContainerStatuses(RMApp app, int numRunningContainers) {
// Total 3 containers
final int total = 3;
numRunningContainers = Math.min(total, numRunningContainers);
List<ContainerStatus> output = new ArrayList<ContainerStatus>();
for (int i = 0; i < total; i++) {
ContainerState cstate = (i >= numRunningContainers) ? ContainerState.COMPLETE : ContainerState.RUNNING;
output.add(ContainerStatus.newInstance(ContainerId.newContainerId(ApplicationAttemptId.newInstance(app.getApplicationId(), 0), i), cstate, "", 0));
}
return output;
}
19
View Complete Implementation : TestAHSWebApp.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
@Test
public void testAppAttemptPage() throws Exception {
Injector injector = WebAppTests.createMockInjector(ApplicationBaseProtocol.clreplaced, mockApplicationHistoryClientService(1, 1, 5));
AppAttemptPage appAttemptPageInstance = injector.getInstance(AppAttemptPage.clreplaced);
appAttemptPageInstance.render();
WebAppTests.flushOutput(injector);
appAttemptPageInstance.set(YarnWebParams.APPLICATION_ATTEMPT_ID, ApplicationAttemptId.newInstance(ApplicationId.newInstance(0, 1), 1).toString());
appAttemptPageInstance.render();
WebAppTests.flushOutput(injector);
}
19
View Complete Implementation : TestDistributedOpportunisticContainerAllocator.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
@Test
public void testRoundRobinRackLocalAllocationSameSchedKey() throws Exception {
ResourceBlacklistRequest blacklistRequest = ResourceBlacklistRequest.newInstance(new ArrayList<>(), new ArrayList<>());
List<ResourceRequest> reqs = Arrays.asList(ResourceRequest.newInstance(PRIORITY_NORMAL, "*", CAPABILITY_1GB, 2, true, null, OPPORTUNISTIC_REQ), ResourceRequest.newInstance(PRIORITY_NORMAL, "h1", CAPABILITY_1GB, 2, true, null, OPPORTUNISTIC_REQ), ResourceRequest.newInstance(PRIORITY_NORMAL, "/r1", CAPABILITY_1GB, 2, true, null, OPPORTUNISTIC_REQ));
ApplicationAttemptId appAttId = ApplicationAttemptId.newInstance(ApplicationId.newInstance(0L, 1), 1);
oppCntxt.updateNodeList(Arrays.asList(RemoteNode.newInstance(NodeId.newInstance("h3", 1234), "h3:1234", "/r2"), RemoteNode.newInstance(NodeId.newInstance("h2", 1234), "h2:1234", "/r1"), RemoteNode.newInstance(NodeId.newInstance("h5", 1234), "h5:1234", "/r1"), RemoteNode.newInstance(NodeId.newInstance("h4", 1234), "h4:1234", "/r2")));
List<Container> containers = allocator.allocateContainers(blacklistRequest, reqs, appAttId, oppCntxt, 1L, "luser");
Set<String> allocatedHosts = new HashSet<>();
for (Container c : containers) {
allocatedHosts.add(c.getNodeHttpAddress());
}
LOG.info("Containers: {}", containers);
replacedert.replacedertTrue(allocatedHosts.contains("h2:1234"));
replacedert.replacedertTrue(allocatedHosts.contains("h5:1234"));
replacedert.replacedertFalse(allocatedHosts.contains("h3:1234"));
replacedert.replacedertFalse(allocatedHosts.contains("h4:1234"));
replacedert.replacedertEquals(2, containers.size());
}
19
View Complete Implementation : TestApplicationHistoryClientService.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
@Test
public void testContainers() throws IOException, YarnException {
ApplicationId appId = ApplicationId.newInstance(0, 1);
ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(appId, 1);
ContainerId containerId = ContainerId.newContainerId(appAttemptId, 1);
ContainerId containerId1 = ContainerId.newContainerId(appAttemptId, 2);
GetContainersRequest request = GetContainersRequest.newInstance(appAttemptId);
GetContainersResponse response = clientService.getContainers(request);
List<ContainerReport> containers = response.getContainerList();
replacedert.replacedertNotNull(containers);
replacedert.replacedertEquals(containerId, containers.get(0).getContainerId());
replacedert.replacedertEquals(containerId1, containers.get(1).getContainerId());
}
19
View Complete Implementation : TestFileSystemApplicationHistoryStore.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
@Test
public void testWriteAfterApplicationFinish() throws IOException {
LOG.info("Starting testWriteAfterApplicationFinish");
ApplicationId appId = ApplicationId.newInstance(0, 1);
writeApplicationStartData(appId);
writeApplicationFinishData(appId);
// write application attempt history data
ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(appId, 1);
try {
writeApplicationAttemptStartData(appAttemptId);
replacedert.fail();
} catch (IOException e) {
replacedert.replacedertTrue(e.getMessage().contains("is not opened"));
}
try {
writeApplicationAttemptFinishData(appAttemptId);
replacedert.fail();
} catch (IOException e) {
replacedert.replacedertTrue(e.getMessage().contains("is not opened"));
}
// write container history data
ContainerId containerId = ContainerId.newContainerId(appAttemptId, 1);
try {
writeContainerStartData(containerId);
replacedert.fail();
} catch (IOException e) {
replacedert.replacedertTrue(e.getMessage().contains("is not opened"));
}
try {
writeContainerFinishData(containerId);
replacedert.fail();
} catch (IOException e) {
replacedert.replacedertTrue(e.getMessage().contains("is not opened"));
}
}
19
View Complete Implementation : TestApplicationHistoryClientService.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
@Test
public void testApplicationAttemptReport() throws IOException, YarnException {
ApplicationId appId = ApplicationId.newInstance(0, 1);
ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(appId, 1);
GetApplicationAttemptReportRequest request = GetApplicationAttemptReportRequest.newInstance(appAttemptId);
GetApplicationAttemptReportResponse response = clientService.getApplicationAttemptReport(request);
ApplicationAttemptReport attemptReport = response.getApplicationAttemptReport();
replacedert.replacedertNotNull(attemptReport);
replacedert.replacedertEquals("appattempt_0_0001_000001", attemptReport.getApplicationAttemptId().toString());
}
19
View Complete Implementation : TestCentralizedOpportunisticContainerAllocator.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
/**
* Tests Opportunistic container should not be allocated on blacklisted
* nodes.
* @throws Exception
*/
@Test
public void testBlacklistRejection() throws Exception {
ResourceBlacklistRequest blacklistRequest = ResourceBlacklistRequest.newInstance(Arrays.asList("h1", "h2"), new ArrayList<>());
List<ResourceRequest> reqs = Collections.singletonList(createResourceRequest(1, "*", 1));
ApplicationAttemptId appAttId = ApplicationAttemptId.newInstance(ApplicationId.newInstance(0L, 1), 1);
allocator.setNodeQueueLoadMonitor(createNodeQueueLoadMonitor(2, 2, 100));
List<Container> containers = allocator.allocateContainers(blacklistRequest, reqs, appAttId, oppCntxt, 1L, "user");
replacedertEquals(0, containers.size());
replacedertEquals(1, oppCntxt.getOutstandingOpReqs().size());
}
19
View Complete Implementation : TestContainerLaunchRPC.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
private void testRPCTimeout(String rpcClreplaced) throws Exception {
Configuration conf = new Configuration();
// set timeout low for the test
conf.setInt("yarn.rpc.nm-command-timeout", 3000);
conf.set(YarnConfiguration.IPC_RPC_IMPL, rpcClreplaced);
YarnRPC rpc = YarnRPC.create(conf);
String bindAddr = "localhost:0";
InetSocketAddress addr = NetUtils.createSocketAddr(bindAddr);
Server server = rpc.getServer(ContainerManagementProtocol.clreplaced, new DummyContainerManager(), addr, conf, null, 1);
server.start();
try {
ContainerManagementProtocol proxy = (ContainerManagementProtocol) rpc.getProxy(ContainerManagementProtocol.clreplaced, server.getListenerAddress(), conf);
ContainerLaunchContext containerLaunchContext = recordFactory.newRecordInstance(ContainerLaunchContext.clreplaced);
ApplicationId applicationId = ApplicationId.newInstance(0, 0);
ApplicationAttemptId applicationAttemptId = ApplicationAttemptId.newInstance(applicationId, 0);
ContainerId containerId = ContainerId.newContainerId(applicationAttemptId, 100);
NodeId nodeId = NodeId.newInstance("localhost", 1234);
Resource resource = Resource.newInstance(1234, 2);
ContainerTokenIdentifier containerTokenIdentifier = new ContainerTokenIdentifier(containerId, "localhost", "user", resource, System.currentTimeMillis() + 10000, 42, 42, Priority.newInstance(0), 0);
Token containerToken = newContainerToken(nodeId, "preplacedword".getBytes(), containerTokenIdentifier);
StartContainerRequest scRequest = StartContainerRequest.newInstance(containerLaunchContext, containerToken);
List<StartContainerRequest> list = new ArrayList<StartContainerRequest>();
list.add(scRequest);
StartContainersRequest allRequests = StartContainersRequest.newInstance(list);
try {
proxy.startContainers(allRequests);
} catch (Exception e) {
LOG.info(StringUtils.stringifyException(e));
replacedert.replacedertEquals("Error, exception is not: " + SocketTimeoutException.clreplaced.getName(), SocketTimeoutException.clreplaced.getName(), e.getClreplaced().getName());
return;
}
} finally {
server.stop();
}
replacedert.fail("timeout exception should have occurred!");
}
19
View Complete Implementation : TestAppLogAggregatorImpl.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
@Test
public void testAggregatorWithRetentionPolicyDisabledShouldUploadAllFiles() throws Exception {
final ApplicationId applicationId = ApplicationId.newInstance(System.currentTimeMillis(), 0);
final ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(applicationId, 0);
final ContainerId containerId = ContainerId.newContainerId(attemptId, 0);
// create artificial log files
final File appLogDir = new File(LOCAL_LOG_DIR, applicationId.toString());
final File containerLogDir = new File(appLogDir, containerId.toString());
containerLogDir.mkdirs();
final Set<File> logFiles = createContainerLogFiles(containerLogDir, 3);
final long logRetentionSecs = 10000;
final long recoveredLogInitedTime = -1;
verifyLogAggregationWithExpectedFiles2DeleteAndUpload(applicationId, containerId, logRetentionSecs, recoveredLogInitedTime, logFiles, logFiles);
}
19
View Complete Implementation : TestCentralizedOpportunisticContainerAllocator.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
/**
* Tests that allocation of Opportunistic containers should be spread out.
* @throws Exception
*/
@Test
public void testRoundRobinSimpleAllocation() throws Exception {
List<ResourceRequest> reqs = Arrays.asList(createResourceRequest(1, ResourceRequest.ANY, 1), createResourceRequest(2, ResourceRequest.ANY, 1), createResourceRequest(3, ResourceRequest.ANY, 1));
ApplicationAttemptId appAttId = ApplicationAttemptId.newInstance(ApplicationId.newInstance(0L, 1), 1);
allocator.setNodeQueueLoadMonitor(createNodeQueueLoadMonitor(3, 2, 3));
List<Container> containers = allocator.allocateContainers(EMPTY_BLACKLIST_REQUEST, reqs, appAttId, oppCntxt, 1L, "user");
LOG.info("Containers: {}", containers);
Set<String> allocatedNodes = new HashSet<>();
for (Container c : containers) {
allocatedNodes.add(c.getNodeId().toString());
}
replacedertTrue(allocatedNodes.contains("h1:1234"));
replacedertTrue(allocatedNodes.contains("h2:1234"));
replacedertTrue(allocatedNodes.contains("h3:1234"));
replacedertEquals(3, containers.size());
}
19
View Complete Implementation : TestPlacementConstraintsUtil.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
@Test
public void testNodeAntiAffinityreplacedignment() throws InvalidAllocationTagsQueryException {
PlacementConstraintManagerService pcm = new MemoryPlacementConstraintManager();
AllocationTagsManager tm = new AllocationTagsManager(rmContext);
// Register App1 with anti-affinity constraint map
pcm.registerApplication(appId1, constraintMap2);
/**
* place container:
* Node0:123 (Rack1):
* container_app1_1 (hbase-m)
*/
RMNode n0_r1 = rmNodes.get(0);
RMNode n1_r1 = rmNodes.get(1);
RMNode n2_r2 = rmNodes.get(2);
RMNode n3_r2 = rmNodes.get(3);
SchedulerNode schedulerNode0 = newSchedulerNode(n0_r1.getHostName(), n0_r1.getRackName(), n0_r1.getNodeID());
SchedulerNode schedulerNode1 = newSchedulerNode(n1_r1.getHostName(), n1_r1.getRackName(), n1_r1.getNodeID());
SchedulerNode schedulerNode2 = newSchedulerNode(n2_r2.getHostName(), n2_r2.getRackName(), n2_r2.getNodeID());
SchedulerNode schedulerNode3 = newSchedulerNode(n3_r2.getHostName(), n3_r2.getRackName(), n3_r2.getNodeID());
// 1 Containers on node 0 with allocationTag 'hbase-m'
ContainerId hbase_m = ContainerId.newContainerId(ApplicationAttemptId.newInstance(appId1, 0), 0);
tm.addContainer(n0_r1.getNodeID(), hbase_m, ImmutableSet.of("hbase-m"));
// 'spark' placement on Node0 should now FAIL
replacedert.replacedertFalse(PlacementConstraintsUtil.canSatisfyConstraints(appId1, createSchedulingRequest(sourceTag1), schedulerNode0, pcm, tm));
// SUCCEED on the rest of the nodes
replacedert.replacedertTrue(PlacementConstraintsUtil.canSatisfyConstraints(appId1, createSchedulingRequest(sourceTag1), schedulerNode1, pcm, tm));
replacedert.replacedertTrue(PlacementConstraintsUtil.canSatisfyConstraints(appId1, createSchedulingRequest(sourceTag1), schedulerNode2, pcm, tm));
replacedert.replacedertTrue(PlacementConstraintsUtil.canSatisfyConstraints(appId1, createSchedulingRequest(sourceTag1), schedulerNode3, pcm, tm));
}
19
View Complete Implementation : TestYARNTokenIdentifier.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
@Test
public void testAMContainerTokenIdentifier() throws IOException {
ContainerId containerID = ContainerId.newContainerId(ApplicationAttemptId.newInstance(ApplicationId.newInstance(1, 1), 1), 1);
String hostName = "host0";
String appSubmitter = "usr0";
Resource r = Resource.newInstance(1024, 1);
long expiryTimeStamp = 1000;
int masterKeyId = 1;
long rmIdentifier = 1;
Priority priority = Priority.newInstance(1);
long creationTime = 1000;
ContainerTokenIdentifier token = new ContainerTokenIdentifier(containerID, hostName, appSubmitter, r, expiryTimeStamp, masterKeyId, rmIdentifier, priority, creationTime, null, CommonNodeLabelsManager.NO_LABEL, ContainerType.APPLICATION_MASTER);
ContainerTokenIdentifier anotherToken = new ContainerTokenIdentifier();
byte[] tokenContent = token.getBytes();
DataInputBuffer dib = new DataInputBuffer();
dib.reset(tokenContent, tokenContent.length);
anotherToken.readFields(dib);
replacedert.replacedertEquals(ContainerType.APPLICATION_MASTER, anotherToken.getContainerType());
replacedert.replacedertEquals(ExecutionType.GUARANTEED, anotherToken.getExecutionType());
token = new ContainerTokenIdentifier(containerID, 0, hostName, appSubmitter, r, expiryTimeStamp, masterKeyId, rmIdentifier, priority, creationTime, null, CommonNodeLabelsManager.NO_LABEL, ContainerType.TASK, ExecutionType.OPPORTUNISTIC);
anotherToken = new ContainerTokenIdentifier();
tokenContent = token.getBytes();
dib = new DataInputBuffer();
dib.reset(tokenContent, tokenContent.length);
anotherToken.readFields(dib);
replacedert.replacedertEquals(ContainerType.TASK, anotherToken.getContainerType());
replacedert.replacedertEquals(ExecutionType.OPPORTUNISTIC, anotherToken.getExecutionType());
}
19
View Complete Implementation : BaseAMRMProxyTest.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
/**
* Return an instance of ApplicationAttemptId using specified identifier. This
* identifier will be used for the ApplicationId too.
*
* @param testAppId
* @return
*/
protected ApplicationAttemptId getApplicationAttemptId(int testAppId) {
return ApplicationAttemptId.newInstance(getApplicationId(testAppId), testAppId);
}
19
View Complete Implementation : TestCentralizedOpportunisticContainerAllocator.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
/**
* Tests that allocation of rack local Opportunistic container requests
* with same allocation request id should be spread out.
* @throws Exception
*/
@Test
public void testRoundRobinRackLocalAllocationSameSchedulerKey() throws Exception {
List<ResourceRequest> reqs = Arrays.asList(createResourceRequest(2, "/r1", 2), createResourceRequest(2, "h5", 2), createResourceRequest(2, ResourceRequest.ANY, 2));
ApplicationAttemptId appAttId = ApplicationAttemptId.newInstance(ApplicationId.newInstance(0L, 1), 1);
NodeQueueLoadMonitor selector = createNodeQueueLoadMonitor(Arrays.asList("h1", "h2", "h3", "h4"), Arrays.asList("/r2", "/r1", "/r3", "/r1"), Arrays.asList(4, 4, 4, 4), Arrays.asList(5, 5, 5, 5));
allocator.setNodeQueueLoadMonitor(selector);
List<Container> containers = allocator.allocateContainers(EMPTY_BLACKLIST_REQUEST, reqs, appAttId, oppCntxt, 1L, "user");
Set<String> allocatedHosts = new HashSet<>();
for (Container c : containers) {
allocatedHosts.add(c.getNodeId().toString());
}
LOG.info("Containers: {}", containers);
replacedertTrue(allocatedHosts.contains("h2:1234"));
replacedertTrue(allocatedHosts.contains("h4:1234"));
replacedertFalse(allocatedHosts.contains("h1:1234"));
replacedertFalse(allocatedHosts.contains("h3:1234"));
replacedertEquals(2, containers.size());
}
19
View Complete Implementation : TestCentralizedOpportunisticContainerAllocator.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
/**
* Tests maximum opportunistic container allocation per AM heartbeat when
* limit is set to -1.
* @throws Exception
*/
@Test
public void testMaxAllocationsPerAMHeartbeatWithNoLimit() throws Exception {
allocator.setMaxAllocationsPerAMHeartbeat(-1);
List<ResourceRequest> reqs = new ArrayList<>();
final int numContainers = 20;
for (int i = 0; i < numContainers; i++) {
reqs.add(createResourceRequest(i + 1, "h1", 1));
}
ApplicationAttemptId appAttId = ApplicationAttemptId.newInstance(ApplicationId.newInstance(0L, 1), 1);
allocator.setNodeQueueLoadMonitor(createNodeQueueLoadMonitor(3, 2, 500));
List<Container> containers = allocator.allocateContainers(EMPTY_BLACKLIST_REQUEST, reqs, appAttId, oppCntxt, 1L, "user");
// all containers should be allocated in single heartbeat.
replacedertEquals(numContainers, containers.size());
}
19
View Complete Implementation : TestDistributedOpportunisticContainerAllocator.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
@Test
public void testSimpleAllocation() throws Exception {
ResourceBlacklistRequest blacklistRequest = ResourceBlacklistRequest.newInstance(new ArrayList<>(), new ArrayList<>());
List<ResourceRequest> reqs = Arrays.asList(ResourceRequest.newInstance(PRIORITY_NORMAL, "*", CAPABILITY_1GB, 1, true, null, OPPORTUNISTIC_REQ));
ApplicationAttemptId appAttId = ApplicationAttemptId.newInstance(ApplicationId.newInstance(0L, 1), 1);
oppCntxt.updateNodeList(Arrays.asList(RemoteNode.newInstance(NodeId.newInstance("h1", 1234), "h1:1234", "/r1")));
List<Container> containers = allocator.allocateContainers(blacklistRequest, reqs, appAttId, oppCntxt, 1L, "luser");
replacedert.replacedertEquals(1, containers.size());
replacedert.replacedertEquals(0, oppCntxt.getOutstandingOpReqs().size());
}
19
View Complete Implementation : TestCentralizedOpportunisticContainerAllocator.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
/**
* Tests allocation of an Opportunistic container from single application.
* @throws Exception
*/
@Test
public void testSimpleAllocation() throws Exception {
List<ResourceRequest> reqs = Collections.singletonList(createResourceRequest(1, "*", 1));
ApplicationAttemptId appAttId = ApplicationAttemptId.newInstance(ApplicationId.newInstance(0L, 1), 1);
allocator.setNodeQueueLoadMonitor(createNodeQueueLoadMonitor(1, 2, 100));
List<Container> containers = allocator.allocateContainers(EMPTY_BLACKLIST_REQUEST, reqs, appAttId, oppCntxt, 1L, "user");
replacedertEquals(1, containers.size());
replacedertEquals(0, oppCntxt.getOutstandingOpReqs().size());
}
19
View Complete Implementation : TestApplicationHistoryClientService.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
@Test
public void testContainerReport() throws IOException, YarnException {
ApplicationId appId = ApplicationId.newInstance(0, 1);
ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(appId, 1);
ContainerId containerId = ContainerId.newContainerId(appAttemptId, 1);
GetContainerReportRequest request = GetContainerReportRequest.newInstance(containerId);
GetContainerReportResponse response = clientService.getContainerReport(request);
ContainerReport container = response.getContainerReport();
replacedert.replacedertNotNull(container);
replacedert.replacedertEquals(containerId, container.getContainerId());
replacedert.replacedertEquals("http://0.0.0.0:8188/applicationhistory/logs/" + "test host:100/container_0_0001_01_000001/" + "container_0_0001_01_000001/user1", container.getLogUrl());
}
19
View Complete Implementation : TestCentralizedOpportunisticContainerAllocator.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
/**
* Tests that allocation of rack local Opportunistic container requests
* should be spread out.
* @throws Exception
*/
@Test
public void testRoundRobinRackLocalAllocation() throws Exception {
List<ResourceRequest> reqs = Arrays.asList(createResourceRequest(1, "/r1", 1), createResourceRequest(1, "h5", 1), createResourceRequest(1, ResourceRequest.ANY, 1), createResourceRequest(2, "/r1", 1), createResourceRequest(2, "h5", 1), createResourceRequest(2, ResourceRequest.ANY, 1));
ApplicationAttemptId appAttId = ApplicationAttemptId.newInstance(ApplicationId.newInstance(0L, 1), 1);
NodeQueueLoadMonitor selector = createNodeQueueLoadMonitor(Arrays.asList("h1", "h2", "h3", "h4"), Arrays.asList("/r2", "/r1", "/r3", "/r1"), Arrays.asList(4, 4, 4, 4), Arrays.asList(5, 5, 5, 5));
allocator.setNodeQueueLoadMonitor(selector);
List<Container> containers = allocator.allocateContainers(EMPTY_BLACKLIST_REQUEST, reqs, appAttId, oppCntxt, 1L, "user");
Set<String> allocatedHosts = new HashSet<>();
for (Container c : containers) {
allocatedHosts.add(c.getNodeId().toString());
}
LOG.info("Containers: {}", containers);
replacedertTrue(allocatedHosts.contains("h2:1234"));
replacedertTrue(allocatedHosts.contains("h4:1234"));
replacedertFalse(allocatedHosts.contains("h1:1234"));
replacedertFalse(allocatedHosts.contains("h3:1234"));
replacedertEquals(2, containers.size());
}
19
View Complete Implementation : TestAHSClient.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
@Test(timeout = 10000)
public void testGetApplicationAttempts() throws YarnException, IOException {
Configuration conf = new Configuration();
final AHSClient client = new MockAHSClient();
client.init(conf);
client.start();
ApplicationId applicationId = ApplicationId.newInstance(1234, 5);
List<ApplicationAttemptReport> reports = client.getApplicationAttempts(applicationId);
replacedert.replacedertNotNull(reports);
replacedert.replacedertEquals(reports.get(0).getApplicationAttemptId(), ApplicationAttemptId.newInstance(applicationId, 1));
replacedert.replacedertEquals(reports.get(1).getApplicationAttemptId(), ApplicationAttemptId.newInstance(applicationId, 2));
client.stop();
}
19
View Complete Implementation : TestNMTimelinePublisher.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
@Test
public void testPublishContainerPausedEvent() {
ApplicationId appId = ApplicationId.newInstance(0, 1);
ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(appId, 1);
ContainerId cId = ContainerId.newContainerId(appAttemptId, 1);
ContainerEvent containerEvent = new ContainerPauseEvent(cId, "test pause");
publisher.createTimelineClient(appId);
publisher.publishContainerEvent(containerEvent);
publisher.stopTimelineClient(appId);
dispatcher.await();
ContainerEnreplacedy cEnreplacedy = new ContainerEnreplacedy();
cEnreplacedy.setId(cId.toString());
TimelineEnreplacedy[] lastPublishedEnreplacedies = timelineClient.getLastPublishedEnreplacedies();
replacedert.replacedertNotNull(lastPublishedEnreplacedies);
replacedert.replacedertEquals(1, lastPublishedEnreplacedies.length);
TimelineEnreplacedy enreplacedy = lastPublishedEnreplacedies[0];
replacedert.replacedertEquals(cEnreplacedy, enreplacedy);
NavigableSet<TimelineEvent> events = enreplacedy.getEvents();
replacedert.replacedertEquals(1, events.size());
replacedert.replacedertEquals(ContainerMetricsConstants.PAUSED_EVENT_TYPE, events.iterator().next().getId());
Map<String, Object> info = enreplacedy.getInfo();
replacedert.replacedertTrue(info.containsKey(ContainerMetricsConstants.DIAGNOSTICS_INFO));
replacedert.replacedertEquals("test pause", info.get(ContainerMetricsConstants.DIAGNOSTICS_INFO));
}
19
View Complete Implementation : TestFileSystemApplicationHistoryStore.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
private void testWriteHistoryData(int num, boolean missingContainer, boolean missingApplicationAttempt) throws IOException {
// write application history data
for (int i = 1; i <= num; ++i) {
ApplicationId appId = ApplicationId.newInstance(0, i);
writeApplicationStartData(appId);
// write application attempt history data
for (int j = 1; j <= num; ++j) {
ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(appId, j);
writeApplicationAttemptStartData(appAttemptId);
if (missingApplicationAttempt && j == num) {
continue;
}
// write container history data
for (int k = 1; k <= num; ++k) {
ContainerId containerId = ContainerId.newContainerId(appAttemptId, k);
writeContainerStartData(containerId);
if (missingContainer && k == num) {
continue;
}
writeContainerFinishData(containerId);
}
writeApplicationAttemptFinishData(appAttemptId);
}
writeApplicationFinishData(appId);
}
}
19
View Complete Implementation : TestMRJobsWithHistoryService.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
private void verifyJobReport(JobReport jobReport, JobId jobId) {
List<AMInfo> amInfos = jobReport.getAMInfos();
replacedert.replacedertEquals(1, amInfos.size());
AMInfo amInfo = amInfos.get(0);
ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(jobId.getAppId(), 1);
ContainerId amContainerId = ContainerId.newContainerId(appAttemptId, 1);
replacedert.replacedertEquals(appAttemptId, amInfo.getAppAttemptId());
replacedert.replacedertEquals(amContainerId, amInfo.getContainerId());
replacedert.replacedertTrue(jobReport.getSubmitTime() > 0);
replacedert.replacedertTrue(jobReport.getStartTime() > 0 && jobReport.getStartTime() >= jobReport.getSubmitTime());
replacedert.replacedertTrue(jobReport.getFinishTime() > 0 && jobReport.getFinishTime() >= jobReport.getStartTime());
}
19
View Complete Implementation : TestQueueState.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
private FiCaSchedulerApp getMockApplication(ApplicationId appId, String user, Resource amResource) {
FiCaSchedulerApp application = mock(FiCaSchedulerApp.clreplaced);
ApplicationAttemptId applicationAttemptId = ApplicationAttemptId.newInstance(appId, 0);
doReturn(applicationAttemptId.getApplicationId()).when(application).getApplicationId();
doReturn(applicationAttemptId).when(application).getApplicationAttemptId();
doReturn(user).when(application).getUser();
doReturn(amResource).when(application).getAMResource();
doReturn(Priority.newInstance(0)).when(application).getPriority();
doReturn(CommonNodeLabelsManager.NO_LABEL).when(application).getAppAMNodeParreplacedionName();
doReturn(amResource).when(application).getAMResource(CommonNodeLabelsManager.NO_LABEL);
when(application.compareInputOrderTo(any(FiCaSchedulerApp.clreplaced))).thenCallRealMethod();
return application;
}
19
View Complete Implementation : TestAHSClient.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
@Test(timeout = 10000)
public void testGetApplicationAttempt() throws YarnException, IOException {
Configuration conf = new Configuration();
final AHSClient client = new MockAHSClient();
client.init(conf);
client.start();
List<ApplicationReport> expectedReports = ((MockAHSClient) client).getReports();
ApplicationId applicationId = ApplicationId.newInstance(1234, 5);
ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(applicationId, 1);
ApplicationAttemptReport report = client.getApplicationAttemptReport(appAttemptId);
replacedert.replacedertNotNull(report);
replacedert.replacedertEquals(report.getApplicationAttemptId().toString(), expectedReports.get(0).getCurrentApplicationAttemptId().toString());
client.stop();
}
19
View Complete Implementation : TestDistributedOpportunisticContainerAllocator.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
@Test
public void testRoundRobinRackLocalAllocation() throws Exception {
ResourceBlacklistRequest blacklistRequest = ResourceBlacklistRequest.newInstance(new ArrayList<>(), new ArrayList<>());
List<ResourceRequest> reqs = Arrays.asList(ResourceRequest.newBuilder().allocationRequestId(1).priority(PRIORITY_NORMAL).resourceName("/r1").capability(CAPABILITY_1GB).relaxLocality(true).executionType(ExecutionType.OPPORTUNISTIC).build(), ResourceRequest.newBuilder().allocationRequestId(1).priority(PRIORITY_NORMAL).resourceName("h1").capability(CAPABILITY_1GB).relaxLocality(true).executionType(ExecutionType.OPPORTUNISTIC).build(), ResourceRequest.newBuilder().allocationRequestId(1).priority(PRIORITY_NORMAL).resourceName(ResourceRequest.ANY).capability(CAPABILITY_1GB).relaxLocality(true).executionType(ExecutionType.OPPORTUNISTIC).build(), ResourceRequest.newBuilder().allocationRequestId(2).priority(PRIORITY_NORMAL).resourceName("/r1").capability(CAPABILITY_1GB).relaxLocality(true).executionType(ExecutionType.OPPORTUNISTIC).build(), ResourceRequest.newBuilder().allocationRequestId(2).priority(PRIORITY_NORMAL).resourceName("h1").capability(CAPABILITY_1GB).relaxLocality(true).executionType(ExecutionType.OPPORTUNISTIC).build(), ResourceRequest.newBuilder().allocationRequestId(2).priority(PRIORITY_NORMAL).resourceName(ResourceRequest.ANY).capability(CAPABILITY_1GB).relaxLocality(true).executionType(ExecutionType.OPPORTUNISTIC).build());
ApplicationAttemptId appAttId = ApplicationAttemptId.newInstance(ApplicationId.newInstance(0L, 1), 1);
oppCntxt.updateNodeList(Arrays.asList(RemoteNode.newInstance(NodeId.newInstance("h3", 1234), "h3:1234", "/r2"), RemoteNode.newInstance(NodeId.newInstance("h2", 1234), "h2:1234", "/r1"), RemoteNode.newInstance(NodeId.newInstance("h5", 1234), "h5:1234", "/r1"), RemoteNode.newInstance(NodeId.newInstance("h4", 1234), "h4:1234", "/r2")));
List<Container> containers = allocator.allocateContainers(blacklistRequest, reqs, appAttId, oppCntxt, 1L, "luser");
Set<String> allocatedHosts = new HashSet<>();
for (Container c : containers) {
allocatedHosts.add(c.getNodeHttpAddress());
}
LOG.info("Containers: {}", containers);
replacedert.replacedertTrue(allocatedHosts.contains("h2:1234"));
replacedert.replacedertTrue(allocatedHosts.contains("h5:1234"));
replacedert.replacedertFalse(allocatedHosts.contains("h3:1234"));
replacedert.replacedertFalse(allocatedHosts.contains("h4:1234"));
replacedert.replacedertEquals(2, containers.size());
}
19
View Complete Implementation : TestCentralizedOpportunisticContainerAllocator.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
/**
* Tests scheduling of many rack local Opportunistic container requests.
* @throws Exception
*/
@Test
public void testLotsOfContainersRackLocalAllocation() throws Exception {
List<ResourceRequest> reqs = new ArrayList<>();
// add 100 container requests.
for (int i = 0; i < 100; i++) {
reqs.add(createResourceRequest(i + 1, ResourceRequest.ANY, 1));
reqs.add(createResourceRequest(i + 1, "h5", 1));
reqs.add(createResourceRequest(i + 1, "/r1", 1));
}
ApplicationAttemptId appAttId = ApplicationAttemptId.newInstance(ApplicationId.newInstance(0L, 1), 1);
NodeQueueLoadMonitor selector = createNodeQueueLoadMonitor(Arrays.asList("h1", "h2", "h3", "h4"), Arrays.asList("/r1", "/r1", "/r1", "/r2"), Arrays.asList(0, 0, 0, 0), Arrays.asList(500, 500, 500, 300));
allocator.setNodeQueueLoadMonitor(selector);
List<Container> containers = new ArrayList<>();
containers = allocator.allocateContainers(EMPTY_BLACKLIST_REQUEST, reqs, appAttId, oppCntxt, 1L, "user");
replacedertEquals(100, containers.size());
}
19
View Complete Implementation : TestQueueStateManager.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
private FiCaSchedulerApp getMockApplication(ApplicationId appId, String user, Resource amResource) {
FiCaSchedulerApp application = mock(FiCaSchedulerApp.clreplaced);
ApplicationAttemptId applicationAttemptId = ApplicationAttemptId.newInstance(appId, 0);
doReturn(applicationAttemptId.getApplicationId()).when(application).getApplicationId();
doReturn(applicationAttemptId).when(application).getApplicationAttemptId();
doReturn(user).when(application).getUser();
doReturn(amResource).when(application).getAMResource();
doReturn(Priority.newInstance(0)).when(application).getPriority();
doReturn(CommonNodeLabelsManager.NO_LABEL).when(application).getAppAMNodeParreplacedionName();
doReturn(amResource).when(application).getAMResource(CommonNodeLabelsManager.NO_LABEL);
when(application.compareInputOrderTo(any(FiCaSchedulerApp.clreplaced))).thenCallRealMethod();
return application;
}
19
View Complete Implementation : TestDistributedOpportunisticContainerAllocator.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
/**
* Tests maximum opportunistic container allocation per AM heartbeat for
* allocation requests with different scheduler key.
* @throws Exception
*/
@Test
public void testMaxAllocationsPerAMHeartbeatDifferentSchedKey() throws Exception {
ResourceBlacklistRequest blacklistRequest = ResourceBlacklistRequest.newInstance(new ArrayList<>(), new ArrayList<>());
allocator.setMaxAllocationsPerAMHeartbeat(2);
final ExecutionTypeRequest oppRequest = ExecutionTypeRequest.newInstance(ExecutionType.OPPORTUNISTIC, true);
List<ResourceRequest> reqs = Arrays.asList(ResourceRequest.newInstance(Priority.newInstance(1), "*", CAPABILITY_1GB, 1, true, null, OPPORTUNISTIC_REQ), ResourceRequest.newInstance(Priority.newInstance(2), "h6", CAPABILITY_1GB, 2, true, null, OPPORTUNISTIC_REQ), ResourceRequest.newInstance(Priority.newInstance(3), "/r3", CAPABILITY_1GB, 2, true, null, OPPORTUNISTIC_REQ));
ApplicationAttemptId appAttId = ApplicationAttemptId.newInstance(ApplicationId.newInstance(0L, 1), 1);
oppCntxt.updateNodeList(Arrays.asList(RemoteNode.newInstance(NodeId.newInstance("h3", 1234), "h3:1234", "/r2"), RemoteNode.newInstance(NodeId.newInstance("h2", 1234), "h2:1234", "/r1"), RemoteNode.newInstance(NodeId.newInstance("h5", 1234), "h5:1234", "/r1"), RemoteNode.newInstance(NodeId.newInstance("h4", 1234), "h4:1234", "/r2")));
List<Container> containers = allocator.allocateContainers(blacklistRequest, reqs, appAttId, oppCntxt, 1L, "user1");
LOG.info("Containers: {}", containers);
// Although capacity is present, but only 2 containers should be allocated
// as max allocation per AM heartbeat is set to 2.
replacedert.replacedertEquals(2, containers.size());
containers = allocator.allocateContainers(blacklistRequest, new ArrayList<>(), appAttId, oppCntxt, 1L, "user1");
LOG.info("Containers: {}", containers);
// 2 more containers should be allocated from pending allocation requests.
replacedert.replacedertEquals(2, containers.size());
containers = allocator.allocateContainers(blacklistRequest, new ArrayList<>(), appAttId, oppCntxt, 1L, "user1");
LOG.info("Containers: {}", containers);
// Remaining 1 container should be allocated.
replacedert.replacedertEquals(1, containers.size());
}
19
View Complete Implementation : TestDistributedOpportunisticContainerAllocator.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
@Test
public void testLotsOfContainersRackLocalAllocationSameSchedKey() throws Exception {
ResourceBlacklistRequest blacklistRequest = ResourceBlacklistRequest.newInstance(new ArrayList<>(), new ArrayList<>());
List<ResourceRequest> reqs = Arrays.asList(ResourceRequest.newInstance(PRIORITY_NORMAL, "*", CAPABILITY_1GB, 1000, true, null, OPPORTUNISTIC_REQ), ResourceRequest.newInstance(PRIORITY_NORMAL, "h1", CAPABILITY_1GB, 1000, true, null, OPPORTUNISTIC_REQ), ResourceRequest.newInstance(PRIORITY_NORMAL, "/r1", CAPABILITY_1GB, 1000, true, null, OPPORTUNISTIC_REQ));
ApplicationAttemptId appAttId = ApplicationAttemptId.newInstance(ApplicationId.newInstance(0L, 1), 1);
oppCntxt.updateNodeList(Arrays.asList(RemoteNode.newInstance(NodeId.newInstance("h3", 1234), "h3:1234", "/r2"), RemoteNode.newInstance(NodeId.newInstance("h2", 1234), "h2:1234", "/r1"), RemoteNode.newInstance(NodeId.newInstance("h5", 1234), "h5:1234", "/r1"), RemoteNode.newInstance(NodeId.newInstance("h4", 1234), "h4:1234", "/r2")));
List<Container> containers = new ArrayList<>();
for (int i = 0; i < 250; i++) {
containers.addAll(allocator.allocateContainers(blacklistRequest, reqs, appAttId, oppCntxt, 1L, "luser"));
}
replacedert.replacedertEquals(1000, containers.size());
}
19
View Complete Implementation : TestNMProxy.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
private ContainerManagementProtocol getNMProxy(Configuration conf) {
ApplicationId appId = ApplicationId.newInstance(1, 1);
ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(appId, 1);
org.apache.hadoop.yarn.api.records.Token nmToken = context.getNMTokenSecretManager().createNMToken(attemptId, context.getNodeId(), user);
final InetSocketAddress address = conf.getSocketAddr(YarnConfiguration.NM_BIND_HOST, YarnConfiguration.NM_ADDRESS, YarnConfiguration.DEFAULT_NM_ADDRESS, YarnConfiguration.DEFAULT_NM_PORT);
Token<NMTokenIdentifier> token = ConverterUtils.convertFromYarn(nmToken, SecurityUtil.buildTokenService(address));
UserGroupInformation ugi = UserGroupInformation.createRemoteUser(user);
ugi.addToken(token);
return NMProxy.createNMProxy(conf, ContainerManagementProtocol.clreplaced, ugi, YarnRPC.create(conf), address);
}
19
View Complete Implementation : TestLocalContainerAllocator.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
@Test
public void testAMRMTokenUpdate() throws Exception {
Configuration conf = new Configuration();
ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(ApplicationId.newInstance(1, 1), 1);
AMRMTokenIdentifier oldTokenId = new AMRMTokenIdentifier(attemptId, 1);
AMRMTokenIdentifier newTokenId = new AMRMTokenIdentifier(attemptId, 2);
Token<AMRMTokenIdentifier> oldToken = new Token<AMRMTokenIdentifier>(oldTokenId.getBytes(), "oldpreplacedword".getBytes(), oldTokenId.getKind(), new Text());
Token<AMRMTokenIdentifier> newToken = new Token<AMRMTokenIdentifier>(newTokenId.getBytes(), "newpreplacedword".getBytes(), newTokenId.getKind(), new Text());
MockScheduler scheduler = new MockScheduler();
scheduler.amToken = newToken;
final LocalContainerAllocator lca = new StubbedLocalContainerAllocator(scheduler);
lca.init(conf);
lca.start();
UserGroupInformation testUgi = UserGroupInformation.createUserForTesting("someuser", new String[0]);
testUgi.addToken(oldToken);
testUgi.doAs(new PrivilegedExceptionAction<Void>() {
@Override
public Void run() throws Exception {
lca.heartbeat();
return null;
}
});
lca.close();
// verify there is only one AMRM token in the UGI and it matches the
// updated token from the RM
int tokenCount = 0;
Token<? extends TokenIdentifier> ugiToken = null;
for (Token<? extends TokenIdentifier> token : testUgi.getTokens()) {
if (AMRMTokenIdentifier.KIND_NAME.equals(token.getKind())) {
ugiToken = token;
++tokenCount;
}
}
replacedert.replacedertEquals("too many AMRM tokens", 1, tokenCount);
replacedert.replacedertArrayEquals("token identifier not updated", newToken.getIdentifier(), ugiToken.getIdentifier());
replacedert.replacedertArrayEquals("token preplacedword not updated", newToken.getPreplacedword(), ugiToken.getPreplacedword());
replacedert.replacedertEquals("AMRM token service not updated", new Text(ClientRMProxy.getAMRMTokenService(conf)), ugiToken.getService());
}
19
View Complete Implementation : TestDistributedScheduler.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
private RequestInterceptor setup(Configuration conf, DistributedScheduler distributedScheduler) {
NodeStatusUpdater nodeStatusUpdater = Mockito.mock(NodeStatusUpdater.clreplaced);
Mockito.when(nodeStatusUpdater.getRMIdentifier()).thenReturn(12345l);
NMContainerTokenSecretManager nmContainerTokenSecretManager = new NMContainerTokenSecretManager(conf);
MasterKey mKey = new MasterKey() {
@Override
public int getKeyId() {
return 1;
}
@Override
public void setKeyId(int keyId) {
}
@Override
public ByteBuffer getBytes() {
return ByteBuffer.allocate(8);
}
@Override
public void setBytes(ByteBuffer bytes) {
}
};
nmContainerTokenSecretManager.setMasterKey(mKey);
OpportunisticContainerAllocator containerAllocator = new DistributedOpportunisticContainerAllocator(nmContainerTokenSecretManager);
NMTokenSecretManagerInNM nmTokenSecretManagerInNM = new NMTokenSecretManagerInNM();
nmTokenSecretManagerInNM.setMasterKey(mKey);
distributedScheduler.initLocal(1234, ApplicationAttemptId.newInstance(ApplicationId.newInstance(1, 1), 1), containerAllocator, nmTokenSecretManagerInNM, "test");
RequestInterceptor finalReqIntcptr = Mockito.mock(RequestInterceptor.clreplaced);
distributedScheduler.setNextInterceptor(finalReqIntcptr);
return finalReqIntcptr;
}
19
View Complete Implementation : TestCentralizedOpportunisticContainerAllocator.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
/**
* Tests allocation of node local Opportunistic container requests.
* @throws Exception
*/
@Test
public void testNodeLocalAllocation() throws Exception {
List<ResourceRequest> reqs = Arrays.asList(createResourceRequest(1, ResourceRequest.ANY, 1), createResourceRequest(2, "/r1", 1), createResourceRequest(2, "h1", 1), createResourceRequest(2, ResourceRequest.ANY, 1), createResourceRequest(3, "/r1", 1), createResourceRequest(3, "h1", 1), createResourceRequest(3, ResourceRequest.ANY, 1));
ApplicationAttemptId appAttId = ApplicationAttemptId.newInstance(ApplicationId.newInstance(0L, 1), 1);
allocator.setNodeQueueLoadMonitor(createNodeQueueLoadMonitor(3, 2, 5));
List<Container> containers = allocator.allocateContainers(EMPTY_BLACKLIST_REQUEST, reqs, appAttId, oppCntxt, 1L, "user");
LOG.info("Containers: {}", containers);
// all 3 containers should be allocated.
replacedertEquals(3, containers.size());
// container with allocation id 2 and 3 should be allocated on node h1
for (Container c : containers) {
if (c.getAllocationRequestId() == 2 || c.getAllocationRequestId() == 3) {
replacedertEquals("h1:1234", c.getNodeId().toString());
}
}
}
19
View Complete Implementation : TestCentralizedOpportunisticContainerAllocator.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
/**
* Tests off switch allocation of Opportunistic containers.
* @throws Exception
*/
@Test
public void testOffSwitchAllocationWhenNoNodeOrRack() throws Exception {
List<ResourceRequest> reqs = Arrays.asList(createResourceRequest(2, "/r3", 2), createResourceRequest(2, "h6", 2), createResourceRequest(2, ResourceRequest.ANY, 2));
ApplicationAttemptId appAttId = ApplicationAttemptId.newInstance(ApplicationId.newInstance(0L, 1), 1);
NodeQueueLoadMonitor selector = createNodeQueueLoadMonitor(Arrays.asList("h1", "h2", "h3", "h4"), Arrays.asList("/r2", "/r1", "/r2", "/r1"), Arrays.asList(4, 4, 4, 4), Arrays.asList(5, 5, 5, 5));
allocator.setNodeQueueLoadMonitor(selector);
List<Container> containers = allocator.allocateContainers(EMPTY_BLACKLIST_REQUEST, reqs, appAttId, oppCntxt, 1L, "user");
LOG.info("Containers: {}", containers);
replacedertEquals(2, containers.size());
}
19
View Complete Implementation : TestProtocolRecords.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
@Test
public void testRegisterNodeManagerRequest() {
ApplicationId appId = ApplicationId.newInstance(123456789, 1);
ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(appId, 1);
ContainerId containerId = ContainerId.newContainerId(attemptId, 1);
NMContainerStatus containerReport = NMContainerStatus.newInstance(containerId, 0, ContainerState.RUNNING, Resource.newInstance(1024, 1), "diagnostics", 0, Priority.newInstance(10), 1234);
List<NMContainerStatus> reports = Arrays.asList(containerReport);
RegisterNodeManagerRequest request = RegisterNodeManagerRequest.newInstance(NodeId.newInstance("1.1.1.1", 1000), 8080, Resource.newInstance(1024, 1), "NM-version-id", reports, Arrays.asList(appId));
RegisterNodeManagerRequest requestProto = new RegisterNodeManagerRequestPBImpl(((RegisterNodeManagerRequestPBImpl) request).getProto());
replacedert.replacedertEquals(containerReport, requestProto.getNMContainerStatuses().get(0));
replacedert.replacedertEquals(8080, requestProto.getHttpPort());
replacedert.replacedertEquals("NM-version-id", requestProto.getNMVersion());
replacedert.replacedertEquals(NodeId.newInstance("1.1.1.1", 1000), requestProto.getNodeId());
replacedert.replacedertEquals(Resource.newInstance(1024, 1), requestProto.getResource());
replacedert.replacedertEquals(1, requestProto.getRunningApplications().size());
replacedert.replacedertEquals(appId, requestProto.getRunningApplications().get(0));
}
19
View Complete Implementation : TestProtocolRecords.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
@Test
public void testNMContainerStatus() {
ApplicationId appId = ApplicationId.newInstance(123456789, 1);
ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(appId, 1);
ContainerId containerId = ContainerId.newContainerId(attemptId, 1);
Resource resource = Resource.newInstance(1000, 200);
NMContainerStatus report = NMContainerStatus.newInstance(containerId, 0, ContainerState.COMPLETE, resource, "diagnostics", ContainerExitStatus.ABORTED, Priority.newInstance(10), 1234);
NMContainerStatus reportProto = new NMContainerStatusPBImpl(((NMContainerStatusPBImpl) report).getProto());
replacedert.replacedertEquals("diagnostics", reportProto.getDiagnostics());
replacedert.replacedertEquals(resource, reportProto.getAllocatedResource());
replacedert.replacedertEquals(ContainerExitStatus.ABORTED, reportProto.getContainerExitStatus());
replacedert.replacedertEquals(ContainerState.COMPLETE, reportProto.getContainerState());
replacedert.replacedertEquals(containerId, reportProto.getContainerId());
replacedert.replacedertEquals(Priority.newInstance(10), reportProto.getPriority());
replacedert.replacedertEquals(1234, reportProto.getCreationTime());
}
19
View Complete Implementation : TestDistributedOpportunisticContainerAllocator.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
@Test
public void testBlacklistRejection() throws Exception {
ResourceBlacklistRequest blacklistRequest = ResourceBlacklistRequest.newInstance(Arrays.asList("h1", "h2"), new ArrayList<>());
List<ResourceRequest> reqs = Arrays.asList(ResourceRequest.newInstance(PRIORITY_NORMAL, "*", CAPABILITY_1GB, 1, true, null, OPPORTUNISTIC_REQ));
ApplicationAttemptId appAttId = ApplicationAttemptId.newInstance(ApplicationId.newInstance(0L, 1), 1);
oppCntxt.updateNodeList(Arrays.asList(RemoteNode.newInstance(NodeId.newInstance("h1", 1234), "h1:1234", "/r1"), RemoteNode.newInstance(NodeId.newInstance("h2", 1234), "h2:1234", "/r2")));
List<Container> containers = allocator.allocateContainers(blacklistRequest, reqs, appAttId, oppCntxt, 1L, "luser");
replacedert.replacedertEquals(0, containers.size());
replacedert.replacedertEquals(1, oppCntxt.getOutstandingOpReqs().size());
}
19
View Complete Implementation : TestApplicationHistoryManagerImpl.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
@Test
public void testApplicationReport() throws IOException, YarnException {
ApplicationId appId = null;
appId = ApplicationId.newInstance(0, 1);
writeApplicationStartData(appId);
writeApplicationFinishData(appId);
ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(appId, 1);
writeApplicationAttemptStartData(appAttemptId);
writeApplicationAttemptFinishData(appAttemptId);
ApplicationReport appReport = applicationHistoryManagerImpl.getApplication(appId);
replacedert.replacedertNotNull(appReport);
replacedert.replacedertEquals(appId, appReport.getApplicationId());
replacedert.replacedertEquals(appAttemptId, appReport.getCurrentApplicationAttemptId());
replacedert.replacedertEquals(appAttemptId.toString(), appReport.getHost());
replacedert.replacedertEquals("test type", appReport.getApplicationType().toString());
replacedert.replacedertEquals("test queue", appReport.getQueue().toString());
}
19
View Complete Implementation : TestNMTimelinePublisher.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
@Test
public void testContainerResourceUsage() {
ApplicationId appId = ApplicationId.newInstance(0, 1);
publisher.createTimelineClient(appId);
Container aContainer = mock(Container.clreplaced);
when(aContainer.getContainerId()).thenReturn(ContainerId.newContainerId(ApplicationAttemptId.newInstance(appId, 1), 0L));
long idPrefix = TimelineServiceHelper.invertLong(aContainer.getContainerId().getContainerId());
publisher.reportContainerResourceUsage(aContainer, 1024L, 8F);
verifyPublishedResourceUsageMetrics(timelineClient, 1024L, 8, idPrefix);
timelineClient.reset();
publisher.reportContainerResourceUsage(aContainer, 1024L, 0.8F);
verifyPublishedResourceUsageMetrics(timelineClient, 1024L, 1, idPrefix);
timelineClient.reset();
publisher.reportContainerResourceUsage(aContainer, 1024L, 0.49F);
verifyPublishedResourceUsageMetrics(timelineClient, 1024L, 0, idPrefix);
timelineClient.reset();
publisher.reportContainerResourceUsage(aContainer, 1024L, (float) ResourceCalculatorProcessTree.UNAVAILABLE);
verifyPublishedResourceUsageMetrics(timelineClient, 1024L, ResourceCalculatorProcessTree.UNAVAILABLE, idPrefix);
}
19
View Complete Implementation : TestAHSWebApp.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
ApplicationHistoryClientService mockApplicationHistoryClientService(int numApps, int numAppAttempts, int numContainers) throws Exception {
ApplicationHistoryManager ahManager = new MockApplicationHistoryManagerImpl(store);
ApplicationHistoryClientService historyClientService = new ApplicationHistoryClientService(ahManager);
for (int i = 1; i <= numApps; ++i) {
ApplicationId appId = ApplicationId.newInstance(0, i);
writeApplicationStartData(appId);
for (int j = 1; j <= numAppAttempts; ++j) {
ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(appId, j);
writeApplicationAttemptStartData(appAttemptId);
for (int k = 1; k <= numContainers; ++k) {
ContainerId containerId = ContainerId.newContainerId(appAttemptId, k);
writeContainerStartData(containerId);
writeContainerFinishData(containerId);
}
writeApplicationAttemptFinishData(appAttemptId);
}
writeApplicationFinishData(appId);
}
return historyClientService;
}
19
View Complete Implementation : TestNMTimelinePublisher.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
@Test
public void testPublishContainerKilledEvent() {
ApplicationId appId = ApplicationId.newInstance(0, 1);
ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(appId, 1);
ContainerId cId = ContainerId.newContainerId(appAttemptId, 1);
ContainerEvent containerEvent = new ContainerKillEvent(cId, 1, "test kill");
publisher.createTimelineClient(appId);
publisher.publishContainerEvent(containerEvent);
publisher.stopTimelineClient(appId);
dispatcher.await();
ContainerEnreplacedy cEnreplacedy = new ContainerEnreplacedy();
cEnreplacedy.setId(cId.toString());
TimelineEnreplacedy[] lastPublishedEnreplacedies = timelineClient.getLastPublishedEnreplacedies();
replacedert.replacedertNotNull(lastPublishedEnreplacedies);
replacedert.replacedertEquals(1, lastPublishedEnreplacedies.length);
TimelineEnreplacedy enreplacedy = lastPublishedEnreplacedies[0];
replacedert.replacedertEquals(cEnreplacedy, enreplacedy);
NavigableSet<TimelineEvent> events = enreplacedy.getEvents();
replacedert.replacedertEquals(1, events.size());
replacedert.replacedertEquals(ContainerMetricsConstants.KILLED_EVENT_TYPE, events.iterator().next().getId());
Map<String, Object> info = enreplacedy.getInfo();
replacedert.replacedertTrue(info.containsKey(ContainerMetricsConstants.DIAGNOSTICS_INFO));
replacedert.replacedertEquals("test kill", info.get(ContainerMetricsConstants.DIAGNOSTICS_INFO));
replacedert.replacedertTrue(info.containsKey(ContainerMetricsConstants.EXIT_STATUS_INFO));
replacedert.replacedertEquals(1, info.get(ContainerMetricsConstants.EXIT_STATUS_INFO));
}
19
View Complete Implementation : TestCentralizedOpportunisticContainerAllocator.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
/**
* Tests maximum opportunistic container allocation per AM heartbeat for
* allocation requests with different scheduler key.
* @throws Exception
*/
@Test
public void testMaxAllocationsPerAMHeartbeatDifferentSchedKey() throws Exception {
allocator.setMaxAllocationsPerAMHeartbeat(2);
List<ResourceRequest> reqs = Arrays.asList(createResourceRequest(1, ResourceRequest.ANY, 1), createResourceRequest(2, "h6", 2), createResourceRequest(3, "/r3", 2));
ApplicationAttemptId appAttId = ApplicationAttemptId.newInstance(ApplicationId.newInstance(0L, 1), 1);
allocator.setNodeQueueLoadMonitor(createNodeQueueLoadMonitor(3, 2, 5));
List<Container> containers = allocator.allocateContainers(EMPTY_BLACKLIST_REQUEST, reqs, appAttId, oppCntxt, 1L, "user");
LOG.info("Containers: {}", containers);
// Although capacity is present, but only 2 containers should be allocated
// as max allocation per AM heartbeat is set to 2.
replacedertEquals(2, containers.size());
containers = allocator.allocateContainers(EMPTY_BLACKLIST_REQUEST, new ArrayList<>(), appAttId, oppCntxt, 1L, "user");
LOG.info("Containers: {}", containers);
// 2 more containers should be allocated from pending allocation requests.
replacedertEquals(2, containers.size());
containers = allocator.allocateContainers(EMPTY_BLACKLIST_REQUEST, new ArrayList<>(), appAttId, oppCntxt, 1L, "user");
LOG.info("Containers: {}", containers);
// Remaining 1 container should be allocated.
replacedertEquals(1, containers.size());
}
19
View Complete Implementation : TestAHSv2ClientImpl.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
@Test
public void testGetAppAttemptReport() throws IOException, YarnException {
final ApplicationId appId = ApplicationId.newInstance(0, 1);
final ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(appId, 1);
when(spyTimelineReaderClient.getApplicationAttemptEnreplacedy(appAttemptId, "ALL", null)).thenReturn(createAppAttemptTimelineEnreplacedy(appAttemptId));
ApplicationAttemptReport report = client.getApplicationAttemptReport(appAttemptId);
replacedertThat(report.getApplicationAttemptId()).isEqualTo(appAttemptId);
replacedertThat(report.getFinishTime()).isEqualTo(Integer.MAX_VALUE + 2L);
replacedertThat(report.getOriginalTrackingUrl()).isEqualTo("test original tracking url");
}
19
View Complete Implementation : TestDistributedOpportunisticContainerAllocator.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
@Test
public void testSimpleRackLocalAllocation() throws Exception {
ResourceBlacklistRequest blacklistRequest = ResourceBlacklistRequest.newInstance(new ArrayList<>(), new ArrayList<>());
List<ResourceRequest> reqs = Arrays.asList(ResourceRequest.newInstance(PRIORITY_NORMAL, "*", CAPABILITY_1GB, 1, true, null, OPPORTUNISTIC_REQ), ResourceRequest.newInstance(PRIORITY_NORMAL, "h1", CAPABILITY_1GB, 1, true, null, OPPORTUNISTIC_REQ), ResourceRequest.newInstance(PRIORITY_NORMAL, "/r1", CAPABILITY_1GB, 1, true, null, OPPORTUNISTIC_REQ));
ApplicationAttemptId appAttId = ApplicationAttemptId.newInstance(ApplicationId.newInstance(0L, 1), 1);
oppCntxt.updateNodeList(Arrays.asList(RemoteNode.newInstance(NodeId.newInstance("h3", 1234), "h3:1234", "/r2"), RemoteNode.newInstance(NodeId.newInstance("h2", 1234), "h2:1234", "/r1"), RemoteNode.newInstance(NodeId.newInstance("h4", 1234), "h4:1234", "/r2")));
List<Container> containers = allocator.allocateContainers(blacklistRequest, reqs, appAttId, oppCntxt, 1L, "luser");
Set<String> allocatedHosts = new HashSet<>();
for (Container c : containers) {
allocatedHosts.add(c.getNodeHttpAddress());
}
replacedert.replacedertTrue(allocatedHosts.contains("h2:1234"));
replacedert.replacedertFalse(allocatedHosts.contains("h3:1234"));
replacedert.replacedertFalse(allocatedHosts.contains("h4:1234"));
replacedert.replacedertEquals(1, containers.size());
}
19
View Complete Implementation : TestFileSystemApplicationHistoryStore.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
@Test
public void testMreplacediveWriteContainerHistoryData() throws IOException {
LOG.info("Starting testMreplacediveWriteContainerHistoryData");
long mb = 1024 * 1024;
long usedDiskBefore = fs.getContentSummary(fsWorkingPath).getLength() / mb;
ApplicationId appId = ApplicationId.newInstance(0, 1);
writeApplicationStartData(appId);
ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(appId, 1);
for (int i = 1; i <= 100000; ++i) {
ContainerId containerId = ContainerId.newContainerId(appAttemptId, i);
writeContainerStartData(containerId);
writeContainerFinishData(containerId);
}
writeApplicationFinishData(appId);
long usedDiskAfter = fs.getContentSummary(fsWorkingPath).getLength() / mb;
replacedert.replacedertTrue((usedDiskAfter - usedDiskBefore) < 20);
}
19
View Complete Implementation : TestRegisterNodeManagerRequest.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
@Test
public void testRegisterNodeManagerRequest() {
RegisterNodeManagerRequest request = RegisterNodeManagerRequest.newInstance(NodeId.newInstance("host", 1234), 1234, Resource.newInstance(0, 0), "version", Arrays.asList(NMContainerStatus.newInstance(ContainerId.newContainerId(ApplicationAttemptId.newInstance(ApplicationId.newInstance(1234L, 1), 1), 1), 0, ContainerState.RUNNING, Resource.newInstance(1024, 1), "good", -1, Priority.newInstance(0), 1234)), Arrays.asList(ApplicationId.newInstance(1234L, 1), ApplicationId.newInstance(1234L, 2)));
// serialze to proto, and get request from proto
RegisterNodeManagerRequest request1 = new RegisterNodeManagerRequestPBImpl(((RegisterNodeManagerRequestPBImpl) request).getProto());
// check values
replacedert.replacedertEquals(request1.getNMContainerStatuses().size(), request.getNMContainerStatuses().size());
replacedert.replacedertEquals(request1.getNMContainerStatuses().get(0).getContainerId(), request.getNMContainerStatuses().get(0).getContainerId());
replacedert.replacedertEquals(request1.getRunningApplications().size(), request.getRunningApplications().size());
replacedert.replacedertEquals(request1.getRunningApplications().get(0), request.getRunningApplications().get(0));
replacedert.replacedertEquals(request1.getRunningApplications().get(1), request.getRunningApplications().get(1));
}
19
View Complete Implementation : TestPlacementConstraintsUtil.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
@Test
public void testRackAntiAffinityreplacedignment() throws InvalidAllocationTagsQueryException {
AllocationTagsManager tm = new AllocationTagsManager(rmContext);
PlacementConstraintManagerService pcm = new MemoryPlacementConstraintManager();
// Register App1 with anti-affinity constraint map
pcm.registerApplication(appId1, constraintMap2);
/**
* Place container:
* Node0:123 (Rack1):
* container_app1_1 (hbase-rs)
*/
RMNode n0_r1 = rmNodes.get(0);
RMNode n1_r1 = rmNodes.get(1);
RMNode n2_r2 = rmNodes.get(2);
RMNode n3_r2 = rmNodes.get(3);
// 1 Containers on Node0-Rack1 with allocationTag 'hbase-rs'
ContainerId hbase_m = ContainerId.newContainerId(ApplicationAttemptId.newInstance(appId1, 0), 0);
tm.addContainer(n0_r1.getNodeID(), hbase_m, ImmutableSet.of("hbase-rs"));
SchedulerNode schedulerNode0 = newSchedulerNode(n0_r1.getHostName(), n0_r1.getRackName(), n0_r1.getNodeID());
SchedulerNode schedulerNode1 = newSchedulerNode(n1_r1.getHostName(), n1_r1.getRackName(), n1_r1.getNodeID());
SchedulerNode schedulerNode2 = newSchedulerNode(n2_r2.getHostName(), n2_r2.getRackName(), n2_r2.getNodeID());
SchedulerNode schedulerNode3 = newSchedulerNode(n3_r2.getHostName(), n3_r2.getRackName(), n3_r2.getNodeID());
// 'zk' placement on Rack1 should FAIL
replacedert.replacedertFalse(PlacementConstraintsUtil.canSatisfyConstraints(appId1, createSchedulingRequest(sourceTag2), schedulerNode0, pcm, tm));
replacedert.replacedertFalse(PlacementConstraintsUtil.canSatisfyConstraints(appId1, createSchedulingRequest(sourceTag2), schedulerNode1, pcm, tm));
// SUCCEED on the rest of the RACKs
replacedert.replacedertTrue(PlacementConstraintsUtil.canSatisfyConstraints(appId1, createSchedulingRequest(sourceTag2), schedulerNode2, pcm, tm));
replacedert.replacedertTrue(PlacementConstraintsUtil.canSatisfyConstraints(appId1, createSchedulingRequest(sourceTag2), schedulerNode3, pcm, tm));
}
19
View Complete Implementation : TestLogAggregationIndexedFileController.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
@Before
public void setUp() throws IOException {
setConf(getTestConf());
appId = ApplicationId.newInstance(123456, 1);
ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(appId, 1);
containerId = ContainerId.newContainerId(attemptId, 1);
nodeId = NodeId.newInstance("localhost", 9999);
fs = FileSystem.get(getConf());
sysOutStream = new ByteArrayOutputStream();
PrintStream sysOut = new PrintStream(sysOutStream);
System.setOut(sysOut);
ByteArrayOutputStream sysErrStream = new ByteArrayOutputStream();
PrintStream sysErr = new PrintStream(sysErrStream);
System.setErr(sysErr);
}
19
View Complete Implementation : TestDistributedOpportunisticContainerAllocator.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
@Test
public void testNodeLocalAllocationSameSchedKey() throws Exception {
ResourceBlacklistRequest blacklistRequest = ResourceBlacklistRequest.newInstance(new ArrayList<>(), new ArrayList<>());
List<ResourceRequest> reqs = Arrays.asList(ResourceRequest.newBuilder().allocationRequestId(2).numContainers(2).priority(PRIORITY_NORMAL).resourceName("/r1").capability(CAPABILITY_1GB).relaxLocality(true).executionType(ExecutionType.OPPORTUNISTIC).build(), ResourceRequest.newBuilder().allocationRequestId(2).numContainers(2).priority(PRIORITY_NORMAL).resourceName("h1").capability(CAPABILITY_1GB).relaxLocality(true).executionType(ExecutionType.OPPORTUNISTIC).build(), ResourceRequest.newBuilder().allocationRequestId(2).numContainers(2).priority(PRIORITY_NORMAL).resourceName(ResourceRequest.ANY).capability(CAPABILITY_1GB).relaxLocality(true).executionType(ExecutionType.OPPORTUNISTIC).build());
ApplicationAttemptId appAttId = ApplicationAttemptId.newInstance(ApplicationId.newInstance(0L, 1), 1);
oppCntxt.updateNodeList(Arrays.asList(RemoteNode.newInstance(NodeId.newInstance("h1", 1234), "h1:1234", "/r1"), RemoteNode.newInstance(NodeId.newInstance("h2", 1234), "h2:1234", "/r1"), RemoteNode.newInstance(NodeId.newInstance("h3", 1234), "h3:1234", "/r1")));
List<Container> containers = allocator.allocateContainers(blacklistRequest, reqs, appAttId, oppCntxt, 1L, "luser");
LOG.info("Containers: {}", containers);
Set<String> allocatedHosts = new HashSet<>();
for (Container c : containers) {
allocatedHosts.add(c.getNodeHttpAddress());
}
replacedert.replacedertEquals(2, containers.size());
replacedert.replacedertTrue(allocatedHosts.contains("h1:1234"));
replacedert.replacedertFalse(allocatedHosts.contains("h2:1234"));
replacedert.replacedertFalse(allocatedHosts.contains("h3:1234"));
}