Here are the examples of the java api org.apache.hadoop.yarn.api.records.Priority taken from open source projects. By voting up you can indicate which examples are most useful and appropriate.
155 Examples
19
View Complete Implementation : UpdateApplicationPriorityResponsePBImpl.java
Copyright Apache License 2.0
Author : aliyun-beta
Copyright Apache License 2.0
Author : aliyun-beta
private PriorityProto convertToProtoFormat(Priority t) {
return ((PriorityPBImpl) t).getProto();
}
19
View Complete Implementation : SchedulerApplicationAttempt.java
Copyright Apache License 2.0
Author : aliyun-beta
Copyright Apache License 2.0
Author : aliyun-beta
public synchronized int getTotalRequiredResources(Priority priority) {
return getResourceRequest(priority, ResourceRequest.ANY).getNumContainers();
}
19
View Complete Implementation : SchedulerApplicationAttempt.java
Copyright Apache License 2.0
Author : aliyun-beta
Copyright Apache License 2.0
Author : aliyun-beta
/**
* Should be called when an application has successfully scheduled a container,
* or when the scheduling locality threshold is relaxed.
* Reset various internal counters which affect delay scheduling
*
* @param priority The priority of the container scheduled.
*/
public synchronized void resetSchedulingOpportunities(Priority priority) {
resetSchedulingOpportunities(priority, System.currentTimeMillis());
}
19
View Complete Implementation : AllocateResponse.java
Copyright Apache License 2.0
Author : aliyun-beta
Copyright Apache License 2.0
Author : aliyun-beta
@Private
@Unstable
public abstract void setApplicationPriority(Priority priority);
19
View Complete Implementation : ContainerHistoryData.java
Copyright Apache License 2.0
Author : aliyun-beta
Copyright Apache License 2.0
Author : aliyun-beta
@Public
@Unstable
public static ContainerHistoryData newInstance(ContainerId containerId, Resource allocatedResource, NodeId replacedignedNode, Priority priority, long startTime, long finishTime, String diagnosticsInfo, int containerExitCode, ContainerState containerState) {
ContainerHistoryData containerHD = new ContainerHistoryData();
containerHD.setContainerId(containerId);
containerHD.setAllocatedResource(allocatedResource);
containerHD.setreplacedignedNode(replacedignedNode);
containerHD.setPriority(priority);
containerHD.setStartTime(startTime);
containerHD.setFinishTime(finishTime);
containerHD.setDiagnosticsInfo(diagnosticsInfo);
containerHD.setContainerExitStatus(containerExitCode);
containerHD.setContainerState(containerState);
return containerHD;
}
19
View Complete Implementation : FiCaSchedulerApp.java
Copyright Apache License 2.0
Author : aliyun-beta
Copyright Apache License 2.0
Author : aliyun-beta
public synchronized boolean unreserve(Priority priority, FiCaSchedulerNode node, RMContainer rmContainer) {
// Cancel increase request (if it has reserved increase request
rmContainer.cancelIncreaseReservation();
// Done with the reservation?
if (internalUnreserve(node, priority)) {
node.unreserveResource(this);
// Update reserved metrics
queue.getMetrics().unreserveResource(getUser(), rmContainer.getReservedResource());
return true;
}
return false;
}
19
View Complete Implementation : ContainerStartDataPBImpl.java
Copyright Apache License 2.0
Author : aliyun-beta
Copyright Apache License 2.0
Author : aliyun-beta
private PriorityProto convertToProtoFormat(Priority priority) {
return ((PriorityPBImpl) priority).getProto();
}
19
View Complete Implementation : AppSchedulingInfo.java
Copyright Apache License 2.0
Author : aliyun-beta
Copyright Apache License 2.0
Author : aliyun-beta
synchronized private void checkForDeactivation() {
boolean deactivate = true;
for (Priority priority : getPriorities()) {
ResourceRequest request = getResourceRequest(priority, ResourceRequest.ANY);
if (request != null) {
if (request.getNumContainers() > 0) {
deactivate = false;
break;
}
}
}
// also we need to check increase request
if (!deactivate) {
deactivate = increaseRequestMap.isEmpty();
}
if (deactivate) {
activeUsersManager.deactivateApplication(user, applicationId);
}
}
19
View Complete Implementation : UpdateApplicationPriorityRequestPBImpl.java
Copyright Apache License 2.0
Author : aliyun-beta
Copyright Apache License 2.0
Author : aliyun-beta
private PriorityProto convertToProtoFormat(Priority t) {
return ((PriorityPBImpl) t).getProto();
}
19
View Complete Implementation : AppSchedulingInfo.java
Copyright Apache License 2.0
Author : aliyun-beta
Copyright Apache License 2.0
Author : aliyun-beta
public synchronized void increaseContainer(SchedContainerChangeRequest increaseRequest) {
NodeId nodeId = increaseRequest.getNodeId();
Priority priority = increaseRequest.getPriority();
ContainerId containerId = increaseRequest.getContainerId();
if (LOG.isDebugEnabled()) {
LOG.debug("allocated increase request : applicationId=" + applicationId + " container=" + containerId + " host=" + increaseRequest.getNodeId() + " user=" + user + " resource=" + increaseRequest.getDeltaCapacity());
}
// Set queue metrics
queue.getMetrics().allocateResources(user, 0, increaseRequest.getDeltaCapacity(), true);
// remove the increase request from pending increase request map
removeIncreaseRequest(nodeId, priority, containerId);
// update usage
appResourceUsage.incUsed(increaseRequest.getNodeParreplacedion(), increaseRequest.getDeltaCapacity());
}
19
View Complete Implementation : SchedulerApplicationAttempt.java
Copyright Apache License 2.0
Author : aliyun-beta
Copyright Apache License 2.0
Author : aliyun-beta
public synchronized boolean removeIncreaseRequest(NodeId nodeId, Priority priority, ContainerId containerId) {
return appSchedulingInfo.removeIncreaseRequest(nodeId, priority, containerId);
}
19
View Complete Implementation : FSAppAttempt.java
Copyright Apache License 2.0
Author : aliyun-beta
Copyright Apache License 2.0
Author : aliyun-beta
/**
* Whether this app has containers requests that could be satisfied on the
* given node, if the node had full space.
*/
private boolean hasContainerForNode(Priority prio, FSSchedulerNode node) {
ResourceRequest anyRequest = getResourceRequest(prio, ResourceRequest.ANY);
ResourceRequest rackRequest = getResourceRequest(prio, node.getRackName());
ResourceRequest nodeRequest = getResourceRequest(prio, node.getNodeName());
return // There must be outstanding requests at the given priority:
anyRequest != null && anyRequest.getNumContainers() > 0 && // If locality relaxation is turned off at *-level, there must be a
// non-zero request for the node's rack:
(anyRequest.getRelaxLocality() || (rackRequest != null && rackRequest.getNumContainers() > 0)) && // If locality relaxation is turned off at rack-level, there must be a
// non-zero request at the node:
(rackRequest == null || rackRequest.getRelaxLocality() || (nodeRequest != null && nodeRequest.getNumContainers() > 0)) && // The requested container must be able to fit on the node:
Resources.lessThanOrEqual(RESOURCE_CALCULATOR, null, anyRequest.getCapability(), node.getRMNode().getTotalCapability()) && // The requested container must fit in queue maximum share:
getQueue().fitsInMaxShare(anyRequest.getCapability());
}
19
View Complete Implementation : FSAppAttempt.java
Copyright Apache License 2.0
Author : aliyun-beta
Copyright Apache License 2.0
Author : aliyun-beta
/**
* Remove the reservation on {@code node} at the given {@link Priority}.
* This dispatches SchedulerNode handlers as well.
*/
public void unreserve(Priority priority, FSSchedulerNode node) {
RMContainer rmContainer = node.getReservedContainer();
unreserveInternal(priority, node);
node.unreserveResource(this);
clearReservation(node);
getMetrics().unreserveResource(getUser(), rmContainer.getContainer().getResource());
}
19
View Complete Implementation : SchedulerApplicationAttempt.java
Copyright Apache License 2.0
Author : aliyun-beta
Copyright Apache License 2.0
Author : aliyun-beta
public synchronized void resetMissedNonParreplacedionedRequestSchedulingOpportunity(Priority priority) {
missedNonParreplacedionedRequestSchedulingOpportunity.setCount(priority, 0);
}
19
View Complete Implementation : AppSchedulingInfo.java
Copyright Apache License 2.0
Author : aliyun-beta
Copyright Apache License 2.0
Author : aliyun-beta
/**
* The {@link ResourceScheduler} is allocating data-local resources to the
* application.
*
* @param allocatedContainers
* resources allocated to the application
*/
synchronized private void allocateOffSwitch(SchedulerNode node, Priority priority, ResourceRequest offSwitchRequest, Container container, List<ResourceRequest> resourceRequests) {
// Update future requirements
decrementOutstanding(offSwitchRequest);
// Update cloned OffRack requests for recovery
resourceRequests.add(cloneResourceRequest(offSwitchRequest));
}
19
View Complete Implementation : ApplicationReportPBImpl.java
Copyright Apache License 2.0
Author : aliyun-beta
Copyright Apache License 2.0
Author : aliyun-beta
private PriorityProto convertToProtoFormat(Priority t) {
return ((PriorityPBImpl) t).getProto();
}
19
View Complete Implementation : FiCaSchedulerApp.java
Copyright Apache License 2.0
Author : aliyun-beta
Copyright Apache License 2.0
Author : aliyun-beta
public void reserve(Priority priority, FiCaSchedulerNode node, RMContainer rmContainer, Container container) {
// Update reserved metrics if this is the first reservation
if (rmContainer == null) {
queue.getMetrics().reserveResource(getUser(), container.getResource());
}
// Inform the application
rmContainer = super.reserve(node, priority, rmContainer, container);
// Update the node
node.reserveResource(this, priority, rmContainer);
}
19
View Complete Implementation : FiCaSchedulerApp.java
Copyright Apache License 2.0
Author : aliyun-beta
Copyright Apache License 2.0
Author : aliyun-beta
public synchronized float getLocalityWaitFactor(Priority priority, int clusterNodes) {
// Estimate: Required unique resources (i.e. hosts + racks)
int requiredResources = Math.max(this.getResourceRequests(priority).size() - 1, 0);
// waitFactor can't be more than '1'
// i.e. no point skipping more than clustersize opportunities
return Math.min(((float) requiredResources / clusterNodes), 1.0f);
}
19
View Complete Implementation : TestFSAppAttempt.java
Copyright Apache License 2.0
Author : aliyun-beta
Copyright Apache License 2.0
Author : aliyun-beta
@Test
public /**
* Ensure that when negative paramaters are given (signaling delay scheduling
* no tin use), the least restrictive locality level is returned.
*/
void testLocalityLevelWithoutDelays() {
FSLeafQueue queue = Mockito.mock(FSLeafQueue.clreplaced);
Priority prio = Mockito.mock(Priority.clreplaced);
Mockito.when(prio.getPriority()).thenReturn(1);
RMContext rmContext = resourceManager.getRMContext();
ApplicationAttemptId applicationAttemptId = createAppAttemptId(1, 1);
FSAppAttempt schedulerApp = new FSAppAttempt(scheduler, applicationAttemptId, "user1", queue, null, rmContext);
replacedertEquals(NodeType.OFF_SWITCH, schedulerApp.getAllowedLocalityLevel(prio, 10, -1.0, -1.0));
}
19
View Complete Implementation : UpdateApplicationPriorityResponse.java
Copyright Apache License 2.0
Author : aliyun-beta
Copyright Apache License 2.0
Author : aliyun-beta
/**
* Set the <code>Priority</code> of the application.
*
* @param priority <code>Priority</code> of the application
*/
public abstract void setApplicationPriority(Priority priority);
19
View Complete Implementation : YarnClient.java
Copyright Apache License 2.0
Author : aliyun-beta
Copyright Apache License 2.0
Author : aliyun-beta
/**
* <p>
* The interface used by client to set priority of an application
* </p>
* @param applicationId
* @param priority
* @return updated priority of an application.
* @throws YarnException
* @throws IOException
*/
@Public
@Unstable
public abstract Priority updateApplicationPriority(ApplicationId applicationId, Priority priority) throws YarnException, IOException;
19
View Complete Implementation : SchedulerApplicationAttempt.java
Copyright Apache License 2.0
Author : aliyun-beta
Copyright Apache License 2.0
Author : aliyun-beta
public synchronized RMContainer reserve(SchedulerNode node, Priority priority, RMContainer rmContainer, Container container) {
// Create RMContainer if necessary
if (rmContainer == null) {
rmContainer = new RMContainerImpl(container, getApplicationAttemptId(), node.getNodeID(), appSchedulingInfo.getUser(), rmContext);
attemptResourceUsage.incReserved(node.getParreplacedion(), container.getResource());
// Reset the re-reservation count
resetReReservations(priority);
} else {
// Note down the re-reservation
addReReservation(priority);
}
commonReserve(node, priority, rmContainer, container.getResource());
return rmContainer;
}
19
View Complete Implementation : RMContainerTokenSecretManager.java
Copyright Apache License 2.0
Author : aliyun-beta
Copyright Apache License 2.0
Author : aliyun-beta
/**
* Helper function for creating ContainerTokens
*
* @param containerId
* @param nodeId
* @param appSubmitter
* @param capability
* @param priority
* @param createTime
* @param logAggregationContext
* @param nodeLabelExpression
* @param containerType
* @return the container-token
*/
public Token createContainerToken(ContainerId containerId, NodeId nodeId, String appSubmitter, Resource capability, Priority priority, long createTime, LogAggregationContext logAggregationContext, String nodeLabelExpression, ContainerType containerType) {
byte[] preplacedword;
ContainerTokenIdentifier tokenIdentifier;
long expiryTimeStamp = System.currentTimeMillis() + containerTokenExpiryInterval;
// Lock so that we use the same MasterKey's keyId and its bytes
this.readLock.lock();
try {
tokenIdentifier = new ContainerTokenIdentifier(containerId, nodeId.toString(), appSubmitter, capability, expiryTimeStamp, this.currentMasterKey.getMasterKey().getKeyId(), ResourceManager.getClusterTimeStamp(), priority, createTime, logAggregationContext, nodeLabelExpression, containerType);
preplacedword = this.createPreplacedword(tokenIdentifier);
} finally {
this.readLock.unlock();
}
return BuilderUtils.newContainerToken(nodeId, preplacedword, tokenIdentifier);
}
19
View Complete Implementation : FSAppAttempt.java
Copyright Apache License 2.0
Author : aliyun-beta
Copyright Apache License 2.0
Author : aliyun-beta
/**
* Reserve a spot for {@code container} on this {@code node}. If
* the container is {@code alreadyReserved} on the node, simply
* update relevant bookeeping. This dispatches ro relevant handlers
* in {@link FSSchedulerNode}..
* return whether reservation was possible with the current threshold limits
*/
private boolean reserve(Priority priority, FSSchedulerNode node, Container container, NodeType type, boolean alreadyReserved) {
if (!reservationExceedsThreshold(node, type)) {
LOG.info("Making reservation: node=" + node.getNodeName() + " app_id=" + getApplicationId());
if (!alreadyReserved) {
getMetrics().reserveResource(getUser(), container.getResource());
RMContainer rmContainer = super.reserve(node, priority, null, container);
node.reserveResource(this, priority, rmContainer);
setReservation(node);
} else {
RMContainer rmContainer = node.getReservedContainer();
super.reserve(node, priority, rmContainer, container);
node.reserveResource(this, priority, rmContainer);
setReservation(node);
}
return true;
}
return false;
}
19
View Complete Implementation : RMContainerReservedEvent.java
Copyright Apache License 2.0
Author : aliyun-beta
Copyright Apache License 2.0
Author : aliyun-beta
/**
* The event signifying that a container has been reserved.
*
* The event encapsulates information on the amount of reservation
* and the node on which the reservation is in effect.
*/
public clreplaced RMContainerReservedEvent extends RMContainerEvent {
private final Resource reservedResource;
private final NodeId reservedNode;
private final Priority reservedPriority;
public RMContainerReservedEvent(ContainerId containerId, Resource reservedResource, NodeId reservedNode, Priority reservedPriority) {
super(containerId, RMContainerEventType.RESERVED);
this.reservedResource = reservedResource;
this.reservedNode = reservedNode;
this.reservedPriority = reservedPriority;
}
public Resource getReservedResource() {
return reservedResource;
}
public NodeId getReservedNode() {
return reservedNode;
}
public Priority getReservedPriority() {
return reservedPriority;
}
}
19
View Complete Implementation : SchedulerApplicationAttempt.java
Copyright Apache License 2.0
Author : aliyun-beta
Copyright Apache License 2.0
Author : aliyun-beta
public synchronized void addSchedulingOpportunity(Priority priority) {
schedulingOpportunities.setCount(priority, schedulingOpportunities.count(priority) + 1);
}
19
View Complete Implementation : ContainerStartData.java
Copyright Apache License 2.0
Author : aliyun-beta
Copyright Apache License 2.0
Author : aliyun-beta
@Public
@Unstable
public static ContainerStartData newInstance(ContainerId containerId, Resource allocatedResource, NodeId replacedignedNode, Priority priority, long startTime) {
ContainerStartData containerSD = Records.newRecord(ContainerStartData.clreplaced);
containerSD.setContainerId(containerId);
containerSD.setAllocatedResource(allocatedResource);
containerSD.setreplacedignedNode(replacedignedNode);
containerSD.setPriority(priority);
containerSD.setStartTime(startTime);
return containerSD;
}
19
View Complete Implementation : ApplicationCLI.java
Copyright Apache License 2.0
Author : aliyun-beta
Copyright Apache License 2.0
Author : aliyun-beta
/**
* Updates priority of an application with the given ID.
*/
private void updateApplicationPriority(String applicationId, String priority) throws YarnException, IOException {
ApplicationId appId = ConverterUtils.toApplicationId(applicationId);
Priority newAppPriority = Priority.newInstance(Integer.parseInt(priority));
sysout.println("Updating priority of an aplication " + applicationId);
Priority updateApplicationPriority = client.updateApplicationPriority(appId, newAppPriority);
if (newAppPriority.equals(updateApplicationPriority)) {
sysout.println("Successfully updated the application " + applicationId + " with priority '" + priority + "'");
} else {
sysout.println("Updated priority of an application " + applicationId + " to cluster max priority OR keeping old priority" + " as application is in final states");
}
}
19
View Complete Implementation : ContainerHistoryData.java
Copyright Apache License 2.0
Author : aliyun-beta
Copyright Apache License 2.0
Author : aliyun-beta
@Public
@Unstable
public void setPriority(Priority priority) {
this.priority = priority;
}
19
View Complete Implementation : NMContainerStatus.java
Copyright Apache License 2.0
Author : aliyun-beta
Copyright Apache License 2.0
Author : aliyun-beta
public static NMContainerStatus newInstance(ContainerId containerId, ContainerState containerState, Resource allocatedResource, String diagnostics, int containerExitStatus, Priority priority, long creationTime, String nodeLabelExpression) {
NMContainerStatus status = Records.newRecord(NMContainerStatus.clreplaced);
status.setContainerId(containerId);
status.setContainerState(containerState);
status.setAllocatedResource(allocatedResource);
status.setDiagnostics(diagnostics);
status.setContainerExitStatus(containerExitStatus);
status.setPriority(priority);
status.setCreationTime(creationTime);
status.setNodeLabelExpression(nodeLabelExpression);
return status;
}
19
View Complete Implementation : AMRMClientAsyncImpl.java
Copyright Apache License 2.0
Author : aliyun-beta
Copyright Apache License 2.0
Author : aliyun-beta
public List<? extends Collection<T>> getMatchingRequests(Priority priority, String resourceName, Resource capability) {
return client.getMatchingRequests(priority, resourceName, capability);
}
19
View Complete Implementation : FairSchedulerTestBase.java
Copyright Apache License 2.0
Author : aliyun-beta
Copyright Apache License 2.0
Author : aliyun-beta
protected ResourceRequest createResourceRequest(int memory, int vcores, String host, int priority, int numContainers, boolean relaxLocality) {
ResourceRequest request = recordFactory.newRecordInstance(ResourceRequest.clreplaced);
request.setCapability(BuilderUtils.newResource(memory, vcores));
request.setResourceName(host);
request.setNumContainers(numContainers);
Priority prio = recordFactory.newRecordInstance(Priority.clreplaced);
prio.setPriority(priority);
request.setPriority(prio);
request.setRelaxLocality(relaxLocality);
request.setNodeLabelExpression(RMNodeLabelsManager.NO_LABEL);
return request;
}
19
View Complete Implementation : PartialJob.java
Copyright Apache License 2.0
Author : aliyun-beta
Copyright Apache License 2.0
Author : aliyun-beta
@Override
public void setJobPriority(Priority priority) {
throw new UnsupportedOperationException("Can't set job's priority in history");
}
19
View Complete Implementation : ApplicationUpdatedEvent.java
Copyright Apache License 2.0
Author : aliyun-beta
Copyright Apache License 2.0
Author : aliyun-beta
public clreplaced ApplicationUpdatedEvent extends SystemMetricsEvent {
private ApplicationId appId;
private String queue;
private Priority applicationPriority;
public ApplicationUpdatedEvent(ApplicationId appId, String queue, long updatedTime, Priority applicationPriority) {
super(SystemMetricsEventType.APP_UPDATED, updatedTime);
this.appId = appId;
this.queue = queue;
this.applicationPriority = applicationPriority;
}
@Override
public int hashCode() {
return appId.hashCode();
}
public ApplicationId getApplicationId() {
return appId;
}
public String getQueue() {
return queue;
}
public Priority getApplicationPriority() {
return applicationPriority;
}
}
19
View Complete Implementation : CompletedJob.java
Copyright Apache License 2.0
Author : aliyun-beta
Copyright Apache License 2.0
Author : aliyun-beta
@Override
public void setJobPriority(Priority priority) {
throw new UnsupportedOperationException("Can't set job's priority in history");
}
19
View Complete Implementation : TestYarnApiClasses.java
Copyright Apache License 2.0
Author : aliyun-beta
Copyright Apache License 2.0
Author : aliyun-beta
/**
* Simple test Resource request.
* Test hashCode, equals and compare.
*/
@Test
public void testResourceRequest() {
Resource resource = recordFactory.newRecordInstance(Resource.clreplaced);
Priority priority = recordFactory.newRecordInstance(Priority.clreplaced);
ResourceRequest original = ResourceRequest.newInstance(priority, "localhost", resource, 2);
ResourceRequest copy = ResourceRequest.newInstance(priority, "localhost", resource, 2);
replacedertTrue(original.equals(copy));
replacedertEquals(0, original.compareTo(copy));
replacedertTrue(original.hashCode() == copy.hashCode());
copy.setNumContainers(1);
replacedertFalse(original.equals(copy));
replacedertNotSame(0, original.compareTo(copy));
replacedertFalse(original.hashCode() == copy.hashCode());
}
19
View Complete Implementation : TestSchedulerUtils.java
Copyright Apache License 2.0
Author : aliyun-beta
Copyright Apache License 2.0
Author : aliyun-beta
@Test
public void testComparePriorities() {
Priority high = Priority.newInstance(1);
Priority low = Priority.newInstance(2);
replacedertTrue(high.compareTo(low) > 0);
}
19
View Complete Implementation : RMContainerTokenSecretManager.java
Copyright Apache License 2.0
Author : aliyun-beta
Copyright Apache License 2.0
Author : aliyun-beta
/**
* Helper function for creating ContainerTokens
*
* @param containerId
* @param nodeId
* @param appSubmitter
* @param capability
* @param priority
* @param createTime
* @return the container-token
*/
public Token createContainerToken(ContainerId containerId, NodeId nodeId, String appSubmitter, Resource capability, Priority priority, long createTime) {
return createContainerToken(containerId, nodeId, appSubmitter, capability, priority, createTime, null, null, ContainerType.TASK);
}
19
View Complete Implementation : AbstractYarnScheduler.java
Copyright Apache License 2.0
Author : aliyun-beta
Copyright Apache License 2.0
Author : aliyun-beta
@Override
public void updateApplicationPriority(Priority newPriority, ApplicationId applicationId) throws YarnException {
// Dummy Implementation till Application Priority changes are done in
// specific scheduler.
}
19
View Complete Implementation : RegularContainerAllocator.java
Copyright Apache License 2.0
Author : aliyun-beta
Copyright Apache License 2.0
Author : aliyun-beta
private boolean canreplacedign(Priority priority, FiCaSchedulerNode node, NodeType type, RMContainer reservedContainer) {
// Clearly we need containers for this application...
if (type == NodeType.OFF_SWITCH) {
if (reservedContainer != null) {
return true;
}
// 'Delay' off-switch
ResourceRequest offSwitchRequest = application.getResourceRequest(priority, ResourceRequest.ANY);
long missedOpportunities = application.getSchedulingOpportunities(priority);
long requiredContainers = offSwitchRequest.getNumContainers();
float localityWaitFactor = getLocalityWaitFactor(priority, rmContext.getScheduler().getNumClusterNodes());
// Cap the delay by the number of nodes in the cluster. Under most conditions
// this means we will consider each node in the cluster before
// accepting an off-switch replacedignment.
return (Math.min(rmContext.getScheduler().getNumClusterNodes(), (requiredContainers * localityWaitFactor)) < missedOpportunities);
}
// Check if we need containers on this rack
ResourceRequest rackLocalRequest = application.getResourceRequest(priority, node.getRackName());
if (rackLocalRequest == null || rackLocalRequest.getNumContainers() <= 0) {
return false;
}
// If we are here, we do need containers on this rack for RACK_LOCAL req
if (type == NodeType.RACK_LOCAL) {
// 'Delay' rack-local just a little bit...
long missedOpportunities = application.getSchedulingOpportunities(priority);
return getActualNodeLocalityDelay() < missedOpportunities;
}
// Check if we need containers on this host
if (type == NodeType.NODE_LOCAL) {
// Now check if we need containers on this host...
ResourceRequest nodeLocalRequest = application.getResourceRequest(priority, node.getNodeName());
if (nodeLocalRequest != null) {
return nodeLocalRequest.getNumContainers() > 0;
}
}
return false;
}
19
View Complete Implementation : ContainerPBImpl.java
Copyright Apache License 2.0
Author : aliyun-beta
Copyright Apache License 2.0
Author : aliyun-beta
private PriorityProto convertToProtoFormat(Priority p) {
return ((PriorityPBImpl) p).getProto();
}
19
View Complete Implementation : TestFSAppAttempt.java
Copyright Apache License 2.0
Author : aliyun-beta
Copyright Apache License 2.0
Author : aliyun-beta
@Test
public void testDelayScheduling() {
FSLeafQueue queue = Mockito.mock(FSLeafQueue.clreplaced);
Priority prio = Mockito.mock(Priority.clreplaced);
Mockito.when(prio.getPriority()).thenReturn(1);
double nodeLocalityThreshold = .5;
double rackLocalityThreshold = .6;
ApplicationAttemptId applicationAttemptId = createAppAttemptId(1, 1);
RMContext rmContext = resourceManager.getRMContext();
FSAppAttempt schedulerApp = new FSAppAttempt(scheduler, applicationAttemptId, "user1", queue, null, rmContext);
// Default level should be node-local
replacedertEquals(NodeType.NODE_LOCAL, schedulerApp.getAllowedLocalityLevel(prio, 10, nodeLocalityThreshold, rackLocalityThreshold));
// First five scheduling opportunities should remain node local
for (int i = 0; i < 5; i++) {
schedulerApp.addSchedulingOpportunity(prio);
replacedertEquals(NodeType.NODE_LOCAL, schedulerApp.getAllowedLocalityLevel(prio, 10, nodeLocalityThreshold, rackLocalityThreshold));
}
// After five it should switch to rack local
schedulerApp.addSchedulingOpportunity(prio);
replacedertEquals(NodeType.RACK_LOCAL, schedulerApp.getAllowedLocalityLevel(prio, 10, nodeLocalityThreshold, rackLocalityThreshold));
// Manually set back to node local
schedulerApp.resetAllowedLocalityLevel(prio, NodeType.NODE_LOCAL);
schedulerApp.resetSchedulingOpportunities(prio);
replacedertEquals(NodeType.NODE_LOCAL, schedulerApp.getAllowedLocalityLevel(prio, 10, nodeLocalityThreshold, rackLocalityThreshold));
// Now escalate again to rack-local, then to off-switch
for (int i = 0; i < 5; i++) {
schedulerApp.addSchedulingOpportunity(prio);
replacedertEquals(NodeType.NODE_LOCAL, schedulerApp.getAllowedLocalityLevel(prio, 10, nodeLocalityThreshold, rackLocalityThreshold));
}
schedulerApp.addSchedulingOpportunity(prio);
replacedertEquals(NodeType.RACK_LOCAL, schedulerApp.getAllowedLocalityLevel(prio, 10, nodeLocalityThreshold, rackLocalityThreshold));
for (int i = 0; i < 6; i++) {
schedulerApp.addSchedulingOpportunity(prio);
replacedertEquals(NodeType.RACK_LOCAL, schedulerApp.getAllowedLocalityLevel(prio, 10, nodeLocalityThreshold, rackLocalityThreshold));
}
schedulerApp.addSchedulingOpportunity(prio);
replacedertEquals(NodeType.OFF_SWITCH, schedulerApp.getAllowedLocalityLevel(prio, 10, nodeLocalityThreshold, rackLocalityThreshold));
}
19
View Complete Implementation : ContainerPBImpl.java
Copyright Apache License 2.0
Author : aliyun-beta
Copyright Apache License 2.0
Author : aliyun-beta
@Override
public void setPriority(Priority priority) {
maybeInitBuilder();
if (priority == null) {
builder.clearPriority();
}
this.priority = priority;
}
19
View Complete Implementation : FiCaSchedulerNode.java
Copyright Apache License 2.0
Author : aliyun-beta
Copyright Apache License 2.0
Author : aliyun-beta
@Override
public synchronized void reserveResource(SchedulerApplicationAttempt application, Priority priority, RMContainer container) {
// Check if it's already reserved
RMContainer reservedContainer = getReservedContainer();
if (reservedContainer != null) {
// Sanity check
if (!container.getContainer().getNodeId().equals(getNodeID())) {
throw new IllegalStateException("Trying to reserve" + " container " + container + " on node " + container.getReservedNode() + " when currently" + " reserved resource " + reservedContainer + " on node " + reservedContainer.getReservedNode());
}
// Cannot reserve more than one application attempt on a given node!
// Reservation is still against attempt.
if (!reservedContainer.getContainer().getId().getApplicationAttemptId().equals(container.getContainer().getId().getApplicationAttemptId())) {
throw new IllegalStateException("Trying to reserve" + " container " + container + " for application " + application.getApplicationAttemptId() + " when currently" + " reserved container " + reservedContainer + " on node " + this);
}
if (LOG.isDebugEnabled()) {
LOG.debug("Updated reserved container " + container.getContainer().getId() + " on node " + this + " for application attempt " + application.getApplicationAttemptId());
}
} else {
if (LOG.isDebugEnabled()) {
LOG.debug("Reserved container " + container.getContainer().getId() + " on node " + this + " for application attempt " + application.getApplicationAttemptId());
}
}
setReservedContainer(container);
}
19
View Complete Implementation : AMRMClient.java
Copyright Apache License 2.0
Author : aliyun-beta
Copyright Apache License 2.0
Author : aliyun-beta
/**
* Get outstanding <code>ContainerRequest</code>s matching the given
* parameters. These ContainerRequests should have been added via
* <code>addContainerRequest</code> earlier in the lifecycle. For performance,
* the AMRMClient may return its internal collection directly without creating
* a copy. Users should not perform mutable operations on the return value.
* Each collection in the list contains requests with identical
* <code>Resource</code> size that fit in the given capability. In a
* collection, requests will be returned in the same order as they were added.
* @return Collection of request matching the parameters
*/
public abstract List<? extends Collection<T>> getMatchingRequests(Priority priority, String resourceName, Resource capability);
19
View Complete Implementation : FSSchedulerNode.java
Copyright Apache License 2.0
Author : aliyun-beta
Copyright Apache License 2.0
Author : aliyun-beta
@Override
public synchronized void reserveResource(SchedulerApplicationAttempt application, Priority priority, RMContainer container) {
// Check if it's already reserved
RMContainer reservedContainer = getReservedContainer();
if (reservedContainer != null) {
// Sanity check
if (!container.getContainer().getNodeId().equals(getNodeID())) {
throw new IllegalStateException("Trying to reserve" + " container " + container + " on node " + container.getReservedNode() + " when currently" + " reserved resource " + reservedContainer + " on node " + reservedContainer.getReservedNode());
}
// Cannot reserve more than one application on a given node!
if (!reservedContainer.getContainer().getId().getApplicationAttemptId().equals(container.getContainer().getId().getApplicationAttemptId())) {
throw new IllegalStateException("Trying to reserve" + " container " + container + " for application " + application.getApplicationId() + " when currently" + " reserved container " + reservedContainer + " on node " + this);
}
LOG.info("Updated reserved container " + container.getContainer().getId() + " on node " + this + " for application " + application.getApplicationId());
} else {
LOG.info("Reserved container " + container.getContainer().getId() + " on node " + this + " for application " + application.getApplicationId());
}
setReservedContainer(container);
this.reservedAppSchedulable = (FSAppAttempt) application;
}
19
View Complete Implementation : TestFSAppAttempt.java
Copyright Apache License 2.0
Author : aliyun-beta
Copyright Apache License 2.0
Author : aliyun-beta
@Test
public void testDelaySchedulingForContinuousScheduling() throws InterruptedException {
FSLeafQueue queue = scheduler.getQueueManager().getLeafQueue("queue", true);
Priority prio = Mockito.mock(Priority.clreplaced);
Mockito.when(prio.getPriority()).thenReturn(1);
ControlledClock clock = new ControlledClock();
scheduler.setClock(clock);
// 5 seconds
long nodeLocalityDelayMs = 5 * 1000L;
// 6 seconds
long rackLocalityDelayMs = 6 * 1000L;
RMContext rmContext = resourceManager.getRMContext();
ApplicationAttemptId applicationAttemptId = createAppAttemptId(1, 1);
FSAppAttempt schedulerApp = new FSAppAttempt(scheduler, applicationAttemptId, "user1", queue, null, rmContext);
// Default level should be node-local
replacedertEquals(NodeType.NODE_LOCAL, schedulerApp.getAllowedLocalityLevelByTime(prio, nodeLocalityDelayMs, rackLocalityDelayMs, clock.getTime()));
// after 4 seconds should remain node local
clock.tickSec(4);
replacedertEquals(NodeType.NODE_LOCAL, schedulerApp.getAllowedLocalityLevelByTime(prio, nodeLocalityDelayMs, rackLocalityDelayMs, clock.getTime()));
// after 6 seconds should switch to rack local
clock.tickSec(2);
replacedertEquals(NodeType.RACK_LOCAL, schedulerApp.getAllowedLocalityLevelByTime(prio, nodeLocalityDelayMs, rackLocalityDelayMs, clock.getTime()));
// manually set back to node local
schedulerApp.resetAllowedLocalityLevel(prio, NodeType.NODE_LOCAL);
schedulerApp.resetSchedulingOpportunities(prio, clock.getTime());
replacedertEquals(NodeType.NODE_LOCAL, schedulerApp.getAllowedLocalityLevelByTime(prio, nodeLocalityDelayMs, rackLocalityDelayMs, clock.getTime()));
// Now escalate again to rack-local, then to off-switch
clock.tickSec(6);
replacedertEquals(NodeType.RACK_LOCAL, schedulerApp.getAllowedLocalityLevelByTime(prio, nodeLocalityDelayMs, rackLocalityDelayMs, clock.getTime()));
clock.tickSec(7);
replacedertEquals(NodeType.OFF_SWITCH, schedulerApp.getAllowedLocalityLevelByTime(prio, nodeLocalityDelayMs, rackLocalityDelayMs, clock.getTime()));
}
19
View Complete Implementation : FSAppAttempt.java
Copyright Apache License 2.0
Author : aliyun-beta
Copyright Apache License 2.0
Author : aliyun-beta
/**
* Called when this application already has an existing reservation on the
* given node. Sees whether we can turn the reservation into an allocation.
* Also checks whether the application needs the reservation anymore, and
* releases it if not.
*
* @param node
* Node that the application has an existing reservation on
* @return whether the reservation on the given node is valid.
*/
public boolean replacedignReservedContainer(FSSchedulerNode node) {
RMContainer rmContainer = node.getReservedContainer();
Priority reservedPriority = rmContainer.getReservedPriority();
if (!isValidReservation(node)) {
// Don't hold the reservation if app can no longer use it
LOG.info("Releasing reservation that cannot be satisfied for " + "application " + getApplicationAttemptId() + " on node " + node);
unreserve(reservedPriority, node);
return false;
}
// Reservation valid; try to fulfill the reservation
if (LOG.isDebugEnabled()) {
LOG.debug("Trying to fulfill reservation for application " + getApplicationAttemptId() + " on node: " + node);
}
// Fail early if the reserved container won't fit.
// Note that we have an replacedumption here that
// there's only one container size per priority.
if (Resources.fitsIn(node.getReservedContainer().getReservedResource(), node.getAvailableResource())) {
replacedignContainer(node, true);
}
return true;
}
18
View Complete Implementation : RMAppManager.java
Copyright Apache License 2.0
Author : aliyun-beta
Copyright Apache License 2.0
Author : aliyun-beta
private RMAppImpl createAndPopulateNewRMApp(ApplicationSubmissionContext submissionContext, long submitTime, String user, boolean isRecovery) throws YarnException {
// Do queue mapping
if (!isRecovery) {
if (rmContext.getQueuePlacementManager() != null) {
// We only do queue mapping when it's a new application
rmContext.getQueuePlacementManager().placeApplication(submissionContext, user);
}
}
ApplicationId applicationId = submissionContext.getApplicationId();
ResourceRequest amReq = validateAndCreateResourceRequest(submissionContext, isRecovery);
// Verify and get the update application priority and set back to
// submissionContext
Priority appPriority = rmContext.getScheduler().checkAndGetApplicationPriority(submissionContext.getPriority(), user, submissionContext.getQueue(), applicationId);
submissionContext.setPriority(appPriority);
// Create RMApp
RMAppImpl application = new RMAppImpl(applicationId, rmContext, this.conf, submissionContext.getApplicationName(), user, submissionContext.getQueue(), submissionContext, this.scheduler, this.masterService, submitTime, submissionContext.getApplicationType(), submissionContext.getApplicationTags(), amReq);
// Concurrent app submissions with same applicationId will fail here
// Concurrent app submissions with different applicationIds will not
// influence each other
if (rmContext.getRMApps().putIfAbsent(applicationId, application) != null) {
String message = "Application with id " + applicationId + " is already present! Cannot add a duplicate!";
LOG.warn(message);
throw new YarnException(message);
}
// Inform the ACLs Manager
this.applicationACLsManager.addApplication(applicationId, submissionContext.getAMContainerSpec().getApplicationACLs());
String appViewACLs = submissionContext.getAMContainerSpec().getApplicationACLs().get(ApplicationAccessType.VIEW_APP);
rmContext.getSystemMetricsPublisher().appACLsUpdated(application, appViewACLs, System.currentTimeMillis());
return application;
}
18
View Complete Implementation : RegularContainerAllocator.java
Copyright Apache License 2.0
Author : aliyun-beta
Copyright Apache License 2.0
Author : aliyun-beta
private ContainerAllocation handleNewContainerAllocation(ContainerAllocation allocationResult, FiCaSchedulerNode node, Priority priority, RMContainer reservedContainer, Container container) {
// Handling container allocation
// Did we previously reserve containers at this 'priority'?
if (reservedContainer != null) {
application.unreserve(priority, node, reservedContainer);
}
// Inform the application
RMContainer allocatedContainer = application.allocate(allocationResult.containerNodeType, node, priority, lastResourceRequest, container);
// Does the application need this resource?
if (allocatedContainer == null) {
// Skip this app if we failed to allocate.
ContainerAllocation ret = new ContainerAllocation(allocationResult.containerToBeUnreserved, null, AllocationState.APP_SKIPPED);
return ret;
}
// Inform the node
node.allocateContainer(allocatedContainer);
// update locality statistics
application.incNumAllocatedContainers(allocationResult.containerNodeType, allocationResult.requestNodeType);
return allocationResult;
}