Here are the examples of the java api org.apache.hadoop.metrics2.lib.DefaultMetricsSystem.shutdown() taken from open source projects. By voting up you can indicate which examples are most useful and appropriate.
31 Examples
19
View Complete Implementation : TestMetricsSystemImpl.java
Copyright Apache License 2.0
Author : aliyun-beta
Copyright Apache License 2.0
Author : aliyun-beta
@Test
public void testStartStopStart() {
// Clear pre-existing source names.
DefaultMetricsSystem.shutdown();
MetricsSystemImpl ms = new MetricsSystemImpl("test");
TestSource ts = new TestSource("ts");
ms.start();
ms.register("ts", "", ts);
MetricsSourceAdapter sa = ms.getSourceAdapter("ts");
replacedertNotNull(sa);
replacedertNotNull(sa.getMBeanName());
ms.stop();
ms.shutdown();
ms.start();
sa = ms.getSourceAdapter("ts");
replacedertNotNull(sa);
replacedertNotNull(sa.getMBeanName());
ms.stop();
ms.shutdown();
}
19
View Complete Implementation : DataNodeMetrics.java
Copyright Apache License 2.0
Author : aliyun-beta
Copyright Apache License 2.0
Author : aliyun-beta
public void shutdown() {
DefaultMetricsSystem.shutdown();
}
19
View Complete Implementation : JobHistoryServer.java
Copyright Apache License 2.0
Author : aliyun-beta
Copyright Apache License 2.0
Author : aliyun-beta
@Override
protected void serviceStop() throws Exception {
DefaultMetricsSystem.shutdown();
super.serviceStop();
}
19
View Complete Implementation : TestSchedulerApplicationAttempt.java
Copyright Apache License 2.0
Author : aliyun-beta
Copyright Apache License 2.0
Author : aliyun-beta
@After
public void tearDown() {
QueueMetrics.clearQueueMetrics();
DefaultMetricsSystem.shutdown();
}
19
View Complete Implementation : WebAppProxyServer.java
Copyright Apache License 2.0
Author : aliyun-beta
Copyright Apache License 2.0
Author : aliyun-beta
@Override
protected void serviceStop() throws Exception {
super.serviceStop();
DefaultMetricsSystem.shutdown();
}
19
View Complete Implementation : StateStoreMetrics.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
public void shutdown() {
DefaultMetricsSystem.shutdown();
reset();
}
19
View Complete Implementation : TestNodeManagerMetrics.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
@After
public void tearDown() {
DefaultMetricsSystem.shutdown();
}
19
View Complete Implementation : ApplicationHistoryServer.java
Copyright Apache License 2.0
Author : yncxcw
Copyright Apache License 2.0
Author : yncxcw
@Override
protected void serviceStop() throws Exception {
if (webApp != null) {
webApp.stop();
}
DefaultMetricsSystem.shutdown();
super.serviceStop();
}
19
View Complete Implementation : NodeManager.java
Copyright Apache License 2.0
Author : yncxcw
Copyright Apache License 2.0
Author : yncxcw
@Override
protected void serviceStop() throws Exception {
if (isStopping.getAndSet(true)) {
return;
}
try {
super.serviceStop();
DefaultMetricsSystem.shutdown();
} finally {
// YARN-3641: NM's services stop get failed shouldn't block the
// release of NMLevelDBStore.
stopRecoveryStore();
}
}
18
View Complete Implementation : TestMetricsSystemImpl.java
Copyright Apache License 2.0
Author : aliyun-beta
Copyright Apache License 2.0
Author : aliyun-beta
@Test
public void testInitFirstVerifyStopInvokedImmediately() throws Exception {
DefaultMetricsSystem.shutdown();
new ConfigBuilder().add("*.period", 8).add("test.sink.test.clreplaced", TestSink.clreplaced.getName()).add("test.*.source.filter.exclude", "s0").add("test.source.s1.metric.filter.exclude", "X*").add("test.sink.sink1.metric.filter.exclude", "Y*").add("test.sink.sink2.metric.filter.exclude", "Y*").save(TestMetricsConfig.getTestFilename("hadoop-metrics2-test"));
MetricsSystemImpl ms = new MetricsSystemImpl("Test");
ms.start();
ms.register("s0", "s0 desc", new TestSource("s0rec"));
TestSource s1 = ms.register("s1", "s1 desc", new TestSource("s1rec"));
s1.c1.incr();
s1.xxx.incr();
s1.g1.set(2);
s1.yyy.incr(2);
s1.s1.add(0);
MetricsSink sink1 = mock(MetricsSink.clreplaced);
MetricsSink sink2 = mock(MetricsSink.clreplaced);
ms.registerSink("sink1", "sink1 desc", sink1);
ms.registerSink("sink2", "sink2 desc", sink2);
// publish the metrics
ms.publishMetricsNow();
ms.stop();
ms.shutdown();
// When we call stop, at most two sources will be consumed by each sink thread.
verify(sink1, atMost(2)).putMetrics(r1.capture());
List<MetricsRecord> mr1 = r1.getAllValues();
verify(sink2, atMost(2)).putMetrics(r2.capture());
List<MetricsRecord> mr2 = r2.getAllValues();
if (mr1.size() != 0 && mr2.size() != 0) {
checkMetricsRecords(mr1);
replacedertEquals("output", mr1, mr2);
} else if (mr1.size() != 0) {
checkMetricsRecords(mr1);
} else if (mr2.size() != 0) {
checkMetricsRecords(mr2);
}
}
18
View Complete Implementation : TestMetricsSystemImpl.java
Copyright Apache License 2.0
Author : aliyun-beta
Copyright Apache License 2.0
Author : aliyun-beta
@Test
public void testInitFirstVerifyCallBacks() throws Exception {
DefaultMetricsSystem.shutdown();
new ConfigBuilder().add("*.period", 8).add("test.sink.test.clreplaced", TestSink.clreplaced.getName()).add("test.*.source.filter.exclude", "s0").add("test.source.s1.metric.filter.exclude", "X*").add("test.sink.sink1.metric.filter.exclude", "Y*").add("test.sink.sink2.metric.filter.exclude", "Y*").save(TestMetricsConfig.getTestFilename("hadoop-metrics2-test"));
MetricsSystemImpl ms = new MetricsSystemImpl("Test");
ms.start();
ms.register("s0", "s0 desc", new TestSource("s0rec"));
TestSource s1 = ms.register("s1", "s1 desc", new TestSource("s1rec"));
s1.c1.incr();
s1.xxx.incr();
s1.g1.set(2);
s1.yyy.incr(2);
s1.s1.add(0);
MetricsSink sink1 = mock(MetricsSink.clreplaced);
MetricsSink sink2 = mock(MetricsSink.clreplaced);
ms.registerSink("sink1", "sink1 desc", sink1);
ms.registerSink("sink2", "sink2 desc", sink2);
// publish the metrics
ms.publishMetricsNow();
try {
verify(sink1, timeout(200).times(2)).putMetrics(r1.capture());
verify(sink2, timeout(200).times(2)).putMetrics(r2.capture());
} finally {
ms.stop();
ms.shutdown();
}
// When we call stop, at most two sources will be consumed by each sink thread.
List<MetricsRecord> mr1 = r1.getAllValues();
List<MetricsRecord> mr2 = r2.getAllValues();
checkMetricsRecords(mr1);
replacedertEquals("output", mr1, mr2);
}
18
View Complete Implementation : TestAppRunnability.java
Copyright Apache License 2.0
Author : aliyun-beta
Copyright Apache License 2.0
Author : aliyun-beta
@After
public void tearDown() {
if (resourceManager != null) {
resourceManager.stop();
resourceManager = null;
}
QueueMetrics.clearQueueMetrics();
DefaultMetricsSystem.shutdown();
}
18
View Complete Implementation : TestEventQueue.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
@After
public void stopEventQueue() {
DefaultMetricsSystem.shutdown();
queue.close();
}
18
View Complete Implementation : TestEventWatcher.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
@After
public void stopLeaseManager() {
leaseManager.shutdown();
DefaultMetricsSystem.shutdown();
}
18
View Complete Implementation : MiniOzoneClusterImpl.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
@Override
public void shutdown() {
try {
LOG.info("Shutting down the Mini Ozone Cluster");
File baseDir = new File(GenericTestUtils.getTempPath(MiniOzoneClusterImpl.clreplaced.getSimpleName() + "-" + scm.getClientProtocolServer().getScmInfo().getClusterId()));
stop();
FileUtils.deleteDirectory(baseDir);
ContainerCache.getInstance(conf).shutdownCache();
DefaultMetricsSystem.shutdown();
} catch (IOException e) {
LOG.error("Exception while shutting down the cluster.", e);
}
}
18
View Complete Implementation : TezChild.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
private void shutdown() {
executor.shutdownNow();
if (taskReporter != null) {
taskReporter.shutdown();
}
RPC.stopProxy(umbilical);
DefaultMetricsSystem.shutdown();
LogManager.shutdown();
}
17
View Complete Implementation : TestResourceTrackerService.java
Copyright Apache License 2.0
Author : aliyun-beta
Copyright Apache License 2.0
Author : aliyun-beta
@After
public void tearDown() {
if (hostFile != null && hostFile.exists()) {
hostFile.delete();
}
ClusterMetrics.destroy();
if (rm != null) {
rm.stop();
}
MetricsSystem ms = DefaultMetricsSystem.instance();
if (ms.getSource("ClusterMetrics") != null) {
DefaultMetricsSystem.shutdown();
}
}
17
View Complete Implementation : TestRM.java
Copyright Apache License 2.0
Author : aliyun-beta
Copyright Apache License 2.0
Author : aliyun-beta
@After
public void tearDown() {
ClusterMetrics.destroy();
QueueMetrics.clearQueueMetrics();
DefaultMetricsSystem.shutdown();
}
17
View Complete Implementation : KMSWebServer.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
public void stop() throws Exception {
httpServer.stop();
pauseMonitor.stop();
JvmMetrics.shutdownSingleton();
DefaultMetricsSystem.shutdown();
}
17
View Complete Implementation : Router.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
@Override
protected void serviceStop() throws Exception {
if (webApp != null) {
webApp.stop();
}
if (isStopping.getAndSet(true)) {
return;
}
super.serviceStop();
DefaultMetricsSystem.shutdown();
}
16
View Complete Implementation : JournalNode.java
Copyright Apache License 2.0
Author : aliyun-beta
Copyright Apache License 2.0
Author : aliyun-beta
/**
* Stop the daemon with the given status code
* @param rc the status code with which to exit (non-zero
* should indicate an error)
*/
public void stop(int rc) {
this.resultCode = rc;
if (rpcServer != null) {
rpcServer.stop();
}
if (httpServer != null) {
try {
httpServer.stop();
} catch (IOException ioe) {
LOG.warn("Unable to stop HTTP server for " + this, ioe);
}
}
for (Journal j : journalsById.values()) {
IOUtils.cleanup(LOG, j);
}
DefaultMetricsSystem.shutdown();
if (journalNodeInfoBeanName != null) {
MBeans.unregister(journalNodeInfoBeanName);
journalNodeInfoBeanName = null;
}
if (tracer != null) {
tracer.close();
tracer = null;
}
}
16
View Complete Implementation : TestClusterMetrics.java
Copyright Apache License 2.0
Author : aliyun-beta
Copyright Apache License 2.0
Author : aliyun-beta
@After
public void tearDown() {
ClusterMetrics.destroy();
MetricsSystem ms = DefaultMetricsSystem.instance();
if (ms.getSource("ClusterMetrics") != null) {
DefaultMetricsSystem.shutdown();
}
}
16
View Complete Implementation : NodeManager.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
@Override
protected void serviceStop() throws Exception {
if (isStopping.getAndSet(true)) {
return;
}
try {
super.serviceStop();
DefaultMetricsSystem.shutdown();
if (null != context) {
context.getContainerExecutor().stop();
// Cleanup ResourcePluginManager
ResourcePluginManager rpm = context.getResourcePluginManager();
if (rpm != null) {
rpm.cleanup();
}
}
} finally {
// YARN-3641: NM's services stop get failed shouldn't block the
// release of NMLevelDBStore.
stopRecoveryStore();
}
}
16
View Complete Implementation : TestEventWatcher.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
@Test
public void testMetrics() throws InterruptedException {
DefaultMetricsSystem.initialize("test");
EventQueue queue = new EventQueue();
Evenreplacedcher<UnderreplicatedEvent, ReplicationCompletedEvent> replicationWatcher = createEvenreplacedcher();
EventHandlerStub<UnderreplicatedEvent> underReplicatedEvents = new EventHandlerStub<>();
queue.addHandler(UNDER_REPLICATED, underReplicatedEvents);
replicationWatcher.start(queue);
// send 3 event to track 3 in-progress activity
UnderreplicatedEvent event1 = new UnderreplicatedEvent(HddsIdFactory.getLongId(), "C1");
UnderreplicatedEvent event2 = new UnderreplicatedEvent(HddsIdFactory.getLongId(), "C2");
UnderreplicatedEvent event3 = new UnderreplicatedEvent(HddsIdFactory.getLongId(), "C1");
queue.fireEvent(WATCH_UNDER_REPLICATED, event1);
queue.fireEvent(WATCH_UNDER_REPLICATED, event2);
queue.fireEvent(WATCH_UNDER_REPLICATED, event3);
// 1st event is completed, don't need to track any more
ReplicationCompletedEvent event1Completed = new ReplicationCompletedEvent(event1.id, "C1", "D1");
queue.fireEvent(REPLICATION_COMPLETED, event1Completed);
// lease manager timeout = 2000L
Thread.sleep(3 * 2000L);
queue.processAll(2000L);
// until now: 3 in-progress activities are tracked with three
// UnderreplicatedEvents. The first one is completed, the remaining two
// are timed out (as the timeout -- defined in the lease manager -- is
// 2000ms).
EvenreplacedcherMetrics metrics = replicationWatcher.getMetrics();
// 3 events are received
replacedert.replacedertEquals(3, metrics.getTrackedEvents().value());
// completed + timed out = all messages
replacedert.replacedertEquals("number of timed out and completed messages should be the same as the" + " all messages", metrics.getTrackedEvents().value(), metrics.getCompletedEvents().value() + metrics.getTimedOutEvents().value());
// _at least_ two are timed out.
replacedert.replacedertTrue("At least two events should be timed out.", metrics.getTimedOutEvents().value() >= 2);
DefaultMetricsSystem.shutdown();
}
15
View Complete Implementation : JournalNode.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
/**
* Stop the daemon with the given status code
* @param rc the status code with which to exit (non-zero
* should indicate an error)
*/
public void stop(int rc) {
this.resultCode = rc;
for (JournalNodeSyncer jSyncer : journalSyncersById.values()) {
jSyncer.stopSync();
}
if (rpcServer != null) {
rpcServer.stop();
}
if (httpServer != null) {
try {
httpServer.stop();
} catch (IOException ioe) {
LOG.warn("Unable to stop HTTP server for " + this, ioe);
}
}
for (Journal j : journalsById.values()) {
IOUtils.cleanupWithLogger(LOG, j);
}
DefaultMetricsSystem.shutdown();
if (journalNodeInfoBeanName != null) {
MBeans.unregister(journalNodeInfoBeanName);
journalNodeInfoBeanName = null;
}
if (tracer != null) {
tracer.close();
tracer = null;
}
}
15
View Complete Implementation : YarnChild.java
Copyright Apache License 2.0
Author : yncxcw
Copyright Apache License 2.0
Author : yncxcw
public static void main(String[] args) throws Throwable {
Thread.setDefaultUncaughtExceptionHandler(new YarnUncaughtExceptionHandler());
LOG.debug("Child starting");
final JobConf job = new JobConf(MRJobConfig.JOB_CONF_FILE);
// Initing with our JobConf allows us to avoid loading confs twice
Limits.init(job);
UserGroupInformation.setConfiguration(job);
String host = args[0];
int port = Integer.parseInt(args[1]);
final InetSocketAddress address = NetUtils.createSocketAddrForHost(host, port);
final TaskAttemptID firstTaskid = TaskAttemptID.forName(args[2]);
long jvmIdLong = Long.parseLong(args[3]);
JVMId jvmId = new JVMId(firstTaskid.getJobID(), firstTaskid.getTaskType() == TaskType.MAP, jvmIdLong);
// initialize metrics
DefaultMetricsSystem.initialize(StringUtils.camelize(firstTaskid.getTaskType().name()) + "Task");
// Security framework already loaded the tokens into current ugi
Credentials credentials = UserGroupInformation.getCurrentUser().getCredentials();
LOG.info("Executing with tokens:");
for (Token<?> token : credentials.getAllTokens()) {
LOG.info(token);
}
// Create TaskUmbilicalProtocol as actual task owner.
UserGroupInformation taskOwner = UserGroupInformation.createRemoteUser(firstTaskid.getJobID().toString());
Token<JobTokenIdentifier> jt = TokenCache.getJobToken(credentials);
SecurityUtil.setTokenService(jt, address);
taskOwner.addToken(jt);
final TaskUmbilicalProtocol umbilical = taskOwner.doAs(new PrivilegedExceptionAction<TaskUmbilicalProtocol>() {
@Override
public TaskUmbilicalProtocol run() throws Exception {
return (TaskUmbilicalProtocol) RPC.getProxy(TaskUmbilicalProtocol.clreplaced, TaskUmbilicalProtocol.versionID, address, job);
}
});
// report non-pid to application master
JvmContext context = new JvmContext(jvmId, "-1000");
LOG.debug("PID: " + System.getenv().get("JVM_PID"));
Task task = null;
UserGroupInformation childUGI = null;
ScheduledExecutorService logSyncer = null;
try {
int idleLoopCount = 0;
JvmTask myTask = null;
;
// poll for new task
for (int idle = 0; null == myTask; ++idle) {
long sleepTimeMilliSecs = Math.min(idle * 500, 1500);
LOG.info("Sleeping for " + sleepTimeMilliSecs + "ms before retrying again. Got null now.");
MILLISECONDS.sleep(sleepTimeMilliSecs);
myTask = umbilical.getTask(context);
}
if (myTask.shouldDie()) {
return;
}
task = myTask.getTask();
YarnChild.taskid = task.getTaskID();
// Create the job-conf and set credentials
configureTask(job, task, credentials, jt);
// Initiate Java VM metrics
JvmMetrics.initSingleton(jvmId.toString(), job.getSessionId());
childUGI = UserGroupInformation.createRemoteUser(System.getenv(ApplicationConstants.Environment.USER.toString()));
// Add tokens to new user so that it may execute its task correctly.
childUGI.addCredentials(credentials);
// set job clreplacedloader if configured before invoking the task
MRApps.setJobClreplacedLoader(job);
logSyncer = TaskLog.createLogSyncer();
// Create a final reference to the task for the doAs block
final Task taskFinal = task;
childUGI.doAs(new PrivilegedExceptionAction<Object>() {
@Override
public Object run() throws Exception {
// use job-specified working directory
setEncryptedSpillKeyIfRequired(taskFinal);
FileSystem.get(job).setWorkingDirectory(job.getWorkingDirectory());
// run the task
taskFinal.run(job, umbilical);
return null;
}
});
} catch (FSError e) {
LOG.fatal("FSError from child", e);
if (!ShutdownHookManager.get().isShutdownInProgress()) {
umbilical.fsError(taskid, e.getMessage());
}
} catch (Exception exception) {
LOG.warn("Exception running child : " + StringUtils.stringifyException(exception));
try {
if (task != null) {
// do cleanup for the task
if (childUGI == null) {
// no need to job into doAs block
task.taskCleanup(umbilical);
} else {
final Task taskFinal = task;
childUGI.doAs(new PrivilegedExceptionAction<Object>() {
@Override
public Object run() throws Exception {
taskFinal.taskCleanup(umbilical);
return null;
}
});
}
}
} catch (Exception e) {
LOG.info("Exception cleaning up: " + StringUtils.stringifyException(e));
}
// Report back any failures, for diagnostic purposes
if (taskid != null) {
if (!ShutdownHookManager.get().isShutdownInProgress()) {
umbilical.fatalError(taskid, StringUtils.stringifyException(exception));
}
}
} catch (Throwable throwable) {
LOG.fatal("Error running child : " + StringUtils.stringifyException(throwable));
if (taskid != null) {
if (!ShutdownHookManager.get().isShutdownInProgress()) {
Throwable tCause = throwable.getCause();
String cause = tCause == null ? throwable.getMessage() : StringUtils.stringifyException(tCause);
umbilical.fatalError(taskid, cause);
}
}
} finally {
RPC.stopProxy(umbilical);
DefaultMetricsSystem.shutdown();
TaskLog.syncLogsShutdown(logSyncer);
}
}
14
View Complete Implementation : ServiceScheduler.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
@Override
public void serviceStop() throws Exception {
LOG.info("Stopping service scheduler");
if (executorService != null) {
executorService.shutdownNow();
}
DefaultMetricsSystem.shutdown();
// only stop the entire service when a graceful stop has been initiated
// (e.g. via client RPC, not through the AM receiving a SIGTERM)
if (gracefulStop) {
if (YarnConfiguration.timelineServiceV2Enabled(getConfig())) {
// mark other component-instances/containers as STOPPED
final Map<ContainerId, ComponentInstance> liveInst = getLiveInstances();
for (Map.Entry<ContainerId, ComponentInstance> instance : liveInst.entrySet()) {
if (!ComponentInstance.isFinalState(instance.getValue().getContainerSpec().getState())) {
LOG.info("{} Component instance state changed from {} to {}", instance.getValue().getCompInstanceName(), instance.getValue().getContainerSpec().getState(), ContainerState.STOPPED);
serviceTimelinePublisher.componentInstanceFinished(instance.getKey(), KILLED_AFTER_APP_COMPLETION, ContainerState.STOPPED, getDiagnostics().toString());
}
}
LOG.info("Service state changed to {}", finalApplicationStatus);
// mark attempt as unregistered
serviceTimelinePublisher.serviceAttemptUnregistered(context, finalApplicationStatus, diagnostics.toString());
}
// unregister AM
amRMClient.unregisterApplicationMaster(finalApplicationStatus, diagnostics.toString(), "");
LOG.info("Service {} unregistered with RM, with attemptId = {} " + ", diagnostics = {} ", app.getName(), context.attemptId, diagnostics);
}
super.serviceStop();
}
13
View Complete Implementation : TestRMHAMetrics.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
@Before
public void setUp() throws Exception {
configuration = new Configuration();
UserGroupInformation.setConfiguration(configuration);
configuration.setBoolean(YarnConfiguration.RM_HA_ENABLED, true);
configuration.set(YarnConfiguration.RM_HA_IDS, RM1_NODE_ID + "," + RM2_NODE_ID);
for (String confKey : YarnConfiguration.getServiceAddressConfKeys(configuration)) {
configuration.set(HAUtil.addSuffix(confKey, RM1_NODE_ID), RM1_ADDRESS);
configuration.set(HAUtil.addSuffix(confKey, RM2_NODE_ID), RM2_ADDRESS);
}
ClusterMetrics.destroy();
QueueMetrics.clearQueueMetrics();
DefaultMetricsSystem.shutdown();
}
12
View Complete Implementation : TestRMHA.java
Copyright Apache License 2.0
Author : aliyun-beta
Copyright Apache License 2.0
Author : aliyun-beta
@Before
public void setUp() throws Exception {
configuration = new Configuration();
UserGroupInformation.setConfiguration(configuration);
configuration.setBoolean(YarnConfiguration.RM_HA_ENABLED, true);
configuration.set(YarnConfiguration.RM_HA_IDS, RM1_NODE_ID + "," + RM2_NODE_ID);
for (String confKey : YarnConfiguration.getServiceAddressConfKeys(configuration)) {
configuration.set(HAUtil.addSuffix(confKey, RM1_NODE_ID), RM1_ADDRESS);
configuration.set(HAUtil.addSuffix(confKey, RM2_NODE_ID), RM2_ADDRESS);
configuration.set(HAUtil.addSuffix(confKey, RM3_NODE_ID), RM3_ADDRESS);
}
// Enable webapp to test web-services also
configuration.setBoolean(MockRM.ENABLE_WEBAPP, true);
configuration.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true);
ClusterMetrics.destroy();
QueueMetrics.clearQueueMetrics();
DefaultMetricsSystem.shutdown();
}
11
View Complete Implementation : MRApp.java
Copyright Apache License 2.0
Author : aliyun-beta
Copyright Apache License 2.0
Author : aliyun-beta
public Job submit(Configuration conf, boolean mapSpeculative, boolean reduceSpeculative) throws Exception {
String user = conf.get(MRJobConfig.USER_NAME, UserGroupInformation.getCurrentUser().getShortUserName());
conf.set(MRJobConfig.USER_NAME, user);
conf.set(MRJobConfig.MR_AM_STAGING_DIR, testAbsPath.toString());
conf.setBoolean(MRJobConfig.MR_AM_CREATE_JH_INTERMEDIATE_BASE_DIR, true);
// TODO: fix the bug where the speculator gets events with
// not-fully-constructed objects. For now, disable speculative exec
conf.setBoolean(MRJobConfig.MAP_SPECULATIVE, mapSpeculative);
conf.setBoolean(MRJobConfig.REDUCE_SPECULATIVE, reduceSpeculative);
init(conf);
start();
DefaultMetricsSystem.shutdown();
Job job = getContext().getAllJobs().values().iterator().next();
if (replacedignedQueue != null) {
job.setQueueName(replacedignedQueue);
}
// Write job.xml
String jobFile = MRApps.getJobFile(conf, user, TypeConverter.fromYarn(job.getID()));
LOG.info("Writing job conf to " + jobFile);
new File(jobFile).getParentFile().mkdirs();
conf.writeXml(new FileOutputStream(jobFile));
return job;
}
9
View Complete Implementation : TestContainerManagerRecovery.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
@Test
public void testNodeManagerMetricsRecovery() throws Exception {
conf.setBoolean(YarnConfiguration.NM_RECOVERY_ENABLED, true);
conf.setBoolean(YarnConfiguration.NM_RECOVERY_SUPERVISED, true);
NMStateStoreService stateStore = new NMMemoryStateStoreService();
stateStore.init(conf);
stateStore.start();
Context context = createContext(conf, stateStore);
ContainerManagerImpl cm = createContainerManager(context, delSrvc);
cm.init(conf);
cm.start();
metrics.addResource(Resource.newInstance(10240, 8));
// add an application by starting a container
ApplicationId appId = ApplicationId.newInstance(0, 1);
ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(appId, 1);
ContainerId cid = ContainerId.newContainerId(attemptId, 1);
Map<String, String> containerEnv = Collections.emptyMap();
Map<String, ByteBuffer> serviceData = Collections.emptyMap();
Map<String, LocalResource> localResources = Collections.emptyMap();
List<String> commands = Arrays.asList("sleep 60s".split(" "));
ContainerLaunchContext clc = ContainerLaunchContext.newInstance(localResources, containerEnv, commands, serviceData, null, null);
StartContainersResponse startResponse = startContainer(context, cm, cid, clc, null, ContainerType.TASK);
replacedertTrue(startResponse.getFailedRequests().isEmpty());
replacedertEquals(1, context.getApplications().size());
Application app = context.getApplications().get(appId);
replacedertNotNull(app);
// make sure the container reaches RUNNING state
waitForNMContainerState(cm, cid, org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerState.RUNNING);
TestNodeManagerMetrics.checkMetrics(1, 0, 0, 0, 0, 1, 1, 1, 9, 1, 7);
// restart and verify metrics could be recovered
cm.stop();
DefaultMetricsSystem.shutdown();
metrics = NodeManagerMetrics.create();
metrics.addResource(Resource.newInstance(10240, 8));
TestNodeManagerMetrics.checkMetrics(0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 8);
context = createContext(conf, stateStore);
cm = createContainerManager(context, delSrvc);
cm.init(conf);
cm.start();
replacedertEquals(1, context.getApplications().size());
app = context.getApplications().get(appId);
replacedertNotNull(app);
TestNodeManagerMetrics.checkMetrics(1, 0, 0, 0, 0, 1, 1, 1, 9, 1, 7);
cm.stop();
}