Here are the examples of the java api org.apache.hadoop.metrics2.lib.MutableCounterLong taken from open source projects. By voting up you can indicate which examples are most useful and appropriate.
78 Examples
19
View Complete Implementation : SafeModeMetrics.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
/**
* This clreplaced is used for maintaining SafeMode metric information, which can
* be used for monitoring during SCM startup when SCM is still in SafeMode.
*/
public clreplaced SafeModeMetrics {
private static final String SOURCE_NAME = SafeModeMetrics.clreplaced.getSimpleName();
// These all values will be set to some values when safemode is enabled.
@Metric
private MutableCounterLong numContainerWithOneReplicaReportedThreshold;
@Metric
private MutableCounterLong currentContainersWithOneReplicaReportedCount;
// When hdds.scm.safemode.pipeline-availability.check is set then only
// below metrics will have some values, otherwise they will be zero.
@Metric
private MutableCounterLong numHealthyPipelinesThreshold;
@Metric
private MutableCounterLong currentHealthyPipelinesCount;
@Metric
private MutableCounterLong numPipelinesWithAtleastOneReplicaReportedThreshold;
@Metric
private MutableCounterLong currentPipelinesWithAtleastOneReplicaReportedCount;
public static SafeModeMetrics create() {
MetricsSystem ms = DefaultMetricsSystem.instance();
return ms.register(SOURCE_NAME, "SCM Safemode Metrics", new SafeModeMetrics());
}
public void setNumHealthyPipelinesThreshold(long val) {
this.numHealthyPipelinesThreshold.incr(val);
}
public void incCurrentHealthyPipelinesCount() {
this.currentHealthyPipelinesCount.incr();
}
public void setNumPipelinesWithAtleastOneReplicaReportedThreshold(long val) {
this.numPipelinesWithAtleastOneReplicaReportedThreshold.incr(val);
}
public void incCurrentHealthyPipelinesWithAtleastOneReplicaReportedCount() {
this.currentPipelinesWithAtleastOneReplicaReportedCount.incr();
}
public void setNumContainerWithOneReplicaReportedThreshold(long val) {
this.numContainerWithOneReplicaReportedThreshold.incr(val);
}
public void incCurrentContainersWithOneReplicaReportedCount() {
this.currentContainersWithOneReplicaReportedCount.incr();
}
public MutableCounterLong getNumHealthyPipelinesThreshold() {
return numHealthyPipelinesThreshold;
}
public MutableCounterLong getCurrentHealthyPipelinesCount() {
return currentHealthyPipelinesCount;
}
public MutableCounterLong getNumPipelinesWithAtleastOneReplicaReportedThreshold() {
return numPipelinesWithAtleastOneReplicaReportedThreshold;
}
public MutableCounterLong getCurrentPipelinesWithAtleastOneReplicaCount() {
return currentPipelinesWithAtleastOneReplicaReportedCount;
}
public MutableCounterLong getNumContainerWithOneReplicaReportedThreshold() {
return numContainerWithOneReplicaReportedThreshold;
}
public MutableCounterLong getCurrentContainersWithOneReplicaReportedCount() {
return currentContainersWithOneReplicaReportedCount;
}
public void unRegister() {
MetricsSystem ms = DefaultMetricsSystem.instance();
ms.unregisterSource(SOURCE_NAME);
}
}
19
View Complete Implementation : EventWatcherMetrics.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
/**
* Metrics for any event watcher.
*/
public clreplaced EvenreplacedcherMetrics {
@Metric
private MutableCounterLong trackedEvents;
@Metric
private MutableCounterLong timedOutEvents;
@Metric
private MutableCounterLong completedEvents;
@Metric
private MutableRate completionTime;
public void incrementTrackedEvents() {
trackedEvents.incr();
}
public void incrementTimedOutEvents() {
timedOutEvents.incr();
}
public void incrementCompletedEvents() {
completedEvents.incr();
}
public void updateFinishingTime(long duration) {
completionTime.add(duration);
}
@VisibleForTesting
public MutableCounterLong getTrackedEvents() {
return trackedEvents;
}
@VisibleForTesting
public MutableCounterLong getTimedOutEvents() {
return timedOutEvents;
}
@VisibleForTesting
public MutableCounterLong getCompletedEvents() {
return completedEvents;
}
@VisibleForTesting
public MutableRate getCompletionTime() {
return completionTime;
}
}
19
View Complete Implementation : FederationStateStoreClientMetrics.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
public static void failedStateStoreCall() {
String methodName = Thread.currentThread().getStackTrace()[2].getMethodName();
MutableCounterLong methodMetric = API_TO_FAILED_CALLS.get(methodName);
if (methodMetric == null) {
LOG.error(UNKOWN_FAIL_ERROR_MSG, methodName);
return;
}
totalFailedCalls.incr();
methodMetric.incr();
}
19
View Complete Implementation : RDBMetrics.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
/**
* Clreplaced to hold RocksDB metrics.
*/
public clreplaced RDBMetrics {
private static final String SOURCE_NAME = RDBMetrics.clreplaced.getSimpleName();
public RDBMetrics() {
}
public static RDBMetrics create() {
MetricsSystem ms = DefaultMetricsSystem.instance();
return ms.register(SOURCE_NAME, "Rocks DB Metrics", new RDBMetrics());
}
@Metric
private MutableCounterLong numDBKeyMayExistChecks;
@Metric
private MutableCounterLong numDBKeyMayExistMisses;
public void incNumDBKeyMayExistChecks() {
numDBKeyMayExistChecks.incr();
}
public void incNumDBKeyMayExistMisses() {
numDBKeyMayExistMisses.incr();
}
@VisibleForTesting
public long getNumDBKeyMayExistChecks() {
return numDBKeyMayExistChecks.value();
}
@VisibleForTesting
public long getNumDBKeyMayExistMisses() {
return numDBKeyMayExistMisses.value();
}
public void unRegister() {
MetricsSystem ms = DefaultMetricsSystem.instance();
ms.unregisterSource(SOURCE_NAME);
}
}
19
View Complete Implementation : S3AInstrumentation.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
/**
* Get the value of a counter.
* If the counter is null, return 0.
* @param name the name of the counter
* @return its value.
*/
public long getCounterValue(String name) {
MutableCounterLong counter = lookupCounter(name);
return counter == null ? 0 : counter.value();
}
19
View Complete Implementation : TestQueueMetricsForCustomResources.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
private Resource convertPreemptedSecondsToResource(QueueMetrics qm) {
QueueMetricsCustomResource customValues = qm.getAggregatedPreemptedSecondsResources();
MutableCounterLong vcoreSeconds = qm.getAggregateVcoreSecondsPreempted();
MutableCounterLong memorySeconds = qm.getAggregateMemoryMBSecondsPreempted();
return Resource.newInstance(memorySeconds.value(), (int) vcoreSeconds.value(), customValues.getValues());
}
18
View Complete Implementation : MetricsReplicationSourceSourceImpl.java
Copyright Apache License 2.0
Author : fengchen8086
Copyright Apache License 2.0
Author : fengchen8086
public clreplaced MetricsReplicationSourceSourceImpl implements MetricsReplicationSourceSource {
private final MetricsReplicationSourceImpl rms;
private final String id;
private final String sizeOfLogQueueKey;
private final String ageOfLastShippedOpKey;
private final String logReadInEditsKey;
private final String logEditsFilteredKey;
private final String shippedBatchesKey;
private final String shippedOpsKey;
private final String shippedKBsKey;
private final String logReadInBytesKey;
private final MutableGaugeLong ageOfLastShippedOpGauge;
private final MutableGaugeLong sizeOfLogQueueGauge;
private final MutableCounterLong logReadInEditsCounter;
private final MutableCounterLong logEditsFilteredCounter;
private final MutableCounterLong shippedBatchesCounter;
private final MutableCounterLong shippedOpsCounter;
private final MutableCounterLong shippedKBsCounter;
private final MutableCounterLong logReadInBytesCounter;
public MetricsReplicationSourceSourceImpl(MetricsReplicationSourceImpl rms, String id) {
this.rms = rms;
this.id = id;
ageOfLastShippedOpKey = "source." + id + ".ageOfLastShippedOp";
ageOfLastShippedOpGauge = rms.getMetricsRegistry().getLongGauge(ageOfLastShippedOpKey, 0L);
sizeOfLogQueueKey = "source." + id + ".sizeOfLogQueue";
sizeOfLogQueueGauge = rms.getMetricsRegistry().getLongGauge(sizeOfLogQueueKey, 0L);
shippedBatchesKey = "source." + this.id + ".shippedBatches";
shippedBatchesCounter = rms.getMetricsRegistry().getLongCounter(shippedBatchesKey, 0L);
shippedOpsKey = "source." + this.id + ".shippedOps";
shippedOpsCounter = rms.getMetricsRegistry().getLongCounter(shippedOpsKey, 0L);
shippedKBsKey = "source." + this.id + ".shippedKBs";
shippedKBsCounter = rms.getMetricsRegistry().getLongCounter(shippedKBsKey, 0L);
logReadInBytesKey = "source." + this.id + ".logReadInBytes";
logReadInBytesCounter = rms.getMetricsRegistry().getLongCounter(logReadInBytesKey, 0L);
logReadInEditsKey = "source." + id + ".logEditsRead";
logReadInEditsCounter = rms.getMetricsRegistry().getLongCounter(logReadInEditsKey, 0L);
logEditsFilteredKey = "source." + id + ".logEditsFiltered";
logEditsFilteredCounter = rms.getMetricsRegistry().getLongCounter(logEditsFilteredKey, 0L);
}
@Override
public void setLastShippedAge(long age) {
ageOfLastShippedOpGauge.set(age);
}
@Override
public void setSizeOfLogQueue(int size) {
sizeOfLogQueueGauge.set(size);
}
@Override
public void incrSizeOfLogQueue(int size) {
sizeOfLogQueueGauge.incr(size);
}
@Override
public void decrSizeOfLogQueue(int size) {
sizeOfLogQueueGauge.decr(size);
}
@Override
public void incrLogReadInEdits(long size) {
logReadInEditsCounter.incr(size);
}
@Override
public void incrLogEditsFiltered(long size) {
logEditsFilteredCounter.incr(size);
}
@Override
public void incrBatchesShipped(int batches) {
shippedBatchesCounter.incr(batches);
}
@Override
public void incrOpsShipped(long ops) {
shippedOpsCounter.incr(ops);
}
@Override
public void incrShippedKBs(long size) {
shippedKBsCounter.incr(size);
}
@Override
public void incrLogReadInBytes(long size) {
logReadInBytesCounter.incr(size);
}
@Override
public void clear() {
rms.removeMetric(ageOfLastShippedOpKey);
rms.removeMetric(sizeOfLogQueueKey);
rms.removeMetric(shippedBatchesKey);
rms.removeMetric(shippedOpsKey);
rms.removeMetric(shippedKBsKey);
rms.removeMetric(logReadInBytesKey);
rms.removeMetric(logReadInEditsKey);
rms.removeMetric(logEditsFilteredKey);
}
@Override
public long getLastShippedAge() {
return ageOfLastShippedOpGauge.value();
}
}
18
View Complete Implementation : MetricsRESTSourceImpl.java
Copyright Apache License 2.0
Author : fengchen8086
Copyright Apache License 2.0
Author : fengchen8086
/**
* Hadoop Two implementation of a metrics2 source that will export metrics from the Rest server to
* the hadoop metrics2 subsystem.
*
* Implements BaseSource through BaseSourceImpl, following the pattern
*/
@InterfaceAudience.Private
public clreplaced MetricsRESTSourceImpl extends BaseSourceImpl implements MetricsRESTSource {
private MutableCounterLong request;
private MutableCounterLong sucGet;
private MutableCounterLong sucPut;
private MutableCounterLong sucDel;
private MutableCounterLong sucScan;
private MutableCounterLong fGet;
private MutableCounterLong fPut;
private MutableCounterLong fDel;
private MutableCounterLong fScan;
public MetricsRESTSourceImpl() {
this(METRICS_NAME, METRICS_DESCRIPTION, CONTEXT, JMX_CONTEXT);
}
public MetricsRESTSourceImpl(String metricsName, String metricsDescription, String metricsContext, String metricsJmxContext) {
super(metricsName, metricsDescription, metricsContext, metricsJmxContext);
}
@Override
public void init() {
super.init();
request = getMetricsRegistry().getLongCounter(REQUEST_KEY, 0l);
sucGet = getMetricsRegistry().getLongCounter(SUCCESSFUL_GET_KEY, 0l);
sucPut = getMetricsRegistry().getLongCounter(SUCCESSFUL_PUT_KEY, 0l);
sucDel = getMetricsRegistry().getLongCounter(SUCCESSFUL_DELETE_KEY, 0l);
sucScan = getMetricsRegistry().getLongCounter(SUCCESSFUL_SCAN_KEY, 0L);
fGet = getMetricsRegistry().getLongCounter(FAILED_GET_KEY, 0l);
fPut = getMetricsRegistry().getLongCounter(FAILED_PUT_KEY, 0l);
fDel = getMetricsRegistry().getLongCounter(FAILED_DELETE_KEY, 0l);
fScan = getMetricsRegistry().getLongCounter(FAILED_SCAN_KEY, 0l);
}
@Override
public void incrementRequests(int inc) {
request.incr(inc);
}
@Override
public void incrementSucessfulGetRequests(int inc) {
sucGet.incr(inc);
}
@Override
public void incrementSucessfulPutRequests(int inc) {
sucPut.incr(inc);
}
@Override
public void incrementSucessfulDeleteRequests(int inc) {
sucDel.incr(inc);
}
@Override
public void incrementFailedGetRequests(int inc) {
fGet.incr(inc);
}
@Override
public void incrementFailedPutRequests(int inc) {
fPut.incr(inc);
}
@Override
public void incrementFailedDeleteRequests(int inc) {
fDel.incr(inc);
}
@Override
public void incrementSucessfulScanRequests(int inc) {
sucScan.incr(inc);
}
@Override
public void incrementFailedScanRequests(int inc) {
fScan.incr(inc);
}
}
18
View Complete Implementation : XceiverClientMetrics.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
/**
* The client metrics for the Storage Container protocol.
*/
@InterfaceAudience.Private
@Metrics(about = "Storage Container Client Metrics", context = "dfs")
public clreplaced XceiverClientMetrics {
public static final String SOURCE_NAME = XceiverClientMetrics.clreplaced.getSimpleName();
@Metric
private MutableCounterLong pendingOps;
@Metric
private MutableCounterLong totalOps;
private MutableCounterLong[] pendingOpsArray;
private MutableCounterLong[] opsArray;
private MutableRate[] containerOpsLatency;
private MetricsRegistry registry;
public XceiverClientMetrics() {
int numEnumEntries = ContainerProtos.Type.values().length;
this.registry = new MetricsRegistry(SOURCE_NAME);
this.pendingOpsArray = new MutableCounterLong[numEnumEntries];
this.opsArray = new MutableCounterLong[numEnumEntries];
this.containerOpsLatency = new MutableRate[numEnumEntries];
for (int i = 0; i < numEnumEntries; i++) {
pendingOpsArray[i] = registry.newCounter("numPending" + ContainerProtos.Type.forNumber(i + 1), "number of pending" + ContainerProtos.Type.forNumber(i + 1) + " ops", (long) 0);
opsArray[i] = registry.newCounter("opCount" + ContainerProtos.Type.forNumber(i + 1), "number of" + ContainerProtos.Type.forNumber(i + 1) + " ops", (long) 0);
containerOpsLatency[i] = registry.newRate(ContainerProtos.Type.forNumber(i + 1) + "Latency", "latency of " + ContainerProtos.Type.forNumber(i + 1) + " ops");
}
}
public static XceiverClientMetrics create() {
DefaultMetricsSystem.initialize(SOURCE_NAME);
MetricsSystem ms = DefaultMetricsSystem.instance();
return ms.register(SOURCE_NAME, "Storage Container Client Metrics", new XceiverClientMetrics());
}
public void incrPendingContainerOpsMetrics(ContainerProtos.Type type) {
pendingOps.incr();
totalOps.incr();
opsArray[type.ordinal()].incr();
pendingOpsArray[type.ordinal()].incr();
}
public void decrPendingContainerOpsMetrics(ContainerProtos.Type type) {
pendingOps.incr(-1);
pendingOpsArray[type.ordinal()].incr(-1);
}
public void addContainerOpsLatency(ContainerProtos.Type type, long latencyNanos) {
containerOpsLatency[type.ordinal()].add(latencyNanos);
}
public long getContainerOpsMetrics(ContainerProtos.Type type) {
return pendingOpsArray[type.ordinal()].value();
}
@VisibleForTesting
public long getTotalOpCount() {
return totalOps.value();
}
@VisibleForTesting
public long getContainerOpCountMetrics(ContainerProtos.Type type) {
return opsArray[type.ordinal()].value();
}
public void unRegister() {
MetricsSystem ms = DefaultMetricsSystem.instance();
ms.unregisterSource(SOURCE_NAME);
}
}
18
View Complete Implementation : CSMMetrics.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
/**
* This clreplaced is for maintaining Container State Machine statistics.
*/
@InterfaceAudience.Private
@Metrics(about = "Container State Machine Metrics", context = "dfs")
public clreplaced CSMMetrics {
public static final String SOURCE_NAME = CSMMetrics.clreplaced.getSimpleName();
// ratis op metrics metrics
@Metric
private MutableCounterLong numWriteStateMachineOps;
@Metric
private MutableCounterLong numQueryStateMachineOps;
@Metric
private MutableCounterLong numApplyTransactionOps;
@Metric
private MutableCounterLong numReadStateMachineOps;
@Metric
private MutableCounterLong numBytesWrittenCount;
@Metric
private MutableCounterLong numBytesCommittedCount;
@Metric
private MutableRate transactionLatency;
private MutableRate[] opsLatency;
private MetricsRegistry registry = null;
// Failure Metrics
@Metric
private MutableCounterLong numWriteStateMachineFails;
@Metric
private MutableCounterLong numWriteDataFails;
@Metric
private MutableCounterLong numQueryStateMachineFails;
@Metric
private MutableCounterLong numApplyTransactionFails;
@Metric
private MutableCounterLong numReadStateMachineFails;
@Metric
private MutableCounterLong numReadStateMachineMissCount;
@Metric
private MutableCounterLong numStartTransactionVerifyFailures;
@Metric
private MutableCounterLong numContainerNotOpenVerifyFailures;
@Metric
private MutableRate applyTransaction;
@Metric
private MutableRate writeStateMachineData;
public CSMMetrics() {
int numCmdTypes = ContainerProtos.Type.values().length;
this.opsLatency = new MutableRate[numCmdTypes];
this.registry = new MetricsRegistry(CSMMetrics.clreplaced.getSimpleName());
for (int i = 0; i < numCmdTypes; i++) {
opsLatency[i] = registry.newRate(ContainerProtos.Type.forNumber(i + 1).toString(), ContainerProtos.Type.forNumber(i + 1) + " op");
}
}
public static CSMMetrics create(RaftGroupId gid) {
MetricsSystem ms = DefaultMetricsSystem.instance();
return ms.register(SOURCE_NAME + gid.toString(), "Container State Machine", new CSMMetrics());
}
public void incNumWriteStateMachineOps() {
numWriteStateMachineOps.incr();
}
public void incNumQueryStateMachineOps() {
numQueryStateMachineOps.incr();
}
public void incNumReadStateMachineOps() {
numReadStateMachineOps.incr();
}
public void incNumApplyTransactionsOps() {
numApplyTransactionOps.incr();
}
public void incNumWriteStateMachineFails() {
numWriteStateMachineFails.incr();
}
public void incNumWriteDataFails() {
numWriteDataFails.incr();
}
public void incNumQueryStateMachineFails() {
numQueryStateMachineFails.incr();
}
public void incNumBytesWrittenCount(long value) {
numBytesWrittenCount.incr(value);
}
public void incNumBytesCommittedCount(long value) {
numBytesCommittedCount.incr(value);
}
public void incNumReadStateMachineFails() {
numReadStateMachineFails.incr();
}
public void incNumReadStateMachineMissCount() {
numReadStateMachineMissCount.incr();
}
public void incNumApplyTransactionsFails() {
numApplyTransactionFails.incr();
}
@VisibleForTesting
public long getNumWriteStateMachineOps() {
return numWriteStateMachineOps.value();
}
@VisibleForTesting
public long getNumQueryStateMachineOps() {
return numQueryStateMachineOps.value();
}
@VisibleForTesting
public long getNumApplyTransactionsOps() {
return numApplyTransactionOps.value();
}
@VisibleForTesting
public long getNumWriteStateMachineFails() {
return numWriteStateMachineFails.value();
}
@VisibleForTesting
public long getNumWriteDataFails() {
return numWriteDataFails.value();
}
@VisibleForTesting
public long getNumQueryStateMachineFails() {
return numQueryStateMachineFails.value();
}
@VisibleForTesting
public long getNumApplyTransactionsFails() {
return numApplyTransactionFails.value();
}
@VisibleForTesting
public long getNumReadStateMachineFails() {
return numReadStateMachineFails.value();
}
@VisibleForTesting
public long getNumReadStateMachineMissCount() {
return numReadStateMachineMissCount.value();
}
@VisibleForTesting
public long getNumReadStateMachineOps() {
return numReadStateMachineOps.value();
}
@VisibleForTesting
public long getNumBytesWrittenCount() {
return numBytesWrittenCount.value();
}
@VisibleForTesting
public long getNumBytesCommittedCount() {
return numBytesCommittedCount.value();
}
public MutableRate getApplyTransactionLatency() {
return applyTransaction;
}
public void incPipelineLatency(ContainerProtos.Type type, long latencyNanos) {
opsLatency[type.ordinal()].add(latencyNanos);
transactionLatency.add(latencyNanos);
}
public void incNumStartTransactionVerifyFailures() {
numStartTransactionVerifyFailures.incr();
}
public void incNumContainerNotOpenVerifyFailures() {
numContainerNotOpenVerifyFailures.incr();
}
public void recordApplyTransactionCompletion(long latencyNanos) {
applyTransaction.add(latencyNanos);
}
public void recordWriteStateMachineCompletion(long latencyNanos) {
writeStateMachineData.add(latencyNanos);
}
public void unRegister() {
MetricsSystem ms = DefaultMetricsSystem.instance();
ms.unregisterSource(SOURCE_NAME);
}
}
18
View Complete Implementation : SCMContainerManagerMetrics.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
/**
* Clreplaced contains metrics related to ContainerManager.
*/
@Metrics(about = "SCM ContainerManager metrics", context = OzoneConsts.OZONE)
public final clreplaced SCMContainerManagerMetrics {
private static final String SOURCE_NAME = SCMContainerManagerMetrics.clreplaced.getSimpleName();
// These are the metrics which will be reset to zero after restart.
// These metrics capture count of number of successful/failure operations
// of create/delete containers in SCM.
@Metric
private MutableCounterLong numSuccessfulCreateContainers;
@Metric
private MutableCounterLong numFailureCreateContainers;
@Metric
private MutableCounterLong numSuccessfulDeleteContainers;
@Metric
private MutableCounterLong numFailureDeleteContainers;
@Metric
private MutableCounterLong numListContainerOps;
@Metric
private MutableCounterLong numContainerReportsProcessedSuccessful;
@Metric
private MutableCounterLong numContainerReportsProcessedFailed;
@Metric
private MutableCounterLong numICRReportsProcessedSuccessful;
@Metric
private MutableCounterLong numICRReportsProcessedFailed;
private SCMContainerManagerMetrics() {
}
/**
* Create and return metrics instance.
* @return SCMContainerManagerMetrics
*/
public static SCMContainerManagerMetrics create() {
MetricsSystem ms = DefaultMetricsSystem.instance();
return ms.register(SOURCE_NAME, "SCM ContainerManager Metrics", new SCMContainerManagerMetrics());
}
/**
* Unregister metrics.
*/
public void unRegister() {
MetricsSystem ms = DefaultMetricsSystem.instance();
ms.unregisterSource(SOURCE_NAME);
}
public void incNumSuccessfulCreateContainers() {
this.numSuccessfulCreateContainers.incr();
}
public void incNumFailureCreateContainers() {
this.numFailureCreateContainers.incr();
}
public void incNumSuccessfulDeleteContainers() {
this.numSuccessfulDeleteContainers.incr();
}
public void incNumFailureDeleteContainers() {
this.numFailureDeleteContainers.incr();
}
public void incNumListContainersOps() {
this.numListContainerOps.incr();
}
public void incNumContainerReportsProcessedSuccessful() {
this.numContainerReportsProcessedSuccessful.incr();
}
public void incNumContainerReportsProcessedFailed() {
this.numContainerReportsProcessedFailed.incr();
}
public void incNumICRReportsProcessedSuccessful() {
this.numICRReportsProcessedSuccessful.incr();
}
public void incNumICRReportsProcessedFailed() {
this.numICRReportsProcessedFailed.incr();
}
public long getNumContainerReportsProcessedSuccessful() {
return numContainerReportsProcessedSuccessful.value();
}
public long getNumContainerReportsProcessedFailed() {
return numContainerReportsProcessedFailed.value();
}
public long getNumICRReportsProcessedSuccessful() {
return numICRReportsProcessedSuccessful.value();
}
public long getNumICRReportsProcessedFailed() {
return numICRReportsProcessedFailed.value();
}
public long getNumSuccessfulCreateContainers() {
return numSuccessfulCreateContainers.value();
}
public long getNumFailureCreateContainers() {
return numFailureCreateContainers.value();
}
public long getNumSuccessfulDeleteContainers() {
return numSuccessfulDeleteContainers.value();
}
public long getNumFailureDeleteContainers() {
return numFailureDeleteContainers.value();
}
public long getNumListContainersOps() {
return numListContainerOps.value();
}
}
18
View Complete Implementation : MetricsReplicationSinkSourceImpl.java
Copyright Apache License 2.0
Author : fengchen8086
Copyright Apache License 2.0
Author : fengchen8086
public clreplaced MetricsReplicationSinkSourceImpl implements MetricsReplicationSinkSource {
private final MutableGaugeLong ageGauge;
private final MutableCounterLong batchesCounter;
private final MutableCounterLong opsCounter;
public MetricsReplicationSinkSourceImpl(MetricsReplicationSourceImpl rms) {
ageGauge = rms.getMetricsRegistry().getLongGauge(SINK_AGE_OF_LAST_APPLIED_OP, 0L);
batchesCounter = rms.getMetricsRegistry().getLongCounter(SINK_APPLIED_BATCHES, 0L);
opsCounter = rms.getMetricsRegistry().getLongCounter(SINK_APPLIED_OPS, 0L);
}
@Override
public void setLastAppliedOpAge(long age) {
ageGauge.set(age);
}
@Override
public void incrAppliedBatches(long batches) {
batchesCounter.incr(batches);
}
@Override
public void incrAppliedOps(long batchsize) {
opsCounter.incr(batchsize);
}
@Override
public long getLastAppliedOpAge() {
return ageGauge.value();
}
}
18
View Complete Implementation : TestEntityGroupFSTimelineStore.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
@Test
public void testParseSummaryLogs() throws Exception {
TimelineDataManager tdm = PluginStoreTestUtils.getTdmWithMemStore(config);
MutableCounterLong scanned = store.metrics.getEnreplacediesReadToSummary();
long beforeScan = scanned.value();
EnreplacedyGroupFSTimelineStore.AppLogs appLogs = store.new AppLogs(mainTestAppId, mainTestAppDirPath, AppState.COMPLETED);
appLogs.scanForLogs();
appLogs.parseSummaryLogs(tdm);
PluginStoreTestUtils.verifyTestEnreplacedies(tdm);
replacedertEquals(beforeScan + 2L, scanned.value());
}
18
View Complete Implementation : MetricsHBaseServerSourceImpl.java
Copyright Apache License 2.0
Author : fengchen8086
Copyright Apache License 2.0
Author : fengchen8086
@InterfaceAudience.Private
public clreplaced MetricsHBaseServerSourceImpl extends BaseSourceImpl implements MetricsHBaseServerSource {
private final MetricsHBaseServerWrapper wrapper;
private final MutableCounterLong authorizationSuccesses;
private final MutableCounterLong authorizationFailures;
private final MutableCounterLong authenticationSuccesses;
private final MutableCounterLong authenticationFailures;
private final MutableCounterLong authenticationFallbacks;
private final MutableCounterLong sentBytes;
private final MutableCounterLong receivedBytes;
private final MutableCounterLong exceptions;
private final MutableCounterLong exceptionsOOO;
private final MutableCounterLong exceptionsBusy;
private final MutableCounterLong exceptionsUnknown;
private final MutableCounterLong exceptionsSanity;
private final MutableCounterLong exceptionsNSRE;
private final MutableCounterLong exceptionsMoved;
private final MutableCounterLong exceptionsMulreplacedooLarge;
private MutableHistogram queueCallTime;
private MutableHistogram processCallTime;
private MutableHistogram totalCallTime;
private MutableHistogram requestSize;
private MutableHistogram responseSize;
public MetricsHBaseServerSourceImpl(String metricsName, String metricsDescription, String metricsContext, String metricsJmxContext, MetricsHBaseServerWrapper wrapper) {
super(metricsName, metricsDescription, metricsContext, metricsJmxContext);
this.wrapper = wrapper;
this.authorizationSuccesses = this.getMetricsRegistry().newCounter(AUTHORIZATION_SUCCESSES_NAME, AUTHORIZATION_SUCCESSES_DESC, 0L);
this.authorizationFailures = this.getMetricsRegistry().newCounter(AUTHORIZATION_FAILURES_NAME, AUTHORIZATION_FAILURES_DESC, 0L);
this.exceptions = this.getMetricsRegistry().newCounter(EXCEPTIONS_NAME, EXCEPTIONS_DESC, 0L);
this.exceptionsOOO = this.getMetricsRegistry().newCounter(EXCEPTIONS_OOO_NAME, EXCEPTIONS_TYPE_DESC, 0L);
this.exceptionsBusy = this.getMetricsRegistry().newCounter(EXCEPTIONS_BUSY_NAME, EXCEPTIONS_TYPE_DESC, 0L);
this.exceptionsUnknown = this.getMetricsRegistry().newCounter(EXCEPTIONS_UNKNOWN_NAME, EXCEPTIONS_TYPE_DESC, 0L);
this.exceptionsSanity = this.getMetricsRegistry().newCounter(EXCEPTIONS_SANITY_NAME, EXCEPTIONS_TYPE_DESC, 0L);
this.exceptionsMoved = this.getMetricsRegistry().newCounter(EXCEPTIONS_MOVED_NAME, EXCEPTIONS_TYPE_DESC, 0L);
this.exceptionsNSRE = this.getMetricsRegistry().newCounter(EXCEPTIONS_NSRE_NAME, EXCEPTIONS_TYPE_DESC, 0L);
this.exceptionsMulreplacedooLarge = this.getMetricsRegistry().newCounter(EXCEPTIONS_MULTI_TOO_LARGE_NAME, EXCEPTIONS_MULTI_TOO_LARGE_DESC, 0L);
this.authenticationSuccesses = this.getMetricsRegistry().newCounter(AUTHENTICATION_SUCCESSES_NAME, AUTHENTICATION_SUCCESSES_DESC, 0L);
this.authenticationFailures = this.getMetricsRegistry().newCounter(AUTHENTICATION_FAILURES_NAME, AUTHENTICATION_FAILURES_DESC, 0L);
this.authenticationFallbacks = this.getMetricsRegistry().newCounter(AUTHENTICATION_FALLBACKS_NAME, AUTHENTICATION_FALLBACKS_DESC, 0L);
this.sentBytes = this.getMetricsRegistry().newCounter(SENT_BYTES_NAME, SENT_BYTES_DESC, 0L);
this.receivedBytes = this.getMetricsRegistry().newCounter(RECEIVED_BYTES_NAME, RECEIVED_BYTES_DESC, 0L);
this.queueCallTime = this.getMetricsRegistry().newTimeHistogram(QUEUE_CALL_TIME_NAME, QUEUE_CALL_TIME_DESC);
this.processCallTime = this.getMetricsRegistry().newTimeHistogram(PROCESS_CALL_TIME_NAME, PROCESS_CALL_TIME_DESC);
this.totalCallTime = this.getMetricsRegistry().newTimeHistogram(TOTAL_CALL_TIME_NAME, TOTAL_CALL_TIME_DESC);
this.requestSize = this.getMetricsRegistry().newSizeHistogram(REQUEST_SIZE_NAME, REQUEST_SIZE_DESC);
this.responseSize = this.getMetricsRegistry().newSizeHistogram(RESPONSE_SIZE_NAME, RESPONSE_SIZE_DESC);
}
@Override
public void authorizationSuccess() {
authorizationSuccesses.incr();
}
@Override
public void authorizationFailure() {
authorizationFailures.incr();
}
@Override
public void authenticationFailure() {
authenticationFailures.incr();
}
@Override
public void authenticationFallback() {
authenticationFallbacks.incr();
}
@Override
public void exception() {
exceptions.incr();
}
@Override
public void outOfOrderException() {
exceptionsOOO.incr();
}
@Override
public void failedSanityException() {
exceptionsSanity.incr();
}
@Override
public void movedRegionException() {
exceptionsMoved.incr();
}
@Override
public void notServingRegionException() {
exceptionsNSRE.incr();
}
@Override
public void unknownScannerException() {
exceptionsUnknown.incr();
}
@Override
public void tooBusyException() {
exceptionsBusy.incr();
}
@Override
public void multiActionTooLargeException() {
exceptionsMulreplacedooLarge.incr();
}
@Override
public void authenticationSuccess() {
authenticationSuccesses.incr();
}
@Override
public void sentBytes(long count) {
this.sentBytes.incr(count);
}
@Override
public void receivedBytes(int count) {
this.receivedBytes.incr(count);
}
@Override
public void sentResponse(long count) {
this.responseSize.add(count);
}
@Override
public void receivedRequest(long count) {
this.requestSize.add(count);
}
@Override
public void dequeuedCall(int qTime) {
queueCallTime.add(qTime);
}
@Override
public void processedCall(int processingTime) {
processCallTime.add(processingTime);
}
@Override
public void queuedAndProcessedCall(int totalTime) {
totalCallTime.add(totalTime);
}
@Override
public void getMetrics(MetricsCollector metricsCollector, boolean all) {
MetricsRecordBuilder mrb = metricsCollector.addRecord(metricsName);
if (wrapper != null) {
mrb.addGauge(Interns.info(QUEUE_SIZE_NAME, QUEUE_SIZE_DESC), wrapper.getTotalQueueSize()).addGauge(Interns.info(GENERAL_QUEUE_NAME, GENERAL_QUEUE_DESC), wrapper.getGeneralQueueLength()).addGauge(Interns.info(REPLICATION_QUEUE_NAME, REPLICATION_QUEUE_DESC), wrapper.getReplicationQueueLength()).addGauge(Interns.info(PRIORITY_QUEUE_NAME, PRIORITY_QUEUE_DESC), wrapper.getPriorityQueueLength()).addGauge(Interns.info(NUM_OPEN_CONNECTIONS_NAME, NUM_OPEN_CONNECTIONS_DESC), wrapper.getNumOpenConnections()).addGauge(Interns.info(NUM_ACTIVE_HANDLER_NAME, NUM_ACTIVE_HANDLER_DESC), wrapper.getActiveRpcHandlerCount());
}
metricsRegistry.snapshot(mrb, all);
}
}
18
View Complete Implementation : MetricsMasterSourceImpl.java
Copyright Apache License 2.0
Author : fengchen8086
Copyright Apache License 2.0
Author : fengchen8086
/**
* Hadoop2 implementation of MetricsMasterSource.
*
* Implements BaseSource through BaseSourceImpl, following the pattern
*/
@InterfaceAudience.Private
public clreplaced MetricsMasterSourceImpl extends BaseSourceImpl implements MetricsMasterSource {
private final MetricsMasterWrapper masterWrapper;
private MutableCounterLong clusterRequestsCounter;
public MetricsMasterSourceImpl(MetricsMasterWrapper masterWrapper) {
this(METRICS_NAME, METRICS_DESCRIPTION, METRICS_CONTEXT, METRICS_JMX_CONTEXT, masterWrapper);
}
public MetricsMasterSourceImpl(String metricsName, String metricsDescription, String metricsContext, String metricsJmxContext, MetricsMasterWrapper masterWrapper) {
super(metricsName, metricsDescription, metricsContext, metricsJmxContext);
this.masterWrapper = masterWrapper;
}
@Override
public void init() {
super.init();
clusterRequestsCounter = metricsRegistry.newCounter(CLUSTER_REQUESTS_NAME, "", 0l);
}
@Override
public void incRequests(final long inc) {
this.clusterRequestsCounter.incr(inc);
}
@Override
public void getMetrics(MetricsCollector metricsCollector, boolean all) {
MetricsRecordBuilder metricsRecordBuilder = metricsCollector.addRecord(metricsName);
// masterWrapper can be null because this function is called inside of init.
if (masterWrapper != null) {
metricsRecordBuilder.addGauge(Interns.info(MASTER_ACTIVE_TIME_NAME, MASTER_ACTIVE_TIME_DESC), masterWrapper.getActiveTime()).addGauge(Interns.info(MASTER_START_TIME_NAME, MASTER_START_TIME_DESC), masterWrapper.getStartTime()).addGauge(Interns.info(AVERAGE_LOAD_NAME, AVERAGE_LOAD_DESC), masterWrapper.getAverageLoad()).tag(Interns.info(LIVE_REGION_SERVERS_NAME, LIVE_REGION_SERVERS_DESC), masterWrapper.getRegionServers()).addGauge(Interns.info(NUM_REGION_SERVERS_NAME, NUMBER_OF_REGION_SERVERS_DESC), masterWrapper.getNumRegionServers()).tag(Interns.info(DEAD_REGION_SERVERS_NAME, DEAD_REGION_SERVERS_DESC), masterWrapper.getDeadRegionServers()).addGauge(Interns.info(NUM_DEAD_REGION_SERVERS_NAME, NUMBER_OF_DEAD_REGION_SERVERS_DESC), masterWrapper.getNumDeadRegionServers()).tag(Interns.info(ZOOKEEPER_QUORUM_NAME, ZOOKEEPER_QUORUM_DESC), masterWrapper.getZookeeperQuorum()).tag(Interns.info(SERVER_NAME_NAME, SERVER_NAME_DESC), masterWrapper.getServerName()).tag(Interns.info(CLUSTER_ID_NAME, CLUSTER_ID_DESC), masterWrapper.getClusterId()).tag(Interns.info(IS_ACTIVE_MASTER_NAME, IS_ACTIVE_MASTER_DESC), String.valueOf(masterWrapper.getIsActiveMaster()));
}
metricsRegistry.snapshot(metricsRecordBuilder, all);
}
}
18
View Complete Implementation : BaseSourceImpl.java
Copyright Apache License 2.0
Author : fengchen8086
Copyright Apache License 2.0
Author : fengchen8086
/**
* Increment a named counter by some value.
*
* @param key the name of the counter
* @param delta the ammount to increment
*/
public void incCounters(String key, long delta) {
MutableCounterLong counter = metricsRegistry.getLongCounter(key, 0l);
counter.incr(delta);
}
18
View Complete Implementation : TabletServerUpdateMetrics.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
public clreplaced TabletServerUpdateMetrics extends TServerMetrics {
private final MutableCounterLong permissionErrorsCounter;
private final MutableCounterLong unknownTabletErrorsCounter;
private final MutableCounterLong constraintViolationsCounter;
private final MutableStat commitPrepStat;
private final MutableStat walogWriteTimeStat;
private final MutableStat commitTimeStat;
private final MutableStat mutationArraySizeStat;
public TabletServerUpdateMetrics() {
super("Updates");
MetricsRegistry registry = super.getRegistry();
permissionErrorsCounter = registry.newCounter("permissionErrors", "Permission Errors", 0L);
unknownTabletErrorsCounter = registry.newCounter("unknownTabletErrors", "Unknown Tablet Errors", 0L);
constraintViolationsCounter = registry.newCounter("constraintViolations", "Table Constraint Violations", 0L);
commitPrepStat = registry.newStat("commitPrep", "preparing to commit mutations", "Ops", "Time", true);
walogWriteTimeStat = registry.newStat("waLogWriteTime", "writing mutations to WAL", "Ops", "Time", true);
commitTimeStat = registry.newStat("commitTime", "committing mutations", "Ops", "Time", true);
mutationArraySizeStat = registry.newStat("mutationArraysSize", "mutation array", "ops", "Size", true);
}
public void addPermissionErrors(long value) {
permissionErrorsCounter.incr(value);
}
public void addUnknownTabletErrors(long value) {
unknownTabletErrorsCounter.incr(value);
}
public void addMutationArraySize(long value) {
mutationArraySizeStat.add(value);
}
public void addCommitPrep(long value) {
commitPrepStat.add(value);
}
public void addConstraintViolations(long value) {
constraintViolationsCounter.incr(value);
}
public void addWalogWriteTime(long value) {
walogWriteTimeStat.add(value);
}
public void addCommitTime(long value) {
commitTimeStat.add(value);
}
}
18
View Complete Implementation : MetricsReplicationGlobalSourceSource.java
Copyright Apache License 2.0
Author : fengchen8086
Copyright Apache License 2.0
Author : fengchen8086
public clreplaced MetricsReplicationGlobalSourceSource implements MetricsReplicationSourceSource {
private final MetricsReplicationSourceImpl rms;
private final MutableGaugeLong ageOfLastShippedOpGauge;
private final MutableGaugeLong sizeOfLogQueueGauge;
private final MutableCounterLong logReadInEditsCounter;
private final MutableCounterLong logEditsFilteredCounter;
private final MutableCounterLong shippedBatchesCounter;
private final MutableCounterLong shippedOpsCounter;
private final MutableCounterLong shippedKBsCounter;
private final MutableCounterLong logReadInBytesCounter;
public MetricsReplicationGlobalSourceSource(MetricsReplicationSourceImpl rms) {
this.rms = rms;
ageOfLastShippedOpGauge = rms.getMetricsRegistry().getLongGauge(SOURCE_AGE_OF_LAST_SHIPPED_OP, 0L);
sizeOfLogQueueGauge = rms.getMetricsRegistry().getLongGauge(SOURCE_SIZE_OF_LOG_QUEUE, 0L);
shippedBatchesCounter = rms.getMetricsRegistry().getLongCounter(SOURCE_SHIPPED_BATCHES, 0L);
shippedOpsCounter = rms.getMetricsRegistry().getLongCounter(SOURCE_SHIPPED_OPS, 0L);
shippedKBsCounter = rms.getMetricsRegistry().getLongCounter(SOURCE_SHIPPED_KBS, 0L);
logReadInBytesCounter = rms.getMetricsRegistry().getLongCounter(SOURCE_LOG_READ_IN_BYTES, 0L);
logReadInEditsCounter = rms.getMetricsRegistry().getLongCounter(SOURCE_LOG_READ_IN_EDITS, 0L);
logEditsFilteredCounter = rms.getMetricsRegistry().getLongCounter(SOURCE_LOG_EDITS_FILTERED, 0L);
}
@Override
public void setLastShippedAge(long age) {
ageOfLastShippedOpGauge.set(age);
}
@Override
public void setSizeOfLogQueue(int size) {
sizeOfLogQueueGauge.set(size);
}
@Override
public void incrSizeOfLogQueue(int size) {
sizeOfLogQueueGauge.incr(size);
}
@Override
public void decrSizeOfLogQueue(int size) {
sizeOfLogQueueGauge.decr(size);
}
@Override
public void incrLogReadInEdits(long size) {
logReadInEditsCounter.incr(size);
}
@Override
public void incrLogEditsFiltered(long size) {
logEditsFilteredCounter.incr(size);
}
@Override
public void incrBatchesShipped(int batches) {
shippedBatchesCounter.incr(batches);
}
@Override
public void incrOpsShipped(long ops) {
shippedOpsCounter.incr(ops);
}
@Override
public void incrShippedKBs(long size) {
shippedKBsCounter.incr(size);
}
@Override
public void incrLogReadInBytes(long size) {
logReadInBytesCounter.incr(size);
}
@Override
public void clear() {
}
@Override
public long getLastShippedAge() {
return ageOfLastShippedOpGauge.value();
}
}
17
View Complete Implementation : RpcMetrics.java
Copyright Apache License 2.0
Author : aliyun-beta
Copyright Apache License 2.0
Author : aliyun-beta
/**
* This clreplaced is for maintaining the various RPC statistics
* and publishing them through the metrics interfaces.
*/
@InterfaceAudience.Private
@Metrics(about = "Aggregate RPC metrics", context = "rpc")
public clreplaced RpcMetrics {
static final Log LOG = LogFactory.getLog(RpcMetrics.clreplaced);
final Server server;
final MetricsRegistry registry;
final String name;
final boolean rpcQuantileEnable;
RpcMetrics(Server server, Configuration conf) {
String port = String.valueOf(server.getListenerAddress().getPort());
name = "RpcActivityForPort" + port;
this.server = server;
registry = new MetricsRegistry("rpc").tag("port", "RPC port", port);
int[] intervals = conf.getInts(CommonConfigurationKeys.RPC_METRICS_PERCENTILES_INTERVALS_KEY);
rpcQuantileEnable = (intervals.length > 0) && conf.getBoolean(CommonConfigurationKeys.RPC_METRICS_QUANTILE_ENABLE, CommonConfigurationKeys.RPC_METRICS_QUANTILE_ENABLE_DEFAULT);
if (rpcQuantileEnable) {
rpcQueueTimeMillisQuantiles = new MutableQuantiles[intervals.length];
rpcProcessingTimeMillisQuantiles = new MutableQuantiles[intervals.length];
for (int i = 0; i < intervals.length; i++) {
int interval = intervals[i];
rpcQueueTimeMillisQuantiles[i] = registry.newQuantiles("rpcQueueTime" + interval + "s", "rpc queue time in milli second", "ops", "latency", interval);
rpcProcessingTimeMillisQuantiles[i] = registry.newQuantiles("rpcProcessingTime" + interval + "s", "rpc processing time in milli second", "ops", "latency", interval);
}
}
LOG.debug("Initialized " + registry);
}
public String name() {
return name;
}
public static RpcMetrics create(Server server, Configuration conf) {
RpcMetrics m = new RpcMetrics(server, conf);
return DefaultMetricsSystem.instance().register(m.name, null, m);
}
@Metric("Number of received bytes")
MutableCounterLong receivedBytes;
@Metric("Number of sent bytes")
MutableCounterLong sentBytes;
@Metric("Queue time")
MutableRate rpcQueueTime;
MutableQuantiles[] rpcQueueTimeMillisQuantiles;
@Metric("Processing time")
MutableRate rpcProcessingTime;
MutableQuantiles[] rpcProcessingTimeMillisQuantiles;
@Metric("Number of authentication failures")
MutableCounterLong rpcAuthenticationFailures;
@Metric("Number of authentication successes")
MutableCounterLong rpcAuthenticationSuccesses;
@Metric("Number of authorization failures")
MutableCounterLong rpcAuthorizationFailures;
@Metric("Number of authorization successes")
MutableCounterLong rpcAuthorizationSuccesses;
@Metric("Number of client backoff requests")
MutableCounterLong rpcClientBackoff;
@Metric("Number of Slow RPC calls")
MutableCounterLong rpcSlowCalls;
@Metric("Number of open connections")
public int numOpenConnections() {
return server.getNumOpenConnections();
}
@Metric("Length of the call queue")
public int callQueueLength() {
return server.getCallQueueLen();
}
// Public instrumentation methods that could be extracted to an
// abstract clreplaced if we decide to do custom instrumentation clreplacedes a la
// JobTrackerInstrumentation. The methods with //@Override comment are
// candidates for abstract methods in a abstract instrumentation clreplaced.
/**
* One authentication failure event
*/
// @Override
public void incrAuthenticationFailures() {
rpcAuthenticationFailures.incr();
}
/**
* One authentication success event
*/
// @Override
public void incrAuthenticationSuccesses() {
rpcAuthenticationSuccesses.incr();
}
/**
* One authorization success event
*/
// @Override
public void incrAuthorizationSuccesses() {
rpcAuthorizationSuccesses.incr();
}
/**
* One authorization failure event
*/
// @Override
public void incrAuthorizationFailures() {
rpcAuthorizationFailures.incr();
}
/**
* Shutdown the instrumentation for the process
*/
// @Override
public void shutdown() {
}
/**
* Increment sent bytes by count
* @param count to increment
*/
// @Override
public void incrSentBytes(int count) {
sentBytes.incr(count);
}
/**
* Increment received bytes by count
* @param count to increment
*/
// @Override
public void incrReceivedBytes(int count) {
receivedBytes.incr(count);
}
/**
* Add an RPC queue time sample
* @param qTime the queue time
*/
// @Override
public void addRpcQueueTime(int qTime) {
rpcQueueTime.add(qTime);
if (rpcQuantileEnable) {
for (MutableQuantiles q : rpcQueueTimeMillisQuantiles) {
q.add(qTime);
}
}
}
/**
* Add an RPC processing time sample
* @param processingTime the processing time
*/
// @Override
public void addRpcProcessingTime(int processingTime) {
rpcProcessingTime.add(processingTime);
if (rpcQuantileEnable) {
for (MutableQuantiles q : rpcProcessingTimeMillisQuantiles) {
q.add(processingTime);
}
}
}
/**
* One client backoff event
*/
// @Override
public void incrClientBackoff() {
rpcClientBackoff.incr();
}
/**
* Increments the Slow RPC counter.
*/
public void incrSlowRpc() {
rpcSlowCalls.incr();
}
/**
* Returns a MutableRate Counter.
* @return Mutable Rate
*/
public MutableRate getRpcProcessingTime() {
return rpcProcessingTime;
}
/**
* Returns the number of samples that we have seen so far.
* @return long
*/
public long getProcessingSampleCount() {
return rpcProcessingTime.lastStat().numSamples();
}
/**
* Returns mean of RPC Processing Times.
* @return double
*/
public double getProcessingMean() {
return rpcProcessingTime.lastStat().mean();
}
/**
* Return Standard Deviation of the Processing Time.
* @return double
*/
public double getProcessingStdDev() {
return rpcProcessingTime.lastStat().stddev();
}
/**
* Returns the number of slow calls.
* @return long
*/
public long getRpcSlowCalls() {
return rpcSlowCalls.value();
}
}
17
View Complete Implementation : S3AInstrumentation.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
/**
* Increment a specific counter.
* No-op if not defined.
* @param op operation
* @param count increment value
*/
public void incrementCounter(Statistic op, long count) {
MutableCounterLong counter = lookupCounter(op.getSymbol());
if (counter != null) {
counter.incr(count);
}
}
17
View Complete Implementation : ContainerMetrics.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
/**
* This clreplaced is for maintaining the various Storage Container
* DataNode statistics and publishing them through the metrics interfaces.
* This also registers the JMX MBean for RPC.
* <p>
* This clreplaced has a number of metrics variables that are publicly accessible;
* these variables (objects) have methods to update their values;
* for example:
* <p> {@link #numOps}.inc()
*/
@InterfaceAudience.Private
@Metrics(about = "Storage Container DataNode Metrics", context = "dfs")
public clreplaced ContainerMetrics {
public static final String STORAGE_CONTAINER_METRICS = "StorageContainerMetrics";
@Metric
private MutableCounterLong numOps;
private MutableCounterLong[] numOpsArray;
private MutableCounterLong[] opsBytesArray;
private MutableRate[] opsLatency;
private MutableQuantiles[][] opsLatQuantiles;
private MetricsRegistry registry = null;
public ContainerMetrics(int[] intervals) {
int numEnumEntries = ContainerProtos.Type.values().length;
final int len = intervals.length;
this.numOpsArray = new MutableCounterLong[numEnumEntries];
this.opsBytesArray = new MutableCounterLong[numEnumEntries];
this.opsLatency = new MutableRate[numEnumEntries];
this.opsLatQuantiles = new MutableQuantiles[numEnumEntries][len];
this.registry = new MetricsRegistry("StorageContainerMetrics");
for (int i = 0; i < numEnumEntries; i++) {
numOpsArray[i] = registry.newCounter("num" + ContainerProtos.Type.forNumber(i + 1), "number of " + ContainerProtos.Type.forNumber(i + 1) + " ops", (long) 0);
opsBytesArray[i] = registry.newCounter("bytes" + ContainerProtos.Type.forNumber(i + 1), "bytes used by " + ContainerProtos.Type.forNumber(i + 1) + "op", (long) 0);
opsLatency[i] = registry.newRate("latency" + ContainerProtos.Type.forNumber(i + 1), ContainerProtos.Type.forNumber(i + 1) + " op");
for (int j = 0; j < len; j++) {
int interval = intervals[j];
String quantileName = ContainerProtos.Type.forNumber(i + 1) + "Nanos" + interval + "s";
opsLatQuantiles[i][j] = registry.newQuantiles(quantileName, "latency of Container ops", "ops", "latency", interval);
}
}
}
public static ContainerMetrics create(Configuration conf) {
MetricsSystem ms = DefaultMetricsSystem.instance();
// Percentile measurement is off by default, by watching no intervals
int[] intervals = conf.getInts(DFSConfigKeys.DFS_METRICS_PERCENTILES_INTERVALS_KEY);
return ms.register(STORAGE_CONTAINER_METRICS, "Storage Container Node Metrics", new ContainerMetrics(intervals));
}
public static void remove() {
MetricsSystem ms = DefaultMetricsSystem.instance();
ms.unregisterSource(STORAGE_CONTAINER_METRICS);
}
public void incContainerOpsMetrics(ContainerProtos.Type type) {
numOps.incr();
numOpsArray[type.ordinal()].incr();
}
public long getContainerOpsMetrics(ContainerProtos.Type type) {
return numOpsArray[type.ordinal()].value();
}
public void incContainerOpsLatencies(ContainerProtos.Type type, long latencyNanos) {
opsLatency[type.ordinal()].add(latencyNanos);
for (MutableQuantiles q : opsLatQuantiles[type.ordinal()]) {
q.add(latencyNanos);
}
}
public void incContainerBytesStats(ContainerProtos.Type type, long bytes) {
opsBytesArray[type.ordinal()].incr(bytes);
}
public long getContainerBytesMetrics(ContainerProtos.Type type) {
return opsBytesArray[type.ordinal()].value();
}
}
17
View Complete Implementation : SingleThreadExecutor.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
/**
* Simple EventExecutor to call all the event handler one-by-one.
*
* @param <P> the payload type of events
*/
@Metrics(context = "EventQueue")
public clreplaced SingleThreadExecutor<P> implements EventExecutor<P> {
private static final String EVENT_QUEUE = "EventQueue";
private static final Logger LOG = LoggerFactory.getLogger(SingleThreadExecutor.clreplaced);
private final String name;
private final ExecutorService executor;
@Metric
private MutableCounterLong queued;
@Metric
private MutableCounterLong done;
@Metric
private MutableCounterLong failed;
/**
* Create SingleThreadExecutor.
*
* @param name Unique name used in monitoring and metrics.
*/
public SingleThreadExecutor(String name) {
this.name = name;
DefaultMetricsSystem.instance().register(EVENT_QUEUE + name, "Event Executor metrics ", this);
executor = Executors.newSingleThreadExecutor(runnable -> {
Thread thread = new Thread(runnable);
thread.setName(EVENT_QUEUE + "-" + name);
return thread;
});
}
@Override
public void onMessage(EventHandler<P> handler, P message, EventPublisher publisher) {
queued.incr();
executor.execute(() -> {
try {
handler.onMessage(message, publisher);
done.incr();
} catch (Exception ex) {
LOG.error("Error on execution message {}", message, ex);
failed.incr();
}
});
}
@Override
public long failedEvents() {
return failed.value();
}
@Override
public long successfulEvents() {
return done.value();
}
@Override
public long queuedEvents() {
return queued.value();
}
@Override
public void close() {
executor.shutdown();
}
@Override
public String getName() {
return name;
}
}
17
View Complete Implementation : SCMMetrics.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
/**
* This clreplaced is for maintaining StorageContainerManager statistics.
*/
@Metrics(about = "Storage Container Manager Metrics", context = "dfs")
public clreplaced SCMMetrics {
public static final String SOURCE_NAME = SCMMetrics.clreplaced.getSimpleName();
/**
* Container stat metrics, the meaning of following metrics
* can be found in {@link ContainerStat}.
*/
@Metric
private MutableGaugeLong lastContainerReportSize;
@Metric
private MutableGaugeLong lastContainerReportUsed;
@Metric
private MutableGaugeLong lastContainerReportKeyCount;
@Metric
private MutableGaugeLong lastContainerReportReadBytes;
@Metric
private MutableGaugeLong lastContainerReportWriteBytes;
@Metric
private MutableGaugeLong lastContainerReportReadCount;
@Metric
private MutableGaugeLong lastContainerReportWriteCount;
@Metric
private MutableCounterLong containerReportSize;
@Metric
private MutableCounterLong containerReportUsed;
@Metric
private MutableCounterLong containerReportKeyCount;
@Metric
private MutableCounterLong containerReportReadBytes;
@Metric
private MutableCounterLong containerReportWriteBytes;
@Metric
private MutableCounterLong containerReportReadCount;
@Metric
private MutableCounterLong containerReportWriteCount;
public SCMMetrics() {
}
public static SCMMetrics create() {
MetricsSystem ms = DefaultMetricsSystem.instance();
return ms.register(SOURCE_NAME, "Storage Container Manager Metrics", new SCMMetrics());
}
public void setLastContainerReportSize(long size) {
this.lastContainerReportSize.set(size);
}
public void setLastContainerReportUsed(long used) {
this.lastContainerReportUsed.set(used);
}
public void setLastContainerReportKeyCount(long keyCount) {
this.lastContainerReportKeyCount.set(keyCount);
}
public void setLastContainerReportReadBytes(long readBytes) {
this.lastContainerReportReadBytes.set(readBytes);
}
public void setLastContainerReportWriteBytes(long writeBytes) {
this.lastContainerReportWriteBytes.set(writeBytes);
}
public void setLastContainerReportReadCount(long readCount) {
this.lastContainerReportReadCount.set(readCount);
}
public void setLastContainerReportWriteCount(long writeCount) {
this.lastContainerReportWriteCount.set(writeCount);
}
public void incrContainerReportSize(long size) {
this.containerReportSize.incr(size);
}
public void incrContainerReportUsed(long used) {
this.containerReportUsed.incr(used);
}
public void incrContainerReportKeyCount(long keyCount) {
this.containerReportKeyCount.incr(keyCount);
}
public void incrContainerReportReadBytes(long readBytes) {
this.containerReportReadBytes.incr(readBytes);
}
public void incrContainerReportWriteBytes(long writeBytes) {
this.containerReportWriteBytes.incr(writeBytes);
}
public void incrContainerReportReadCount(long readCount) {
this.containerReportReadCount.incr(readCount);
}
public void incrContainerReportWriteCount(long writeCount) {
this.containerReportWriteCount.incr(writeCount);
}
public void setLastContainerStat(ContainerStat newStat) {
this.lastContainerReportSize.set(newStat.getSize().get());
this.lastContainerReportUsed.set(newStat.getUsed().get());
this.lastContainerReportKeyCount.set(newStat.getKeyCount().get());
this.lastContainerReportReadBytes.set(newStat.getReadBytes().get());
this.lastContainerReportWriteBytes.set(newStat.getWriteBytes().get());
this.lastContainerReportReadCount.set(newStat.getReadCount().get());
this.lastContainerReportWriteCount.set(newStat.getWriteCount().get());
}
public void incrContainerStat(ContainerStat deltaStat) {
this.containerReportSize.incr(deltaStat.getSize().get());
this.containerReportUsed.incr(deltaStat.getUsed().get());
this.containerReportKeyCount.incr(deltaStat.getKeyCount().get());
this.containerReportReadBytes.incr(deltaStat.getReadBytes().get());
this.containerReportWriteBytes.incr(deltaStat.getWriteBytes().get());
this.containerReportReadCount.incr(deltaStat.getReadCount().get());
this.containerReportWriteCount.incr(deltaStat.getWriteCount().get());
}
public void decrContainerStat(ContainerStat deltaStat) {
this.containerReportSize.incr(-1 * deltaStat.getSize().get());
this.containerReportUsed.incr(-1 * deltaStat.getUsed().get());
this.containerReportKeyCount.incr(-1 * deltaStat.getKeyCount().get());
this.containerReportReadBytes.incr(-1 * deltaStat.getReadBytes().get());
this.containerReportWriteBytes.incr(-1 * deltaStat.getWriteBytes().get());
this.containerReportReadCount.incr(-1 * deltaStat.getReadCount().get());
this.containerReportWriteCount.incr(-1 * deltaStat.getWriteCount().get());
}
public void unRegister() {
MetricsSystem ms = DefaultMetricsSystem.instance();
ms.unregisterSource(SOURCE_NAME);
}
}
17
View Complete Implementation : SCMNodeMetrics.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
/**
* This clreplaced maintains Node related metrics.
*/
@InterfaceAudience.Private
@Metrics(about = "SCM NodeManager Metrics", context = OzoneConsts.OZONE)
public final clreplaced SCMNodeMetrics implements MetricsSource {
private static final String SOURCE_NAME = SCMNodeMetrics.clreplaced.getSimpleName();
@Metric
private MutableCounterLong numHBProcessed;
@Metric
private MutableCounterLong numHBProcessingFailed;
@Metric
private MutableCounterLong numNodeReportProcessed;
@Metric
private MutableCounterLong numNodeReportProcessingFailed;
private final MetricsRegistry registry;
private final NodeManagerMXBean managerMXBean;
private final MetricsInfo recordInfo = Interns.info("SCMNodeManager", "SCM NodeManager metrics");
/**
* Private constructor.
*/
private SCMNodeMetrics(NodeManagerMXBean managerMXBean) {
this.managerMXBean = managerMXBean;
this.registry = new MetricsRegistry(recordInfo);
}
/**
* Create and returns SCMNodeMetrics instance.
*
* @return SCMNodeMetrics
*/
public static SCMNodeMetrics create(NodeManagerMXBean managerMXBean) {
MetricsSystem ms = DefaultMetricsSystem.instance();
return ms.register(SOURCE_NAME, "SCM NodeManager Metrics", new SCMNodeMetrics(managerMXBean));
}
/**
* Unregister the metrics instance.
*/
public void unRegister() {
MetricsSystem ms = DefaultMetricsSystem.instance();
ms.unregisterSource(SOURCE_NAME);
}
/**
* Increments number of heartbeat processed count.
*/
void incNumHBProcessed() {
numHBProcessed.incr();
}
/**
* Increments number of heartbeat processing failed count.
*/
void incNumHBProcessingFailed() {
numHBProcessingFailed.incr();
}
/**
* Increments number of node report processed count.
*/
void incNumNodeReportProcessed() {
numNodeReportProcessed.incr();
}
/**
* Increments number of node report processing failed count.
*/
void incNumNodeReportProcessingFailed() {
numNodeReportProcessingFailed.incr();
}
/**
* Get aggregated counter and gauage metrics.
*/
@Override
@SuppressWarnings("SuspiciousMethodCalls")
public void getMetrics(MetricsCollector collector, boolean all) {
Map<String, Integer> nodeCount = managerMXBean.getNodeCount();
Map<String, Long> nodeInfo = managerMXBean.getNodeInfo();
registry.snapshot(// Add annotated ones first
collector.addRecord(registry.info()).addGauge(Interns.info("HealthyNodes", "Number of healthy datanodes"), nodeCount.get(HEALTHY.toString())).addGauge(Interns.info("StaleNodes", "Number of stale datanodes"), nodeCount.get(STALE.toString())).addGauge(Interns.info("DeadNodes", "Number of dead datanodes"), nodeCount.get(DEAD.toString())).addGauge(Interns.info("DecommissioningNodes", "Number of decommissioning datanodes"), nodeCount.get(DECOMMISSIONING.toString())).addGauge(Interns.info("DecommissionedNodes", "Number of decommissioned datanodes"), nodeCount.get(DECOMMISSIONED.toString())).addGauge(Interns.info("DiskCapacity", "Total disk capacity"), nodeInfo.get("DISKCapacity")).addGauge(Interns.info("DiskUsed", "Total disk capacity used"), nodeInfo.get("DISKUsed")).addGauge(Interns.info("DiskRemaining", "Total disk capacity remaining"), nodeInfo.get("DISKRemaining")).addGauge(Interns.info("SSDCapacity", "Total ssd capacity"), nodeInfo.get("SSDCapacity")).addGauge(Interns.info("SSDUsed", "Total ssd capacity used"), nodeInfo.get("SSDUsed")).addGauge(Interns.info("SSDRemaining", "Total disk capacity remaining"), nodeInfo.get("SSDRemaining")), all);
}
}
17
View Complete Implementation : OMMetrics.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
/**
* This clreplaced is for maintaining Ozone Manager statistics.
*/
@InterfaceAudience.Private
@Metrics(about = "Ozone Manager Metrics", context = "dfs")
public clreplaced OMMetrics {
private static final String SOURCE_NAME = OMMetrics.clreplaced.getSimpleName();
// OM request type op metrics
@Metric
private MutableCounterLong numVolumeOps;
@Metric
private MutableCounterLong numBucketOps;
@Metric
private MutableCounterLong numKeyOps;
@Metric
private MutableCounterLong numFSOps;
// OM op metrics
@Metric
private MutableCounterLong numVolumeCreates;
@Metric
private MutableCounterLong numVolumeUpdates;
@Metric
private MutableCounterLong numVolumeInfos;
@Metric
private MutableCounterLong numVolumeCheckAccesses;
@Metric
private MutableCounterLong numBucketCreates;
@Metric
private MutableCounterLong numVolumeDeletes;
@Metric
private MutableCounterLong numBucketInfos;
@Metric
private MutableCounterLong numBucketUpdates;
@Metric
private MutableCounterLong numBucketDeletes;
@Metric
private MutableCounterLong numKeyAllocate;
@Metric
private MutableCounterLong numKeyLookup;
@Metric
private MutableCounterLong numKeyRenames;
@Metric
private MutableCounterLong numKeyDeletes;
@Metric
private MutableCounterLong numBucketLists;
@Metric
private MutableCounterLong numKeyLists;
@Metric
private MutableCounterLong numTrashKeyLists;
@Metric
private MutableCounterLong numVolumeLists;
@Metric
private MutableCounterLong numKeyCommits;
@Metric
private MutableCounterLong numBlockAllocations;
@Metric
private MutableCounterLong numGetServiceLists;
@Metric
private MutableCounterLong numBucketS3Lists;
@Metric
private MutableCounterLong numInitiateMultipartUploads;
@Metric
private MutableCounterLong numCompleteMultipartUploads;
@Metric
private MutableCounterLong numGetFileStatus;
@Metric
private MutableCounterLong numCreateDirectory;
@Metric
private MutableCounterLong numCreateFile;
@Metric
private MutableCounterLong numLookupFile;
@Metric
private MutableCounterLong numListStatus;
// Failure Metrics
@Metric
private MutableCounterLong numVolumeCreateFails;
@Metric
private MutableCounterLong numVolumeUpdateFails;
@Metric
private MutableCounterLong numVolumeInfoFails;
@Metric
private MutableCounterLong numVolumeDeleteFails;
@Metric
private MutableCounterLong numBucketCreateFails;
@Metric
private MutableCounterLong numVolumeCheckAccessFails;
@Metric
private MutableCounterLong numBucketInfoFails;
@Metric
private MutableCounterLong numBucketUpdateFails;
@Metric
private MutableCounterLong numBucketDeleteFails;
@Metric
private MutableCounterLong numKeyAllocateFails;
@Metric
private MutableCounterLong numKeyLookupFails;
@Metric
private MutableCounterLong numKeyRenameFails;
@Metric
private MutableCounterLong numKeyDeleteFails;
@Metric
private MutableCounterLong numBucketListFails;
@Metric
private MutableCounterLong numKeyListFails;
@Metric
private MutableCounterLong numTrashKeyListFails;
@Metric
private MutableCounterLong numVolumeListFails;
@Metric
private MutableCounterLong numKeyCommitFails;
@Metric
private MutableCounterLong numBlockAllocationFails;
@Metric
private MutableCounterLong numGetServiceListFails;
@Metric
private MutableCounterLong numBucketS3ListFails;
@Metric
private MutableCounterLong numInitiateMultipartUploadFails;
@Metric
private MutableCounterLong numCommitMultipartUploadParts;
@Metric
private MutableCounterLong numCommitMultipartUploadPartFails;
@Metric
private MutableCounterLong numCompleteMultipartUploadFails;
@Metric
private MutableCounterLong numAbortMultipartUploads;
@Metric
private MutableCounterLong numAbortMultipartUploadFails;
@Metric
private MutableCounterLong numListMultipartUploadParts;
@Metric
private MutableCounterLong numListMultipartUploadPartFails;
@Metric
private MutableCounterLong numGetFileStatusFails;
@Metric
private MutableCounterLong numCreateDirectoryFails;
@Metric
private MutableCounterLong numCreateFileFails;
@Metric
private MutableCounterLong numLookupFileFails;
@Metric
private MutableCounterLong numListStatusFails;
// Metrics for total number of volumes, buckets and keys
@Metric
private MutableCounterLong numVolumes;
@Metric
private MutableCounterLong numBuckets;
@Metric
private MutableCounterLong numS3Buckets;
// TODO: This metric is an estimate and it may be inaccurate on restart if the
// OM process was not shutdown cleanly. Key creations/deletions in the last
// few minutes before restart may not be included in this count.
@Metric
private MutableCounterLong numKeys;
// Metrics to track checkpointing statistics from last run.
@Metric
private MutableGaugeLong lastCheckpointCreationTimeTaken;
@Metric
private MutableGaugeLong lastCheckpointStreamingTimeTaken;
@Metric
private MutableCounterLong numBucketS3Creates;
@Metric
private MutableCounterLong numBucketS3CreateFails;
@Metric
private MutableCounterLong numBucketS3Deletes;
@Metric
private MutableCounterLong numBucketS3DeleteFails;
@Metric
private MutableCounterLong numListMultipartUploadFails;
@Metric
private MutableCounterLong numListMultipartUploads;
public OMMetrics() {
}
public static OMMetrics create() {
MetricsSystem ms = DefaultMetricsSystem.instance();
return ms.register(SOURCE_NAME, "Ozone Manager Metrics", new OMMetrics());
}
public void incNumS3BucketCreates() {
numBucketOps.incr();
numBucketS3Creates.incr();
}
public void incNumS3BucketCreateFails() {
numBucketS3CreateFails.incr();
}
public void incNumS3BucketDeletes() {
numBucketOps.incr();
numBucketS3Deletes.incr();
}
public void incNumS3BucketDeleteFails() {
numBucketOps.incr();
numBucketS3DeleteFails.incr();
}
public void incNumS3Buckets() {
numS3Buckets.incr();
}
public void decNumS3Buckets() {
numS3Buckets.incr();
}
public void incNumVolumes() {
numVolumes.incr();
}
public void decNumVolumes() {
numVolumes.incr(-1);
}
public void incNumBuckets() {
numBuckets.incr();
}
public void decNumBuckets() {
numBuckets.incr(-1);
}
public void incNumKeys() {
numKeys.incr();
}
public void decNumKeys() {
numKeys.incr(-1);
}
public void setNumVolumes(long val) {
long oldVal = this.numVolumes.value();
this.numVolumes.incr(val - oldVal);
}
public void setNumBuckets(long val) {
long oldVal = this.numBuckets.value();
this.numBuckets.incr(val - oldVal);
}
public void setNumKeys(long val) {
long oldVal = this.numKeys.value();
this.numKeys.incr(val - oldVal);
}
public long getNumVolumes() {
return numVolumes.value();
}
public long getNumBuckets() {
return numBuckets.value();
}
public long getNumKeys() {
return numKeys.value();
}
public void incNumVolumeCreates() {
numVolumeOps.incr();
numVolumeCreates.incr();
}
public void incNumVolumeUpdates() {
numVolumeOps.incr();
numVolumeUpdates.incr();
}
public void incNumVolumeInfos() {
numVolumeOps.incr();
numVolumeInfos.incr();
}
public void incNumVolumeDeletes() {
numVolumeOps.incr();
numVolumeDeletes.incr();
}
public void incNumVolumeCheckAccesses() {
numVolumeOps.incr();
numVolumeCheckAccesses.incr();
}
public void incNumBucketCreates() {
numBucketOps.incr();
numBucketCreates.incr();
}
public void incNumBucketInfos() {
numBucketOps.incr();
numBucketInfos.incr();
}
public void incNumBucketUpdates() {
numBucketOps.incr();
numBucketUpdates.incr();
}
public void incNumBucketDeletes() {
numBucketOps.incr();
numBucketDeletes.incr();
}
public void incNumBucketLists() {
numBucketOps.incr();
numBucketLists.incr();
}
public void incNumKeyLists() {
numKeyOps.incr();
numKeyLists.incr();
}
public void incNumTrashKeyLists() {
numKeyOps.incr();
numTrashKeyLists.incr();
}
public void incNumVolumeLists() {
numVolumeOps.incr();
numVolumeLists.incr();
}
public void incNumListS3Buckets() {
numBucketOps.incr();
numBucketS3Lists.incr();
}
public void incNumListS3BucketsFails() {
numBucketOps.incr();
numBucketS3ListFails.incr();
}
public void incNumInitiateMultipartUploads() {
numKeyOps.incr();
numInitiateMultipartUploads.incr();
}
public void incNumInitiateMultipartUploadFails() {
numInitiateMultipartUploadFails.incr();
}
public void incNumCommitMultipartUploadParts() {
numKeyOps.incr();
numCommitMultipartUploadParts.incr();
}
public void incNumCommitMultipartUploadPartFails() {
numCommitMultipartUploadPartFails.incr();
}
public void incNumCompleteMultipartUploads() {
numKeyOps.incr();
numCompleteMultipartUploads.incr();
}
public void incNumCompleteMultipartUploadFails() {
numCompleteMultipartUploadFails.incr();
}
public void incNumAbortMultipartUploads() {
numKeyOps.incr();
numAbortMultipartUploads.incr();
}
public void incNumListMultipartUploadFails() {
numListMultipartUploadFails.incr();
}
public void incNumListMultipartUploads() {
numKeyOps.incr();
numListMultipartUploads.incr();
}
public void incNumAbortMultipartUploadFails() {
numAbortMultipartUploadFails.incr();
}
public void incNumListMultipartUploadParts() {
numKeyOps.incr();
numListMultipartUploadParts.incr();
}
public void incNumGetFileStatus() {
numKeyOps.incr();
numFSOps.incr();
numGetFileStatus.incr();
}
public void incNumGetFileStatusFails() {
numGetFileStatusFails.incr();
}
public void incNumCreateDirectory() {
numKeyOps.incr();
numFSOps.incr();
numCreateDirectory.incr();
}
public void incNumCreateDirectoryFails() {
numCreateDirectoryFails.incr();
}
public void incNumCreateFile() {
numKeyOps.incr();
numFSOps.incr();
numCreateFile.incr();
}
public void incNumCreateFileFails() {
numCreateFileFails.incr();
}
public void incNumLookupFile() {
numKeyOps.incr();
numFSOps.incr();
numLookupFile.incr();
}
public void incNumLookupFileFails() {
numLookupFileFails.incr();
}
public void incNumListStatus() {
numKeyOps.incr();
numFSOps.incr();
numListStatus.incr();
}
public void incNumListStatusFails() {
numListStatusFails.incr();
}
public void incNumListMultipartUploadPartFails() {
numListMultipartUploadPartFails.incr();
}
public void incNumGetServiceLists() {
numGetServiceLists.incr();
}
public void incNumVolumeCreateFails() {
numVolumeCreateFails.incr();
}
public void incNumVolumeUpdateFails() {
numVolumeUpdateFails.incr();
}
public void incNumVolumeInfoFails() {
numVolumeInfoFails.incr();
}
public void incNumVolumeDeleteFails() {
numVolumeDeleteFails.incr();
}
public void incNumVolumeCheckAccessFails() {
numVolumeCheckAccessFails.incr();
}
public void incNumBucketCreateFails() {
numBucketCreateFails.incr();
}
public void incNumBucketInfoFails() {
numBucketInfoFails.incr();
}
public void incNumBucketUpdateFails() {
numBucketUpdateFails.incr();
}
public void incNumBucketDeleteFails() {
numBucketDeleteFails.incr();
}
public void incNumKeyAllocates() {
numKeyOps.incr();
numKeyAllocate.incr();
}
public void incNumKeyAllocateFails() {
numKeyAllocateFails.incr();
}
public void incNumKeyLookups() {
numKeyOps.incr();
numKeyLookup.incr();
}
public void incNumKeyLookupFails() {
numKeyLookupFails.incr();
}
public void incNumKeyRenames() {
numKeyOps.incr();
numKeyRenames.incr();
}
public void incNumKeyRenameFails() {
numKeyOps.incr();
numKeyRenameFails.incr();
}
public void incNumKeyDeleteFails() {
numKeyDeleteFails.incr();
}
public void incNumKeyDeletes() {
numKeyOps.incr();
numKeyDeletes.incr();
}
public void incNumKeyCommits() {
numKeyOps.incr();
numKeyCommits.incr();
}
public void incNumKeyCommitFails() {
numKeyCommitFails.incr();
}
public void incNumBlockAllocateCalls() {
numBlockAllocations.incr();
}
public void incNumBlockAllocateCallFails() {
numBlockAllocationFails.incr();
}
public void incNumBucketListFails() {
numBucketListFails.incr();
}
public void incNumKeyListFails() {
numKeyListFails.incr();
}
public void incNumTrashKeyListFails() {
numTrashKeyListFails.incr();
}
public void incNumVolumeListFails() {
numVolumeListFails.incr();
}
public void incNumGetServiceListFails() {
numGetServiceListFails.incr();
}
public void setLastCheckpointCreationTimeTaken(long val) {
this.lastCheckpointCreationTimeTaken.set(val);
}
public void setLastCheckpointStreamingTimeTaken(long val) {
this.lastCheckpointStreamingTimeTaken.set(val);
}
@VisibleForTesting
public long getNumVolumeCreates() {
return numVolumeCreates.value();
}
@VisibleForTesting
public long getNumVolumeUpdates() {
return numVolumeUpdates.value();
}
@VisibleForTesting
public long getNumVolumeInfos() {
return numVolumeInfos.value();
}
@VisibleForTesting
public long getNumVolumeDeletes() {
return numVolumeDeletes.value();
}
@VisibleForTesting
public long getNumVolumeCheckAccesses() {
return numVolumeCheckAccesses.value();
}
@VisibleForTesting
public long getNumBucketCreates() {
return numBucketCreates.value();
}
@VisibleForTesting
public long getNumBucketInfos() {
return numBucketInfos.value();
}
@VisibleForTesting
public long getNumBucketUpdates() {
return numBucketUpdates.value();
}
@VisibleForTesting
public long getNumBucketDeletes() {
return numBucketDeletes.value();
}
@VisibleForTesting
public long getNumBucketLists() {
return numBucketLists.value();
}
@VisibleForTesting
public long getNumVolumeLists() {
return numVolumeLists.value();
}
@VisibleForTesting
public long getNumKeyLists() {
return numKeyLists.value();
}
@VisibleForTesting
public long getNumTrashKeyLists() {
return numTrashKeyLists.value();
}
@VisibleForTesting
public long getNumGetServiceLists() {
return numGetServiceLists.value();
}
@VisibleForTesting
public long getNumVolumeCreateFails() {
return numVolumeCreateFails.value();
}
@VisibleForTesting
public long getNumVolumeUpdateFails() {
return numVolumeUpdateFails.value();
}
@VisibleForTesting
public long getNumVolumeInfoFails() {
return numVolumeInfoFails.value();
}
@VisibleForTesting
public long getNumVolumeDeleteFails() {
return numVolumeDeleteFails.value();
}
@VisibleForTesting
public long getNumVolumeCheckAccessFails() {
return numVolumeCheckAccessFails.value();
}
@VisibleForTesting
public long getNumBucketCreateFails() {
return numBucketCreateFails.value();
}
@VisibleForTesting
public long getNumBucketInfoFails() {
return numBucketInfoFails.value();
}
@VisibleForTesting
public long getNumBucketUpdateFails() {
return numBucketUpdateFails.value();
}
@VisibleForTesting
public long getNumBucketDeleteFails() {
return numBucketDeleteFails.value();
}
@VisibleForTesting
public long getNumKeyAllocates() {
return numKeyAllocate.value();
}
@VisibleForTesting
public long getNumKeyAllocateFails() {
return numKeyAllocateFails.value();
}
@VisibleForTesting
public long getNumKeyLookups() {
return numKeyLookup.value();
}
@VisibleForTesting
public long getNumKeyLookupFails() {
return numKeyLookupFails.value();
}
@VisibleForTesting
public long getNumKeyRenames() {
return numKeyRenames.value();
}
@VisibleForTesting
public long getNumKeyRenameFails() {
return numKeyRenameFails.value();
}
@VisibleForTesting
public long getNumKeyDeletes() {
return numKeyDeletes.value();
}
@VisibleForTesting
public long getNumKeyDeletesFails() {
return numKeyDeleteFails.value();
}
@VisibleForTesting
public long getNumBucketListFails() {
return numBucketListFails.value();
}
@VisibleForTesting
public long getNumKeyListFails() {
return numKeyListFails.value();
}
@VisibleForTesting
public long getNumTrashKeyListFails() {
return numTrashKeyListFails.value();
}
@VisibleForTesting
public long getNumFSOps() {
return numFSOps.value();
}
@VisibleForTesting
public long getNumGetFileStatus() {
return numGetFileStatus.value();
}
@VisibleForTesting
public long getNumListStatus() {
return numListStatus.value();
}
@VisibleForTesting
public long getNumVolumeListFails() {
return numVolumeListFails.value();
}
@VisibleForTesting
public long getNumKeyCommits() {
return numKeyCommits.value();
}
@VisibleForTesting
public long getNumKeyCommitFails() {
return numKeyCommitFails.value();
}
@VisibleForTesting
public long getNumBlockAllocates() {
return numBlockAllocations.value();
}
@VisibleForTesting
public long getNumBlockAllocateFails() {
return numBlockAllocationFails.value();
}
@VisibleForTesting
public long getNumGetServiceListFails() {
return numGetServiceListFails.value();
}
@VisibleForTesting
public long getNumListS3Buckets() {
return numBucketS3Lists.value();
}
@VisibleForTesting
public long getNumListS3BucketsFails() {
return numBucketS3ListFails.value();
}
public long getNumInitiateMultipartUploads() {
return numInitiateMultipartUploads.value();
}
public long getNumInitiateMultipartUploadFails() {
return numInitiateMultipartUploadFails.value();
}
public long getNumAbortMultipartUploads() {
return numAbortMultipartUploads.value();
}
public long getNumAbortMultipartUploadFails() {
return numAbortMultipartUploadFails.value();
}
@VisibleForTesting
public long getLastCheckpointCreationTimeTaken() {
return lastCheckpointCreationTimeTaken.value();
}
@VisibleForTesting
public long getLastCheckpointStreamingTimeTaken() {
return lastCheckpointStreamingTimeTaken.value();
}
public void unRegister() {
MetricsSystem ms = DefaultMetricsSystem.instance();
ms.unregisterSource(SOURCE_NAME);
}
}
17
View Complete Implementation : OzoneManagerDoubleBufferMetrics.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
/**
* Clreplaced which maintains metrics related to OzoneManager DoubleBuffer.
*/
public clreplaced OzoneManagerDoubleBufferMetrics {
private static final String SOURCE_NAME = OzoneManagerDoubleBufferMetrics.clreplaced.getSimpleName();
@Metric(about = "Total Number of flush operations happened in " + "OzoneManagerDoubleBuffer.")
private MutableCounterLong totalNumOfFlushOperations;
@Metric(about = "Total Number of flushed transactions happened in " + "OzoneManagerDoubleBuffer.")
private MutableCounterLong totalNumOfFlushedTransactions;
@Metric(about = "Max Number of transactions flushed in a iteration in " + "OzoneManagerDoubleBuffer. This will provide a value which is maximum " + "number of transactions flushed in a single flush iteration till now.")
private MutableCounterLong maxNumberOfTransactionsFlushedInOneIteration;
public static OzoneManagerDoubleBufferMetrics create() {
MetricsSystem ms = DefaultMetricsSystem.instance();
return ms.register(SOURCE_NAME, "OzoneManager DoubleBuffer Metrics", new OzoneManagerDoubleBufferMetrics());
}
public void incrTotalNumOfFlushOperations() {
this.totalNumOfFlushOperations.incr();
}
public void incrTotalSizeOfFlushedTransactions(long flushedTransactions) {
this.totalNumOfFlushedTransactions.incr(flushedTransactions);
}
public void setMaxNumberOfTransactionsFlushedInOneIteration(long maxTransactions) {
// We should set the value with maxTransactions, so decrement old value
// first and then add the new value.
this.maxNumberOfTransactionsFlushedInOneIteration.incr(Math.negateExact(getMaxNumberOfTransactionsFlushedInOneIteration()) + maxTransactions);
}
public long getTotalNumOfFlushOperations() {
return totalNumOfFlushOperations.value();
}
public long getTotalNumOfFlushedTransactions() {
return totalNumOfFlushedTransactions.value();
}
public long getMaxNumberOfTransactionsFlushedInOneIteration() {
return maxNumberOfTransactionsFlushedInOneIteration.value();
}
public void unRegister() {
MetricsSystem ms = DefaultMetricsSystem.instance();
ms.unregisterSource(SOURCE_NAME);
}
}
17
View Complete Implementation : QueryMetrics.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
/**
* properties and methods about query.
*/
@ThreadSafe
@Metrics(name = "Query", about = "Query metrics", context = "Kylin")
public clreplaced QueryMetrics {
final MetricsRegistry registry = new MetricsRegistry("Query");
@Metric
MutableCounterLong querySuccessCount;
@Metric
MutableCounterLong queryFailCount;
@Metric
MutableCounterLong queryCount;
@Metric
MutableCounterLong cacheHitCount;
MutableQuantiles[] cacheHitCountQuantiles;
@Metric
MutableRate queryLatency;
MutableQuantiles[] queryLatencyTimeMillisQuantiles;
@Metric
MutableRate scanRowCount;
MutableQuantiles[] scanRowCountQuantiles;
@Metric
MutableRate resultRowCount;
MutableQuantiles[] resultRowCountQuantiles;
public QueryMetrics(int[] intervals) {
queryLatencyTimeMillisQuantiles = new MutableQuantiles[intervals.length];
scanRowCountQuantiles = new MutableQuantiles[intervals.length];
resultRowCountQuantiles = new MutableQuantiles[intervals.length];
cacheHitCountQuantiles = new MutableQuantiles[intervals.length];
for (int i = 0; i < intervals.length; i++) {
int interval = intervals[i];
queryLatencyTimeMillisQuantiles[i] = registry.newQuantiles("QueryLatency" + interval + "s", "Query queue time in milli second", "ops", "", interval);
scanRowCountQuantiles[i] = registry.newQuantiles("ScanRowCount" + interval + "s", "Scan row count in milli second", "ops", "", interval);
resultRowCountQuantiles[i] = registry.newQuantiles("ResultRowCount" + interval + "s", "Result row count in milli second", "ops", "", interval);
cacheHitCountQuantiles[i] = registry.newQuantiles("CacheHitCount" + interval + "s", "Cache Hit Count in milli second", "ops", "", interval);
}
queryLatency = registry.newRate("QueryLatency", "", true);
scanRowCount = registry.newRate("ScanRowCount", "", true);
resultRowCount = registry.newRate("ResultRowCount", "", true);
}
public void shutdown() {
DefaultMetricsSystem.shutdown();
}
public void incrQuerySuccessCount() {
querySuccessCount.incr();
}
public void incrQueryFailCount() {
queryFailCount.incr();
}
public void incrQueryCount() {
queryCount.incr();
}
public void addQueryLatency(long latency) {
queryLatency.add(latency);
for (MutableQuantiles m : queryLatencyTimeMillisQuantiles) {
m.add(latency);
}
}
public void addScanRowCount(long count) {
scanRowCount.add(count);
for (MutableQuantiles m : scanRowCountQuantiles) {
m.add(count);
}
}
public void addResultRowCount(long count) {
resultRowCount.add(count);
for (MutableQuantiles m : resultRowCountQuantiles) {
m.add(count);
}
}
public void addCacheHitCount(long count) {
cacheHitCount.incr(count);
for (MutableQuantiles m : cacheHitCountQuantiles) {
m.add(count);
}
}
public QueryMetrics registerWith(String name) {
return DefaultMetricsSystem.instance().register(name, "Query", this);
}
}
17
View Complete Implementation : MetricsBalancerSourceImpl.java
Copyright Apache License 2.0
Author : fengchen8086
Copyright Apache License 2.0
Author : fengchen8086
@InterfaceAudience.Private
public clreplaced MetricsBalancerSourceImpl extends BaseSourceImpl implements MetricsBalancerSource {
private MutableHistogram blanceClusterHisto;
private MutableCounterLong miscCount;
public MetricsBalancerSourceImpl() {
this(METRICS_NAME, METRICS_DESCRIPTION, METRICS_CONTEXT, METRICS_JMX_CONTEXT);
}
public MetricsBalancerSourceImpl(String metricsName, String metricsDescription, String metricsContext, String metricsJmxContext) {
super(metricsName, metricsDescription, metricsContext, metricsJmxContext);
}
@Override
public void init() {
blanceClusterHisto = metricsRegistry.newTimeHistogram(BALANCE_CLUSTER);
miscCount = metricsRegistry.newCounter(MISC_INVOATION_COUNT, "", 0L);
}
@Override
public void updateBalanceCluster(long time) {
blanceClusterHisto.add(time);
}
@Override
public void incrMiscInvocations() {
miscCount.incr();
}
}
17
View Complete Implementation : MetricsRegionServerSourceImpl.java
Copyright Apache License 2.0
Author : fengchen8086
Copyright Apache License 2.0
Author : fengchen8086
/**
* Hadoop2 implementation of MetricsRegionServerSource.
*
* Implements BaseSource through BaseSourceImpl, following the pattern
*/
@InterfaceAudience.Private
public clreplaced MetricsRegionServerSourceImpl extends BaseSourceImpl implements MetricsRegionServerSource {
final MetricsRegionServerWrapper rsWrap;
private final MetricHistogram putHisto;
private final MetricHistogram deleteHisto;
private final MetricHistogram getHisto;
private final MetricHistogram incrementHisto;
private final MetricHistogram appendHisto;
private final MetricHistogram replayHisto;
private final MetricHistogram scanNextHisto;
private final MutableCounterLong slowPut;
private final MutableCounterLong slowDelete;
private final MutableCounterLong slowGet;
private final MutableCounterLong slowIncrement;
private final MutableCounterLong slowAppend;
private final MutableCounterLong splitRequest;
private final MutableCounterLong splitSuccess;
private final MetricHistogram splitTimeHisto;
private final MetricHistogram flushTimeHisto;
public MetricsRegionServerSourceImpl(MetricsRegionServerWrapper rsWrap) {
this(METRICS_NAME, METRICS_DESCRIPTION, METRICS_CONTEXT, METRICS_JMX_CONTEXT, rsWrap);
}
public MetricsRegionServerSourceImpl(String metricsName, String metricsDescription, String metricsContext, String metricsJmxContext, MetricsRegionServerWrapper rsWrap) {
super(metricsName, metricsDescription, metricsContext, metricsJmxContext);
this.rsWrap = rsWrap;
putHisto = getMetricsRegistry().newTimeHistogram(MUTATE_KEY);
slowPut = getMetricsRegistry().newCounter(SLOW_MUTATE_KEY, SLOW_MUTATE_DESC, 0L);
deleteHisto = getMetricsRegistry().newTimeHistogram(DELETE_KEY);
slowDelete = getMetricsRegistry().newCounter(SLOW_DELETE_KEY, SLOW_DELETE_DESC, 0L);
getHisto = getMetricsRegistry().newTimeHistogram(GET_KEY);
slowGet = getMetricsRegistry().newCounter(SLOW_GET_KEY, SLOW_GET_DESC, 0L);
incrementHisto = getMetricsRegistry().newTimeHistogram(INCREMENT_KEY);
slowIncrement = getMetricsRegistry().newCounter(SLOW_INCREMENT_KEY, SLOW_INCREMENT_DESC, 0L);
appendHisto = getMetricsRegistry().newTimeHistogram(APPEND_KEY);
slowAppend = getMetricsRegistry().newCounter(SLOW_APPEND_KEY, SLOW_APPEND_DESC, 0L);
replayHisto = getMetricsRegistry().newTimeHistogram(REPLAY_KEY);
scanNextHisto = getMetricsRegistry().newTimeHistogram(SCAN_NEXT_KEY);
splitTimeHisto = getMetricsRegistry().newTimeHistogram(SPLIT_KEY);
flushTimeHisto = getMetricsRegistry().newTimeHistogram(FLUSH_KEY);
splitRequest = getMetricsRegistry().newCounter(SPLIT_REQUEST_KEY, SPLIT_REQUEST_DESC, 0L);
splitSuccess = getMetricsRegistry().newCounter(SPLIT_SUCCESS_KEY, SPLIT_SUCCESS_DESC, 0L);
}
@Override
public void updatePut(long t) {
putHisto.add(t);
}
@Override
public void updateDelete(long t) {
deleteHisto.add(t);
}
@Override
public void updateGet(long t) {
getHisto.add(t);
}
@Override
public void updateIncrement(long t) {
incrementHisto.add(t);
}
@Override
public void updateAppend(long t) {
appendHisto.add(t);
}
@Override
public void updateReplay(long t) {
replayHisto.add(t);
}
@Override
public void updateScannerNext(long scanSize) {
scanNextHisto.add(scanSize);
}
@Override
public void incrSlowPut() {
slowPut.incr();
}
@Override
public void incrSlowDelete() {
slowDelete.incr();
}
@Override
public void incrSlowGet() {
slowGet.incr();
}
@Override
public void incrSlowIncrement() {
slowIncrement.incr();
}
@Override
public void incrSlowAppend() {
slowAppend.incr();
}
@Override
public void incrSplitRequest() {
splitRequest.incr();
}
@Override
public void incrSplitSuccess() {
splitSuccess.incr();
}
@Override
public void updateSplitTime(long t) {
splitTimeHisto.add(t);
}
@Override
public void updateFlushTime(long t) {
flushTimeHisto.add(t);
}
/**
* Yes this is a get function that doesn't return anything. Thanks Hadoop for breaking all
* expectations of java programmers. Instead of returning anything Hadoop metrics expects
* getMetrics to push the metrics into the collector.
*
* @param metricsCollector Collector to accept metrics
* @param all push all or only changed?
*/
@Override
public void getMetrics(MetricsCollector metricsCollector, boolean all) {
MetricsRecordBuilder mrb = metricsCollector.addRecord(metricsName);
// rsWrap can be null because this function is called inside of init.
if (rsWrap != null) {
mrb.addGauge(Interns.info(REGION_COUNT, REGION_COUNT_DESC), rsWrap.getNumOnlineRegions()).addGauge(Interns.info(STORE_COUNT, STORE_COUNT_DESC), rsWrap.getNumStores()).addGauge(Interns.info(WALFILE_COUNT, WALFILE_COUNT_DESC), rsWrap.getNumWALFiles()).addGauge(Interns.info(WALFILE_SIZE, WALFILE_SIZE_DESC), rsWrap.getWALFileSize()).addGauge(Interns.info(STOREFILE_COUNT, STOREFILE_COUNT_DESC), rsWrap.getNumStoreFiles()).addGauge(Interns.info(MEMSTORE_SIZE, MEMSTORE_SIZE_DESC), rsWrap.getMemstoreSize()).addGauge(Interns.info(STOREFILE_SIZE, STOREFILE_SIZE_DESC), rsWrap.getStoreFileSize()).addGauge(Interns.info(RS_START_TIME_NAME, RS_START_TIME_DESC), rsWrap.getStartCode()).addCounter(Interns.info(TOTAL_REQUEST_COUNT, TOTAL_REQUEST_COUNT_DESC), rsWrap.getTotalRequestCount()).addCounter(Interns.info(READ_REQUEST_COUNT, READ_REQUEST_COUNT_DESC), rsWrap.getReadRequestsCount()).addCounter(Interns.info(WRITE_REQUEST_COUNT, WRITE_REQUEST_COUNT_DESC), rsWrap.getWriteRequestsCount()).addCounter(Interns.info(CHECK_MUTATE_FAILED_COUNT, CHECK_MUTATE_FAILED_COUNT_DESC), rsWrap.getCheckAndMutateChecksFailed()).addCounter(Interns.info(CHECK_MUTATE_PreplacedED_COUNT, CHECK_MUTATE_PreplacedED_COUNT_DESC), rsWrap.getCheckAndMutateChecksPreplaceded()).addGauge(Interns.info(STOREFILE_INDEX_SIZE, STOREFILE_INDEX_SIZE_DESC), rsWrap.getStoreFileIndexSize()).addGauge(Interns.info(STATIC_INDEX_SIZE, STATIC_INDEX_SIZE_DESC), rsWrap.getTotalStaticIndexSize()).addGauge(Interns.info(STATIC_BLOOM_SIZE, STATIC_BLOOM_SIZE_DESC), rsWrap.getTotalStaticBloomSize()).addGauge(Interns.info(NUMBER_OF_MUTATIONS_WITHOUT_WAL, NUMBER_OF_MUTATIONS_WITHOUT_WAL_DESC), rsWrap.getNumMutationsWithoutWAL()).addGauge(Interns.info(DATA_SIZE_WITHOUT_WAL, DATA_SIZE_WITHOUT_WAL_DESC), rsWrap.getDataInMemoryWithoutWAL()).addGauge(Interns.info(PERCENT_FILES_LOCAL, PERCENT_FILES_LOCAL_DESC), rsWrap.getPercentFileLocal()).addGauge(Interns.info(PERCENT_FILES_LOCAL_SECONDARY_REGIONS, PERCENT_FILES_LOCAL_SECONDARY_REGIONS_DESC), rsWrap.getPercentFileLocalSecondaryRegions()).addGauge(Interns.info(SPLIT_QUEUE_LENGTH, SPLIT_QUEUE_LENGTH_DESC), rsWrap.getSplitQueueSize()).addGauge(Interns.info(COMPACTION_QUEUE_LENGTH, COMPACTION_QUEUE_LENGTH_DESC), rsWrap.getCompactionQueueSize()).addGauge(Interns.info(FLUSH_QUEUE_LENGTH, FLUSH_QUEUE_LENGTH_DESC), rsWrap.getFlushQueueSize()).addGauge(Interns.info(BLOCK_CACHE_FREE_SIZE, BLOCK_CACHE_FREE_DESC), rsWrap.getBlockCacheFreeSize()).addGauge(Interns.info(BLOCK_CACHE_COUNT, BLOCK_CACHE_COUNT_DESC), rsWrap.getBlockCacheCount()).addGauge(Interns.info(BLOCK_CACHE_SIZE, BLOCK_CACHE_SIZE_DESC), rsWrap.getBlockCacheSize()).addCounter(Interns.info(BLOCK_CACHE_HIT_COUNT, BLOCK_CACHE_HIT_COUNT_DESC), rsWrap.getBlockCacheHitCount()).addCounter(Interns.info(BLOCK_CACHE_PRIMARY_HIT_COUNT, BLOCK_CACHE_PRIMARY_HIT_COUNT_DESC), rsWrap.getBlockCachePrimaryHitCount()).addCounter(Interns.info(BLOCK_CACHE_MISS_COUNT, BLOCK_COUNT_MISS_COUNT_DESC), rsWrap.getBlockCacheMissCount()).addCounter(Interns.info(BLOCK_CACHE_PRIMARY_MISS_COUNT, BLOCK_COUNT_PRIMARY_MISS_COUNT_DESC), rsWrap.getBlockCachePrimaryMissCount()).addCounter(Interns.info(BLOCK_CACHE_EVICTION_COUNT, BLOCK_CACHE_EVICTION_COUNT_DESC), rsWrap.getBlockCacheEvictedCount()).addCounter(Interns.info(BLOCK_CACHE_PRIMARY_EVICTION_COUNT, BLOCK_CACHE_PRIMARY_EVICTION_COUNT_DESC), rsWrap.getBlockCachePrimaryEvictedCount()).addGauge(Interns.info(BLOCK_CACHE_HIT_PERCENT, BLOCK_CACHE_HIT_PERCENT_DESC), rsWrap.getBlockCacheHitPercent()).addGauge(Interns.info(BLOCK_CACHE_EXPRESS_HIT_PERCENT, BLOCK_CACHE_EXPRESS_HIT_PERCENT_DESC), rsWrap.getBlockCacheHitCachingPercent()).addCounter(Interns.info(BLOCK_CACHE_FAILED_INSERTION_COUNT, BLOCK_CACHE_FAILED_INSERTION_COUNT_DESC), rsWrap.getBlockCacheFailedInsertions()).addCounter(Interns.info(UPDATES_BLOCKED_TIME, UPDATES_BLOCKED_DESC), rsWrap.getUpdatesBlockedTime()).addCounter(Interns.info(FLUSHED_CELLS, FLUSHED_CELLS_DESC), rsWrap.getFlushedCellsCount()).addCounter(Interns.info(COMPACTED_CELLS, COMPACTED_CELLS_DESC), rsWrap.getCompactedCellsCount()).addCounter(Interns.info(MAJOR_COMPACTED_CELLS, MAJOR_COMPACTED_CELLS_DESC), rsWrap.getMajorCompactedCellsCount()).addCounter(Interns.info(FLUSHED_CELLS_SIZE, FLUSHED_CELLS_SIZE_DESC), rsWrap.getFlushedCellsSize()).addCounter(Interns.info(COMPACTED_CELLS_SIZE, COMPACTED_CELLS_SIZE_DESC), rsWrap.getCompactedCellsSize()).addCounter(Interns.info(MAJOR_COMPACTED_CELLS_SIZE, MAJOR_COMPACTED_CELLS_SIZE_DESC), rsWrap.getMajorCompactedCellsSize()).addCounter(Interns.info(BLOCKED_REQUESTS_COUNT, BLOCKED_REQUESTS_COUNT_DESC), rsWrap.getBlockedRequestsCount()).tag(Interns.info(ZOOKEEPER_QUORUM_NAME, ZOOKEEPER_QUORUM_DESC), rsWrap.getZookeeperQuorum()).tag(Interns.info(SERVER_NAME_NAME, SERVER_NAME_DESC), rsWrap.getServerName()).tag(Interns.info(CLUSTER_ID_NAME, CLUSTER_ID_DESC), rsWrap.getClusterId());
}
metricsRegistry.snapshot(mrb, all);
}
}
17
View Complete Implementation : MetricsWALSourceImpl.java
Copyright Apache License 2.0
Author : fengchen8086
Copyright Apache License 2.0
Author : fengchen8086
/**
* Clreplaced that transitions metrics from MetricsWAL into the metrics subsystem.
*
* Implements BaseSource through BaseSourceImpl, following the pattern.
* @see org.apache.hadoop.hbase.regionserver.wal.MetricsWAL
*/
@InterfaceAudience.Private
public clreplaced MetricsWALSourceImpl extends BaseSourceImpl implements MetricsWALSource {
private final MetricHistogram appendSizeHisto;
private final MetricHistogram appendTimeHisto;
private final MetricHistogram syncTimeHisto;
private final MutableCounterLong appendCount;
private final MutableCounterLong slowAppendCount;
private final MutableCounterLong logRollRequested;
private final MutableCounterLong lowReplicationLogRollRequested;
public MetricsWALSourceImpl() {
this(METRICS_NAME, METRICS_DESCRIPTION, METRICS_CONTEXT, METRICS_JMX_CONTEXT);
}
public MetricsWALSourceImpl(String metricsName, String metricsDescription, String metricsContext, String metricsJmxContext) {
super(metricsName, metricsDescription, metricsContext, metricsJmxContext);
// Create and store the metrics that will be used.
appendTimeHisto = this.getMetricsRegistry().newTimeHistogram(APPEND_TIME, APPEND_TIME_DESC);
appendSizeHisto = this.getMetricsRegistry().newSizeHistogram(APPEND_SIZE, APPEND_SIZE_DESC);
appendCount = this.getMetricsRegistry().newCounter(APPEND_COUNT, APPEND_COUNT_DESC, 0l);
slowAppendCount = this.getMetricsRegistry().newCounter(SLOW_APPEND_COUNT, SLOW_APPEND_COUNT_DESC, 0l);
syncTimeHisto = this.getMetricsRegistry().newTimeHistogram(SYNC_TIME, SYNC_TIME_DESC);
logRollRequested = this.getMetricsRegistry().newCounter(ROLL_REQUESTED, ROLL_REQUESTED_DESC, 0L);
lowReplicationLogRollRequested = this.getMetricsRegistry().newCounter(LOW_REPLICA_ROLL_REQUESTED, LOW_REPLICA_ROLL_REQUESTED_DESC, 0L);
}
@Override
public void incrementAppendSize(long size) {
appendSizeHisto.add(size);
}
@Override
public void incrementAppendTime(long time) {
appendTimeHisto.add(time);
}
@Override
public void incrementAppendCount() {
appendCount.incr();
}
@Override
public void incrementSlowAppendCount() {
slowAppendCount.incr();
}
@Override
public void incrementSyncTime(long time) {
syncTimeHisto.add(time);
}
@Override
public void incrementLogRollRequested() {
logRollRequested.incr();
}
@Override
public void incrementLowReplicationLogRoll() {
lowReplicationLogRollRequested.incr();
}
}
17
View Complete Implementation : RpcMetrics.java
Copyright Apache License 2.0
Author : yncxcw
Copyright Apache License 2.0
Author : yncxcw
/**
* This clreplaced is for maintaining the various RPC statistics
* and publishing them through the metrics interfaces.
*/
@InterfaceAudience.Private
@Metrics(about = "Aggregate RPC metrics", context = "rpc")
public clreplaced RpcMetrics {
static final Log LOG = LogFactory.getLog(RpcMetrics.clreplaced);
final Server server;
final MetricsRegistry registry;
final String name;
final boolean rpcQuantileEnable;
RpcMetrics(Server server, Configuration conf) {
String port = String.valueOf(server.getListenerAddress().getPort());
name = "RpcActivityForPort" + port;
this.server = server;
registry = new MetricsRegistry("rpc").tag("port", "RPC port", port);
int[] intervals = conf.getInts(CommonConfigurationKeys.RPC_METRICS_PERCENTILES_INTERVALS_KEY);
rpcQuantileEnable = (intervals.length > 0) && conf.getBoolean(CommonConfigurationKeys.RPC_METRICS_QUANTILE_ENABLE, CommonConfigurationKeys.RPC_METRICS_QUANTILE_ENABLE_DEFAULT);
if (rpcQuantileEnable) {
rpcQueueTimeMillisQuantiles = new MutableQuantiles[intervals.length];
rpcProcessingTimeMillisQuantiles = new MutableQuantiles[intervals.length];
for (int i = 0; i < intervals.length; i++) {
int interval = intervals[i];
rpcQueueTimeMillisQuantiles[i] = registry.newQuantiles("rpcQueueTime" + interval + "s", "rpc queue time in milli second", "ops", "latency", interval);
rpcProcessingTimeMillisQuantiles[i] = registry.newQuantiles("rpcProcessingTime" + interval + "s", "rpc processing time in milli second", "ops", "latency", interval);
}
}
LOG.debug("Initialized " + registry);
}
public String name() {
return name;
}
public static RpcMetrics create(Server server, Configuration conf) {
RpcMetrics m = new RpcMetrics(server, conf);
return DefaultMetricsSystem.instance().register(m.name, null, m);
}
@Metric("Number of received bytes")
MutableCounterLong receivedBytes;
@Metric("Number of sent bytes")
MutableCounterLong sentBytes;
@Metric("Queue time")
MutableRate rpcQueueTime;
MutableQuantiles[] rpcQueueTimeMillisQuantiles;
@Metric("Processsing time")
MutableRate rpcProcessingTime;
MutableQuantiles[] rpcProcessingTimeMillisQuantiles;
@Metric("Number of authentication failures")
MutableCounterLong rpcAuthenticationFailures;
@Metric("Number of authentication successes")
MutableCounterLong rpcAuthenticationSuccesses;
@Metric("Number of authorization failures")
MutableCounterLong rpcAuthorizationFailures;
@Metric("Number of authorization sucesses")
MutableCounterLong rpcAuthorizationSuccesses;
@Metric("Number of open connections")
public int numOpenConnections() {
return server.getNumOpenConnections();
}
@Metric("Length of the call queue")
public int callQueueLength() {
return server.getCallQueueLen();
}
// Public instrumentation methods that could be extracted to an
// abstract clreplaced if we decide to do custom instrumentation clreplacedes a la
// JobTrackerInstrumenation. The methods with //@Override comment are
// candidates for abstract methods in a abstract instrumentation clreplaced.
/**
* One authentication failure event
*/
// @Override
public void incrAuthenticationFailures() {
rpcAuthenticationFailures.incr();
}
/**
* One authentication success event
*/
// @Override
public void incrAuthenticationSuccesses() {
rpcAuthenticationSuccesses.incr();
}
/**
* One authorization success event
*/
// @Override
public void incrAuthorizationSuccesses() {
rpcAuthorizationSuccesses.incr();
}
/**
* One authorization failure event
*/
// @Override
public void incrAuthorizationFailures() {
rpcAuthorizationFailures.incr();
}
/**
* Shutdown the instrumentation for the process
*/
// @Override
public void shutdown() {
}
/**
* Increment sent bytes by count
* @param count to increment
*/
// @Override
public void incrSentBytes(int count) {
sentBytes.incr(count);
}
/**
* Increment received bytes by count
* @param count to increment
*/
// @Override
public void incrReceivedBytes(int count) {
receivedBytes.incr(count);
}
/**
* Add an RPC queue time sample
* @param qTime the queue time
*/
// @Override
public void addRpcQueueTime(int qTime) {
rpcQueueTime.add(qTime);
if (rpcQuantileEnable) {
for (MutableQuantiles q : rpcQueueTimeMillisQuantiles) {
q.add(qTime);
}
}
}
/**
* Add an RPC processing time sample
* @param processingTime the processing time
*/
// @Override
public void addRpcProcessingTime(int processingTime) {
rpcProcessingTime.add(processingTime);
if (rpcQuantileEnable) {
for (MutableQuantiles q : rpcProcessingTimeMillisQuantiles) {
q.add(processingTime);
}
}
}
}
16
View Complete Implementation : Nfs3Metrics.java
Copyright Apache License 2.0
Author : aliyun-beta
Copyright Apache License 2.0
Author : aliyun-beta
/**
* This clreplaced is for maintaining the various NFS gateway activity statistics and
* publishing them through the metrics interfaces.
*/
@InterfaceAudience.Private
@Metrics(about = "Nfs3 metrics", context = "dfs")
public clreplaced Nfs3Metrics {
// All mutable rates are in nanoseconds
// No metric for nullProcedure;
@Metric
MutableRate getattr;
@Metric
MutableRate setattr;
@Metric
MutableRate lookup;
@Metric
MutableRate access;
@Metric
MutableRate readlink;
@Metric
MutableRate read;
final MutableQuantiles[] readNanosQuantiles;
@Metric
MutableRate write;
final MutableQuantiles[] writeNanosQuantiles;
@Metric
MutableRate create;
@Metric
MutableRate mkdir;
@Metric
MutableRate symlink;
@Metric
MutableRate mknod;
@Metric
MutableRate remove;
@Metric
MutableRate rmdir;
@Metric
MutableRate rename;
@Metric
MutableRate link;
@Metric
MutableRate readdir;
@Metric
MutableRate readdirplus;
@Metric
MutableRate fsstat;
@Metric
MutableRate fsinfo;
@Metric
MutableRate pathconf;
@Metric
MutableRate commit;
final MutableQuantiles[] commitNanosQuantiles;
@Metric
MutableCounterLong bytesWritten;
@Metric
MutableCounterLong bytesRead;
final MetricsRegistry registry = new MetricsRegistry("nfs3");
final String name;
JvmMetrics jvmMetrics = null;
public Nfs3Metrics(String name, String sessionId, int[] intervals, final JvmMetrics jvmMetrics) {
this.name = name;
this.jvmMetrics = jvmMetrics;
registry.tag(SessionId, sessionId);
final int len = intervals.length;
readNanosQuantiles = new MutableQuantiles[len];
writeNanosQuantiles = new MutableQuantiles[len];
commitNanosQuantiles = new MutableQuantiles[len];
for (int i = 0; i < len; i++) {
int interval = intervals[i];
readNanosQuantiles[i] = registry.newQuantiles("readProcessNanos" + interval + "s", "Read process in ns", "ops", "latency", interval);
writeNanosQuantiles[i] = registry.newQuantiles("writeProcessNanos" + interval + "s", "Write process in ns", "ops", "latency", interval);
commitNanosQuantiles[i] = registry.newQuantiles("commitProcessNanos" + interval + "s", "Commit process in ns", "ops", "latency", interval);
}
}
public static Nfs3Metrics create(Configuration conf, String gatewayName) {
String sessionId = conf.get(DFSConfigKeys.DFS_METRICS_SESSION_ID_KEY);
MetricsSystem ms = DefaultMetricsSystem.instance();
JvmMetrics jm = JvmMetrics.create(gatewayName, sessionId, ms);
// Percentile measurement is [50th,75th,90th,95th,99th] currently
int[] intervals = conf.getInts(NfsConfigKeys.NFS_METRICS_PERCENTILES_INTERVALS_KEY);
return ms.register(new Nfs3Metrics(gatewayName, sessionId, intervals, jm));
}
public String name() {
return name;
}
public JvmMetrics getJvmMetrics() {
return jvmMetrics;
}
public void incrBytesWritten(long bytes) {
bytesWritten.incr(bytes);
}
public void incrBytesRead(long bytes) {
bytesRead.incr(bytes);
}
public void addGetattr(long latencyNanos) {
getattr.add(latencyNanos);
}
public void addSetattr(long latencyNanos) {
setattr.add(latencyNanos);
}
public void addLookup(long latencyNanos) {
lookup.add(latencyNanos);
}
public void addAccess(long latencyNanos) {
access.add(latencyNanos);
}
public void addReadlink(long latencyNanos) {
readlink.add(latencyNanos);
}
public void addRead(long latencyNanos) {
read.add(latencyNanos);
for (MutableQuantiles q : readNanosQuantiles) {
q.add(latencyNanos);
}
}
public void addWrite(long latencyNanos) {
write.add(latencyNanos);
for (MutableQuantiles q : writeNanosQuantiles) {
q.add(latencyNanos);
}
}
public void addCreate(long latencyNanos) {
create.add(latencyNanos);
}
public void addMkdir(long latencyNanos) {
mkdir.add(latencyNanos);
}
public void addSymlink(long latencyNanos) {
symlink.add(latencyNanos);
}
public void addMknod(long latencyNanos) {
mknod.add(latencyNanos);
}
public void addRemove(long latencyNanos) {
remove.add(latencyNanos);
}
public void addRmdir(long latencyNanos) {
rmdir.add(latencyNanos);
}
public void addRename(long latencyNanos) {
rename.add(latencyNanos);
}
public void addLink(long latencyNanos) {
link.add(latencyNanos);
}
public void addReaddir(long latencyNanos) {
readdir.add(latencyNanos);
}
public void addReaddirplus(long latencyNanos) {
readdirplus.add(latencyNanos);
}
public void addFsstat(long latencyNanos) {
fsstat.add(latencyNanos);
}
public void addFsinfo(long latencyNanos) {
fsinfo.add(latencyNanos);
}
public void addPathconf(long latencyNanos) {
pathconf.add(latencyNanos);
}
public void addCommit(long latencyNanos) {
commit.add(latencyNanos);
for (MutableQuantiles q : commitNanosQuantiles) {
q.add(latencyNanos);
}
}
}
16
View Complete Implementation : QueueMetrics.java
Copyright Apache License 2.0
Author : aliyun-beta
Copyright Apache License 2.0
Author : aliyun-beta
@InterfaceAudience.Private
@Metrics(context = "yarn")
public clreplaced QueueMetrics implements MetricsSource {
@Metric("# of apps submitted")
MutableCounterInt appsSubmitted;
@Metric("# of running apps")
MutableGaugeInt appsRunning;
@Metric("# of pending apps")
MutableGaugeInt appsPending;
@Metric("# of apps completed")
MutableCounterInt appsCompleted;
@Metric("# of apps killed")
MutableCounterInt appsKilled;
@Metric("# of apps failed")
MutableCounterInt appsFailed;
@Metric("Allocated memory in MB")
MutableGaugeInt allocatedMB;
@Metric("Allocated CPU in virtual cores")
MutableGaugeInt allocatedVCores;
@Metric("# of allocated containers")
MutableGaugeInt allocatedContainers;
@Metric("Aggregate # of allocated containers")
MutableCounterLong aggregateContainersAllocated;
@Metric("Aggregate # of allocated node-local containers")
MutableCounterLong aggregateNodeLocalContainersAllocated;
@Metric("Aggregate # of allocated rack-local containers")
MutableCounterLong aggregateRackLocalContainersAllocated;
@Metric("Aggregate # of allocated off-switch containers")
MutableCounterLong aggregateOffSwitchContainersAllocated;
@Metric("Aggregate # of released containers")
MutableCounterLong aggregateContainersReleased;
@Metric("Available memory in MB")
MutableGaugeInt availableMB;
@Metric("Available CPU in virtual cores")
MutableGaugeInt availableVCores;
@Metric("Pending memory allocation in MB")
MutableGaugeInt pendingMB;
@Metric("Pending CPU allocation in virtual cores")
MutableGaugeInt pendingVCores;
@Metric("# of pending containers")
MutableGaugeInt pendingContainers;
@Metric("# of reserved memory in MB")
MutableGaugeInt reservedMB;
@Metric("Reserved CPU in virtual cores")
MutableGaugeInt reservedVCores;
@Metric("# of reserved containers")
MutableGaugeInt reservedContainers;
@Metric("# of active users")
MutableGaugeInt activeUsers;
@Metric("# of active applications")
MutableGaugeInt activeApplications;
@Metric("App Attempt First Container Allocation Delay")
MutableRate appAttemptFirstContainerAllocationDelay;
private final MutableGaugeInt[] runningTime;
private TimeBucketMetrics<ApplicationId> runBuckets;
static final Logger LOG = LoggerFactory.getLogger(QueueMetrics.clreplaced);
static final MetricsInfo RECORD_INFO = info("QueueMetrics", "Metrics for the resource scheduler");
protected static final MetricsInfo QUEUE_INFO = info("Queue", "Metrics by queue");
protected static final MetricsInfo USER_INFO = info("User", "Metrics by user");
static final Splitter Q_SPLITTER = Splitter.on('.').omitEmptyStrings().trimResults();
protected final MetricsRegistry registry;
protected final String queueName;
protected final QueueMetrics parent;
protected final MetricsSystem metricsSystem;
protected final Map<String, QueueMetrics> users;
protected final Configuration conf;
protected QueueMetrics(MetricsSystem ms, String queueName, Queue parent, boolean enableUserMetrics, Configuration conf) {
registry = new MetricsRegistry(RECORD_INFO);
this.queueName = queueName;
this.parent = parent != null ? parent.getMetrics() : null;
this.users = enableUserMetrics ? new HashMap<String, QueueMetrics>() : null;
metricsSystem = ms;
this.conf = conf;
runningTime = buildBuckets(conf);
}
protected QueueMetrics tag(MetricsInfo info, String value) {
registry.tag(info, value);
return this;
}
protected static StringBuilder sourceName(String queueName) {
StringBuilder sb = new StringBuilder(RECORD_INFO.name());
int i = 0;
for (String node : Q_SPLITTER.split(queueName)) {
sb.append(",q").append(i++).append('=').append(node);
}
return sb;
}
public synchronized static QueueMetrics forQueue(String queueName, Queue parent, boolean enableUserMetrics, Configuration conf) {
return forQueue(DefaultMetricsSystem.instance(), queueName, parent, enableUserMetrics, conf);
}
/**
* Helper method to clear cache.
*/
@Private
public synchronized static void clearQueueMetrics() {
queueMetrics.clear();
}
/**
* Simple metrics cache to help prevent re-registrations.
*/
protected final static Map<String, QueueMetrics> queueMetrics = new HashMap<String, QueueMetrics>();
public synchronized static QueueMetrics forQueue(MetricsSystem ms, String queueName, Queue parent, boolean enableUserMetrics, Configuration conf) {
QueueMetrics metrics = queueMetrics.get(queueName);
if (metrics == null) {
metrics = new QueueMetrics(ms, queueName, parent, enableUserMetrics, conf).tag(QUEUE_INFO, queueName);
// Register with the MetricsSystems
if (ms != null) {
metrics = ms.register(sourceName(queueName).toString(), "Metrics for queue: " + queueName, metrics);
}
queueMetrics.put(queueName, metrics);
}
return metrics;
}
public synchronized QueueMetrics getUserMetrics(String userName) {
if (users == null) {
return null;
}
QueueMetrics metrics = users.get(userName);
if (metrics == null) {
metrics = new QueueMetrics(metricsSystem, queueName, null, false, conf);
users.put(userName, metrics);
metricsSystem.register(sourceName(queueName).append(",user=").append(userName).toString(), "Metrics for user '" + userName + "' in queue '" + queueName + "'", metrics.tag(QUEUE_INFO, queueName).tag(USER_INFO, userName));
}
return metrics;
}
private ArrayList<Integer> parseInts(String value) {
ArrayList<Integer> result = new ArrayList<Integer>();
for (String s : value.split(",")) {
result.add(Integer.parseInt(s.trim()));
}
return result;
}
private MutableGaugeInt[] buildBuckets(Configuration conf) {
ArrayList<Integer> buckets = parseInts(conf.get(YarnConfiguration.RM_METRICS_RUNTIME_BUCKETS, YarnConfiguration.DEFAULT_RM_METRICS_RUNTIME_BUCKETS));
MutableGaugeInt[] result = new MutableGaugeInt[buckets.size() + 1];
result[0] = registry.newGauge("running_0", "", 0);
long[] cuts = new long[buckets.size()];
for (int i = 0; i < buckets.size(); ++i) {
result[i + 1] = registry.newGauge("running_" + buckets.get(i), "", 0);
// covert from min to ms
cuts[i] = buckets.get(i) * 1000L * 60;
}
this.runBuckets = new TimeBucketMetrics<ApplicationId>(cuts);
return result;
}
private void updateRunningTime() {
int[] counts = runBuckets.getBucketCounts(System.currentTimeMillis());
for (int i = 0; i < counts.length; ++i) {
runningTime[i].set(counts[i]);
}
}
public void getMetrics(MetricsCollector collector, boolean all) {
updateRunningTime();
registry.snapshot(collector.addRecord(registry.info()), all);
}
public void submitApp(String user) {
appsSubmitted.incr();
QueueMetrics userMetrics = getUserMetrics(user);
if (userMetrics != null) {
userMetrics.submitApp(user);
}
if (parent != null) {
parent.submitApp(user);
}
}
public void submitAppAttempt(String user) {
appsPending.incr();
QueueMetrics userMetrics = getUserMetrics(user);
if (userMetrics != null) {
userMetrics.submitAppAttempt(user);
}
if (parent != null) {
parent.submitAppAttempt(user);
}
}
public void runAppAttempt(ApplicationId appId, String user) {
runBuckets.add(appId, System.currentTimeMillis());
appsRunning.incr();
appsPending.decr();
QueueMetrics userMetrics = getUserMetrics(user);
if (userMetrics != null) {
userMetrics.runAppAttempt(appId, user);
}
if (parent != null) {
parent.runAppAttempt(appId, user);
}
}
public void finishAppAttempt(ApplicationId appId, boolean isPending, String user) {
runBuckets.remove(appId);
if (isPending) {
appsPending.decr();
} else {
appsRunning.decr();
}
QueueMetrics userMetrics = getUserMetrics(user);
if (userMetrics != null) {
userMetrics.finishAppAttempt(appId, isPending, user);
}
if (parent != null) {
parent.finishAppAttempt(appId, isPending, user);
}
}
public void finishApp(String user, RMAppState rmAppFinalState) {
switch(rmAppFinalState) {
case KILLED:
appsKilled.incr();
break;
case FAILED:
appsFailed.incr();
break;
default:
appsCompleted.incr();
break;
}
QueueMetrics userMetrics = getUserMetrics(user);
if (userMetrics != null) {
userMetrics.finishApp(user, rmAppFinalState);
}
if (parent != null) {
parent.finishApp(user, rmAppFinalState);
}
}
public void moveAppFrom(AppSchedulingInfo app) {
if (app.isPending()) {
appsPending.decr();
} else {
appsRunning.decr();
}
QueueMetrics userMetrics = getUserMetrics(app.getUser());
if (userMetrics != null) {
userMetrics.moveAppFrom(app);
}
if (parent != null) {
parent.moveAppFrom(app);
}
}
public void moveAppTo(AppSchedulingInfo app) {
if (app.isPending()) {
appsPending.incr();
} else {
appsRunning.incr();
}
QueueMetrics userMetrics = getUserMetrics(app.getUser());
if (userMetrics != null) {
userMetrics.moveAppTo(app);
}
if (parent != null) {
parent.moveAppTo(app);
}
}
/**
* Set available resources. To be called by scheduler periodically as
* resources become available.
* @param limit resource limit
*/
public void setAvailableResourcesToQueue(Resource limit) {
availableMB.set(limit.getMemory());
availableVCores.set(limit.getVirtualCores());
}
/**
* Set available resources. To be called by scheduler periodically as
* resources become available.
* @param user
* @param limit resource limit
*/
public void setAvailableResourcesToUser(String user, Resource limit) {
QueueMetrics userMetrics = getUserMetrics(user);
if (userMetrics != null) {
userMetrics.setAvailableResourcesToQueue(limit);
}
}
/**
* Increment pending resource metrics
* @param user
* @param containers
* @param res the TOTAL delta of resources note this is different from
* the other APIs which use per container resource
*/
public void incrPendingResources(String user, int containers, Resource res) {
_incrPendingResources(containers, res);
QueueMetrics userMetrics = getUserMetrics(user);
if (userMetrics != null) {
userMetrics.incrPendingResources(user, containers, res);
}
if (parent != null) {
parent.incrPendingResources(user, containers, res);
}
}
private void _incrPendingResources(int containers, Resource res) {
pendingContainers.incr(containers);
pendingMB.incr(res.getMemory() * containers);
pendingVCores.incr(res.getVirtualCores() * containers);
}
public void decrPendingResources(String user, int containers, Resource res) {
_decrPendingResources(containers, res);
QueueMetrics userMetrics = getUserMetrics(user);
if (userMetrics != null) {
userMetrics.decrPendingResources(user, containers, res);
}
if (parent != null) {
parent.decrPendingResources(user, containers, res);
}
}
private void _decrPendingResources(int containers, Resource res) {
// if #container = 0, means change container resource
pendingContainers.decr(containers);
pendingMB.decr(res.getMemory() * Math.max(containers, 1));
pendingVCores.decr(res.getVirtualCores() * Math.max(containers, 1));
}
public void incrNodeTypeAggregations(String user, NodeType type) {
if (type == NodeType.NODE_LOCAL) {
aggregateNodeLocalContainersAllocated.incr();
} else if (type == NodeType.RACK_LOCAL) {
aggregateRackLocalContainersAllocated.incr();
} else if (type == NodeType.OFF_SWITCH) {
aggregateOffSwitchContainersAllocated.incr();
} else {
return;
}
QueueMetrics userMetrics = getUserMetrics(user);
if (userMetrics != null) {
userMetrics.incrNodeTypeAggregations(user, type);
}
if (parent != null) {
parent.incrNodeTypeAggregations(user, type);
}
}
public void allocateResources(String user, int containers, Resource res, boolean decrPending) {
// if #containers = 0, means change container resource
allocatedContainers.incr(containers);
aggregateContainersAllocated.incr(containers);
allocatedMB.incr(res.getMemory() * Math.max(containers, 1));
allocatedVCores.incr(res.getVirtualCores() * Math.max(containers, 1));
if (decrPending) {
_decrPendingResources(containers, res);
}
QueueMetrics userMetrics = getUserMetrics(user);
if (userMetrics != null) {
userMetrics.allocateResources(user, containers, res, decrPending);
}
if (parent != null) {
parent.allocateResources(user, containers, res, decrPending);
}
}
public void releaseResources(String user, int containers, Resource res) {
// if #container = 0, means change container resource.
allocatedContainers.decr(containers);
aggregateContainersReleased.incr(containers);
allocatedMB.decr(res.getMemory() * Math.max(containers, 1));
allocatedVCores.decr(res.getVirtualCores() * Math.max(containers, 1));
QueueMetrics userMetrics = getUserMetrics(user);
if (userMetrics != null) {
userMetrics.releaseResources(user, containers, res);
}
if (parent != null) {
parent.releaseResources(user, containers, res);
}
}
public void reserveResource(String user, Resource res) {
reservedContainers.incr();
reservedMB.incr(res.getMemory());
reservedVCores.incr(res.getVirtualCores());
QueueMetrics userMetrics = getUserMetrics(user);
if (userMetrics != null) {
userMetrics.reserveResource(user, res);
}
if (parent != null) {
parent.reserveResource(user, res);
}
}
public void unreserveResource(String user, Resource res) {
reservedContainers.decr();
reservedMB.decr(res.getMemory());
reservedVCores.decr(res.getVirtualCores());
QueueMetrics userMetrics = getUserMetrics(user);
if (userMetrics != null) {
userMetrics.unreserveResource(user, res);
}
if (parent != null) {
parent.unreserveResource(user, res);
}
}
public void incrActiveUsers() {
activeUsers.incr();
}
public void decrActiveUsers() {
activeUsers.decr();
}
public void activateApp(String user) {
activeApplications.incr();
QueueMetrics userMetrics = getUserMetrics(user);
if (userMetrics != null) {
userMetrics.activateApp(user);
}
if (parent != null) {
parent.activateApp(user);
}
}
public void deactivateApp(String user) {
activeApplications.decr();
QueueMetrics userMetrics = getUserMetrics(user);
if (userMetrics != null) {
userMetrics.deactivateApp(user);
}
if (parent != null) {
parent.deactivateApp(user);
}
}
public void addAppAttemptFirstContainerAllocationDelay(long latency) {
appAttemptFirstContainerAllocationDelay.add(latency);
}
public int getAppsSubmitted() {
return appsSubmitted.value();
}
public int getAppsRunning() {
return appsRunning.value();
}
public int getAppsPending() {
return appsPending.value();
}
public int getAppsCompleted() {
return appsCompleted.value();
}
public int getAppsKilled() {
return appsKilled.value();
}
public int getAppsFailed() {
return appsFailed.value();
}
public Resource getAllocatedResources() {
return BuilderUtils.newResource(allocatedMB.value(), allocatedVCores.value());
}
public int getAllocatedMB() {
return allocatedMB.value();
}
public int getAllocatedVirtualCores() {
return allocatedVCores.value();
}
public int getAllocatedContainers() {
return allocatedContainers.value();
}
public int getAvailableMB() {
return availableMB.value();
}
public int getAvailableVirtualCores() {
return availableVCores.value();
}
public int getPendingMB() {
return pendingMB.value();
}
public int getPendingVirtualCores() {
return pendingVCores.value();
}
public int getPendingContainers() {
return pendingContainers.value();
}
public int getReservedMB() {
return reservedMB.value();
}
public int getReservedVirtualCores() {
return reservedVCores.value();
}
public int getReservedContainers() {
return reservedContainers.value();
}
public int getActiveUsers() {
return activeUsers.value();
}
public int getActiveApps() {
return activeApplications.value();
}
public MetricsSystem getMetricsSystem() {
return metricsSystem;
}
public long getAggregateAllocatedContainers() {
return aggregateContainersAllocated.value();
}
public long getAggregateNodeLocalContainersAllocated() {
return aggregateNodeLocalContainersAllocated.value();
}
public long getAggregateRackLocalContainersAllocated() {
return aggregateRackLocalContainersAllocated.value();
}
public long getAggregateOffSwitchContainersAllocated() {
return aggregateOffSwitchContainersAllocated.value();
}
public long getAggegatedReleasedContainers() {
return aggregateContainersReleased.value();
}
}
16
View Complete Implementation : DataNodeMetrics.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
/**
* This clreplaced is for maintaining the various DataNode statistics
* and publishing them through the metrics interfaces.
* This also registers the JMX MBean for RPC.
* <p>
* This clreplaced has a number of metrics variables that are publicly accessible;
* these variables (objects) have methods to update their values;
* for example:
* <p> {@link #blocksRead}.inc()
*/
@InterfaceAudience.Private
@Metrics(about = "DataNode metrics", context = "dfs")
public clreplaced DataNodeMetrics {
@Metric
MutableCounterLong bytesWritten;
@Metric("Milliseconds spent writing")
MutableCounterLong totalWriteTime;
@Metric
MutableCounterLong bytesRead;
@Metric("Milliseconds spent reading")
MutableCounterLong totalReadTime;
@Metric
MutableCounterLong blocksWritten;
@Metric
MutableCounterLong blocksRead;
@Metric
MutableCounterLong blocksReplicated;
@Metric
MutableCounterLong blocksRemoved;
@Metric
MutableCounterLong blocksVerified;
@Metric
MutableCounterLong blockVerificationFailures;
@Metric
MutableCounterLong blocksCached;
@Metric
MutableCounterLong blocksUncached;
@Metric
MutableCounterLong readsFromLocalClient;
@Metric
MutableCounterLong readsFromRemoteClient;
@Metric
MutableCounterLong writesFromLocalClient;
@Metric
MutableCounterLong writesFromRemoteClient;
@Metric
MutableCounterLong blocksGetLocalPathInfo;
@Metric("Bytes read by remote client")
MutableCounterLong remoteBytesRead;
@Metric("Bytes written by remote client")
MutableCounterLong remoteBytesWritten;
// RamDisk metrics on read/write
@Metric
MutableCounterLong ramDiskBlocksWrite;
@Metric
MutableCounterLong ramDiskBlocksWriteFallback;
@Metric
MutableCounterLong ramDiskBytesWrite;
@Metric
MutableCounterLong ramDiskBlocksReadHits;
// RamDisk metrics on eviction
@Metric
MutableCounterLong ramDiskBlocksEvicted;
@Metric
MutableCounterLong ramDiskBlocksEvictedWithoutRead;
@Metric
MutableRate ramDiskBlocksEvictionWindowMs;
final MutableQuantiles[] ramDiskBlocksEvictionWindowMsQuantiles;
// RamDisk metrics on lazy persist
@Metric
MutableCounterLong ramDiskBlocksLazyPersisted;
@Metric
MutableCounterLong ramDiskBlocksDeletedBeforeLazyPersisted;
@Metric
MutableCounterLong ramDiskBytesLazyPersisted;
@Metric
MutableRate ramDiskBlocksLazyPersistWindowMs;
final MutableQuantiles[] ramDiskBlocksLazyPersistWindowMsQuantiles;
@Metric
MutableCounterLong fsyncCount;
@Metric
MutableCounterLong volumeFailures;
@Metric("Count of network errors on the datanode")
MutableCounterLong datanodeNetworkErrors;
@Metric("Count of active dataNode xceivers")
private MutableGaugeInt dataNodeActiveXceiversCount;
@Metric
MutableRate readBlockOp;
@Metric
MutableRate writeBlockOp;
@Metric
MutableRate blockChecksumOp;
@Metric
MutableRate copyBlockOp;
@Metric
MutableRate replaceBlockOp;
@Metric
MutableRate heartbeats;
@Metric
MutableRate heartbeatsTotal;
@Metric
MutableRate lifelines;
@Metric
MutableRate blockReports;
@Metric
MutableRate incrementalBlockReports;
@Metric
MutableRate cacheReports;
@Metric
MutableRate packetAckRoundTripTimeNanos;
final MutableQuantiles[] packetAckRoundTripTimeNanosQuantiles;
@Metric
MutableRate flushNanos;
final MutableQuantiles[] flushNanosQuantiles;
@Metric
MutableRate fsyncNanos;
final MutableQuantiles[] fsyncNanosQuantiles;
@Metric
MutableRate sendDataPacketBlockedOnNetworkNanos;
final MutableQuantiles[] sendDataPacketBlockedOnNetworkNanosQuantiles;
@Metric
MutableRate sendDataPacketTransferNanos;
final MutableQuantiles[] sendDataPacketTransferNanosQuantiles;
@Metric("Count of blocks in pending IBR")
private MutableGaugeLong blocksInPendingIBR;
@Metric("Count of blocks at receiving status in pending IBR")
private MutableGaugeLong blocksReceivingInPendingIBR;
@Metric("Count of blocks at received status in pending IBR")
private MutableGaugeLong blocksReceivedInPendingIBR;
@Metric("Count of blocks at deleted status in pending IBR")
private MutableGaugeLong blocksDeletedInPendingIBR;
@Metric("Count of erasure coding reconstruction tasks")
MutableCounterLong ecReconstructionTasks;
@Metric("Count of erasure coding failed reconstruction tasks")
MutableCounterLong ecFailedReconstructionTasks;
@Metric("Nanoseconds spent by decoding tasks")
MutableCounterLong ecDecodingTimeNanos;
@Metric("Bytes read by erasure coding worker")
MutableCounterLong ecReconstructionBytesRead;
@Metric("Bytes written by erasure coding worker")
MutableCounterLong ecReconstructionBytesWritten;
@Metric("Bytes remote read by erasure coding worker")
MutableCounterLong ecReconstructionRemoteBytesRead;
@Metric("Milliseconds spent on read by erasure coding worker")
private MutableCounterLong ecReconstructionReadTimeMillis;
@Metric("Milliseconds spent on decoding by erasure coding worker")
private MutableCounterLong ecReconstructionDecodingTimeMillis;
@Metric("Milliseconds spent on write by erasure coding worker")
private MutableCounterLong ecReconstructionWriteTimeMillis;
@Metric("Sum of all BPServiceActors command queue length")
private MutableCounterLong sumOfActorCommandQueueLength;
@Metric("Num of processed commands of all BPServiceActors")
private MutableCounterLong numProcessedCommands;
final MetricsRegistry registry = new MetricsRegistry("datanode");
@Metric("Milliseconds spent on calling NN rpc")
private MutableRatesWithAggregation nnRpcLatency = registry.newRatesWithAggregation("nnRpcLatency");
final String name;
JvmMetrics jvmMetrics = null;
private DataNodeUsageReportUtil dnUsageReportUtil;
public DataNodeMetrics(String name, String sessionId, int[] intervals, final JvmMetrics jvmMetrics) {
this.name = name;
this.jvmMetrics = jvmMetrics;
registry.tag(SessionId, sessionId);
final int len = intervals.length;
dnUsageReportUtil = new DataNodeUsageReportUtil();
packetAckRoundTripTimeNanosQuantiles = new MutableQuantiles[len];
flushNanosQuantiles = new MutableQuantiles[len];
fsyncNanosQuantiles = new MutableQuantiles[len];
sendDataPacketBlockedOnNetworkNanosQuantiles = new MutableQuantiles[len];
sendDataPacketTransferNanosQuantiles = new MutableQuantiles[len];
ramDiskBlocksEvictionWindowMsQuantiles = new MutableQuantiles[len];
ramDiskBlocksLazyPersistWindowMsQuantiles = new MutableQuantiles[len];
for (int i = 0; i < len; i++) {
int interval = intervals[i];
packetAckRoundTripTimeNanosQuantiles[i] = registry.newQuantiles("packetAckRoundTripTimeNanos" + interval + "s", "Packet Ack RTT in ns", "ops", "latency", interval);
flushNanosQuantiles[i] = registry.newQuantiles("flushNanos" + interval + "s", "Disk flush latency in ns", "ops", "latency", interval);
fsyncNanosQuantiles[i] = registry.newQuantiles("fsyncNanos" + interval + "s", "Disk fsync latency in ns", "ops", "latency", interval);
sendDataPacketBlockedOnNetworkNanosQuantiles[i] = registry.newQuantiles("sendDataPacketBlockedOnNetworkNanos" + interval + "s", "Time blocked on network while sending a packet in ns", "ops", "latency", interval);
sendDataPacketTransferNanosQuantiles[i] = registry.newQuantiles("sendDataPacketTransferNanos" + interval + "s", "Time reading from disk and writing to network while sending " + "a packet in ns", "ops", "latency", interval);
ramDiskBlocksEvictionWindowMsQuantiles[i] = registry.newQuantiles("ramDiskBlocksEvictionWindows" + interval + "s", "Time between the RamDisk block write and eviction in ms", "ops", "latency", interval);
ramDiskBlocksLazyPersistWindowMsQuantiles[i] = registry.newQuantiles("ramDiskBlocksLazyPersistWindows" + interval + "s", "Time between the RamDisk block write and disk persist in ms", "ops", "latency", interval);
}
}
public static DataNodeMetrics create(Configuration conf, String dnName) {
String sessionId = conf.get(DFSConfigKeys.DFS_METRICS_SESSION_ID_KEY);
MetricsSystem ms = DefaultMetricsSystem.instance();
JvmMetrics jm = JvmMetrics.create("DataNode", sessionId, ms);
String name = "DataNodeActivity-" + (dnName.isEmpty() ? "UndefinedDataNodeName" + ThreadLocalRandom.current().nextInt() : dnName.replace(':', '-'));
// Percentile measurement is off by default, by watching no intervals
int[] intervals = conf.getInts(DFSConfigKeys.DFS_METRICS_PERCENTILES_INTERVALS_KEY);
return ms.register(name, null, new DataNodeMetrics(name, sessionId, intervals, jm));
}
public String name() {
return name;
}
public JvmMetrics getJvmMetrics() {
return jvmMetrics;
}
public void addHeartbeat(long latency, String rpcMetricSuffix) {
heartbeats.add(latency);
if (rpcMetricSuffix != null) {
nnRpcLatency.add("HeartbeatsFor" + rpcMetricSuffix, latency);
}
}
public void addHeartbeatTotal(long latency, String rpcMetricSuffix) {
heartbeatsTotal.add(latency);
if (rpcMetricSuffix != null) {
nnRpcLatency.add("HeartbeatsTotalFor" + rpcMetricSuffix, latency);
}
}
public void addLifeline(long latency, String rpcMetricSuffix) {
lifelines.add(latency);
if (rpcMetricSuffix != null) {
nnRpcLatency.add("LifelinesFor" + rpcMetricSuffix, latency);
}
}
public void addBlockReport(long latency, String rpcMetricSuffix) {
blockReports.add(latency);
if (rpcMetricSuffix != null) {
nnRpcLatency.add("BlockReportsFor" + rpcMetricSuffix, latency);
}
}
public void addIncrementalBlockReport(long latency, String rpcMetricSuffix) {
incrementalBlockReports.add(latency);
if (rpcMetricSuffix != null) {
nnRpcLatency.add("IncrementalBlockReportsFor" + rpcMetricSuffix, latency);
}
}
public void addCacheReport(long latency) {
cacheReports.add(latency);
}
public void incrBlocksReplicated() {
blocksReplicated.incr();
}
public void incrBlocksWritten() {
blocksWritten.incr();
}
public void incrBlocksRemoved(int delta) {
blocksRemoved.incr(delta);
}
public void incrBytesWritten(int delta) {
bytesWritten.incr(delta);
}
public void incrBlockVerificationFailures() {
blockVerificationFailures.incr();
}
public void incrBlocksVerified() {
blocksVerified.incr();
}
public void incrBlocksCached(int delta) {
blocksCached.incr(delta);
}
public void incrBlocksUncached(int delta) {
blocksUncached.incr(delta);
}
public void addReadBlockOp(long latency) {
readBlockOp.add(latency);
}
public void addWriteBlockOp(long latency) {
writeBlockOp.add(latency);
}
public void addReplaceBlockOp(long latency) {
replaceBlockOp.add(latency);
}
public void addCopyBlockOp(long latency) {
copyBlockOp.add(latency);
}
public void addBlockChecksumOp(long latency) {
blockChecksumOp.add(latency);
}
public void incrBytesRead(int delta) {
bytesRead.incr(delta);
}
public void incrBlocksRead() {
blocksRead.incr();
}
public void incrFsyncCount() {
fsyncCount.incr();
}
public void incrTotalWriteTime(long timeTaken) {
totalWriteTime.incr(timeTaken);
}
public void incrTotalReadTime(long timeTaken) {
totalReadTime.incr(timeTaken);
}
public void addPacketAckRoundTripTimeNanos(long latencyNanos) {
packetAckRoundTripTimeNanos.add(latencyNanos);
for (MutableQuantiles q : packetAckRoundTripTimeNanosQuantiles) {
q.add(latencyNanos);
}
}
public void addFlushNanos(long latencyNanos) {
flushNanos.add(latencyNanos);
for (MutableQuantiles q : flushNanosQuantiles) {
q.add(latencyNanos);
}
}
public void addFsyncNanos(long latencyNanos) {
fsyncNanos.add(latencyNanos);
for (MutableQuantiles q : fsyncNanosQuantiles) {
q.add(latencyNanos);
}
}
public void shutdown() {
DefaultMetricsSystem.shutdown();
}
public void incrWritesFromClient(boolean local, long size) {
if (local) {
writesFromLocalClient.incr();
} else {
writesFromRemoteClient.incr();
remoteBytesWritten.incr(size);
}
}
public void incrReadsFromClient(boolean local, long size) {
if (local) {
readsFromLocalClient.incr();
} else {
readsFromRemoteClient.incr();
remoteBytesRead.incr(size);
}
}
public void incrVolumeFailures() {
volumeFailures.incr();
}
public void incrDatanodeNetworkErrors() {
datanodeNetworkErrors.incr();
}
/**
* Increment for getBlockLocalPathInfo calls
*/
public void incrBlocksGetLocalPathInfo() {
blocksGetLocalPathInfo.incr();
}
public void addSendDataPacketBlockedOnNetworkNanos(long latencyNanos) {
sendDataPacketBlockedOnNetworkNanos.add(latencyNanos);
for (MutableQuantiles q : sendDataPacketBlockedOnNetworkNanosQuantiles) {
q.add(latencyNanos);
}
}
public void addSendDataPacketTransferNanos(long latencyNanos) {
sendDataPacketTransferNanos.add(latencyNanos);
for (MutableQuantiles q : sendDataPacketTransferNanosQuantiles) {
q.add(latencyNanos);
}
}
public void incrRamDiskBlocksWrite() {
ramDiskBlocksWrite.incr();
}
public void incrRamDiskBlocksWriteFallback() {
ramDiskBlocksWriteFallback.incr();
}
public void addRamDiskBytesWrite(long bytes) {
ramDiskBytesWrite.incr(bytes);
}
public void incrRamDiskBlocksReadHits() {
ramDiskBlocksReadHits.incr();
}
public void incrRamDiskBlocksEvicted() {
ramDiskBlocksEvicted.incr();
}
public void incrRamDiskBlocksEvictedWithoutRead() {
ramDiskBlocksEvictedWithoutRead.incr();
}
public void addRamDiskBlocksEvictionWindowMs(long latencyMs) {
ramDiskBlocksEvictionWindowMs.add(latencyMs);
for (MutableQuantiles q : ramDiskBlocksEvictionWindowMsQuantiles) {
q.add(latencyMs);
}
}
public void incrRamDiskBlocksLazyPersisted() {
ramDiskBlocksLazyPersisted.incr();
}
public void incrRamDiskBlocksDeletedBeforeLazyPersisted() {
ramDiskBlocksDeletedBeforeLazyPersisted.incr();
}
public void incrRamDiskBytesLazyPersisted(long bytes) {
ramDiskBytesLazyPersisted.incr(bytes);
}
public void addRamDiskBlocksLazyPersistWindowMs(long latencyMs) {
ramDiskBlocksLazyPersistWindowMs.add(latencyMs);
for (MutableQuantiles q : ramDiskBlocksLazyPersistWindowMsQuantiles) {
q.add(latencyMs);
}
}
/**
* Resets blocks in pending IBR to zero.
*/
public void resetBlocksInPendingIBR() {
blocksInPendingIBR.set(0);
blocksReceivingInPendingIBR.set(0);
blocksReceivedInPendingIBR.set(0);
blocksDeletedInPendingIBR.set(0);
}
public void incrBlocksInPendingIBR() {
blocksInPendingIBR.incr();
}
public void incrBlocksReceivingInPendingIBR() {
blocksReceivingInPendingIBR.incr();
}
public void incrBlocksReceivedInPendingIBR() {
blocksReceivedInPendingIBR.incr();
}
public void incrBlocksDeletedInPendingIBR() {
blocksDeletedInPendingIBR.incr();
}
public void incrECReconstructionTasks() {
ecReconstructionTasks.incr();
}
public void incrECFailedReconstructionTasks() {
ecFailedReconstructionTasks.incr();
}
public void incrDataNodeActiveXceiversCount() {
dataNodeActiveXceiversCount.incr();
}
public void decrDataNodeActiveXceiversCount() {
dataNodeActiveXceiversCount.decr();
}
public void setDataNodeActiveXceiversCount(int value) {
dataNodeActiveXceiversCount.set(value);
}
public void incrECDecodingTime(long decodingTimeNanos) {
ecDecodingTimeNanos.incr(decodingTimeNanos);
}
public void incrECReconstructionBytesRead(long bytes) {
ecReconstructionBytesRead.incr(bytes);
}
public void incrECReconstructionRemoteBytesRead(long bytes) {
ecReconstructionRemoteBytesRead.incr(bytes);
}
public void incrECReconstructionBytesWritten(long bytes) {
ecReconstructionBytesWritten.incr(bytes);
}
public void incrECReconstructionReadTime(long millis) {
ecReconstructionReadTimeMillis.incr(millis);
}
public void incrECReconstructionWriteTime(long millis) {
ecReconstructionWriteTimeMillis.incr(millis);
}
public void incrECReconstructionDecodingTime(long millis) {
ecReconstructionDecodingTimeMillis.incr(millis);
}
public DataNodeUsageReport getDNUsageReport(long timeSinceLastReport) {
return dnUsageReportUtil.getUsageReport(bytesWritten.value(), bytesRead.value(), totalWriteTime.value(), totalReadTime.value(), blocksWritten.value(), blocksRead.value(), timeSinceLastReport);
}
public void incrActorCmdQueueLength(int delta) {
sumOfActorCommandQueueLength.incr(delta);
}
public void incrNumProcessedCommands() {
numProcessedCommands.incr();
}
}
16
View Complete Implementation : DataNodeMetrics.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
/**
* This clreplaced is for maintaining the various DataNode statistics
* and publishing them through the metrics interfaces.
* This also registers the JMX MBean for RPC.
* <p>
* This clreplaced has a number of metrics variables that are publicly accessible;
* these variables (objects) have methods to update their values;
* for example:
* <p> {@link #blocksRead}.inc()
*/
@InterfaceAudience.Private
@Metrics(about = "DataNode metrics", context = "dfs")
public clreplaced DataNodeMetrics {
@Metric
MutableCounterLong bytesWritten;
@Metric
MutableCounterLong bytesRead;
@Metric
MutableCounterLong blocksWritten;
@Metric
MutableCounterLong blocksRead;
@Metric
MutableCounterLong blocksReplicated;
@Metric
MutableCounterLong blocksRemoved;
@Metric
MutableCounterLong blocksVerified;
@Metric
MutableCounterLong blockVerificationFailures;
@Metric
MutableCounterLong readsFromLocalClient;
@Metric
MutableCounterLong readsFromRemoteClient;
@Metric
MutableCounterLong writesFromLocalClient;
@Metric
MutableCounterLong writesFromRemoteClient;
@Metric
MutableCounterLong volumeFailures;
@Metric
MutableRate readBlockOp;
@Metric
MutableRate writeBlockOp;
@Metric
MutableRate blockChecksumOp;
@Metric
MutableRate copyBlockOp;
@Metric
MutableRate replaceBlockOp;
@Metric
MutableRate heartbeats;
@Metric
MutableRate blockReports;
final MetricsRegistry registry = new MetricsRegistry("datanode");
final String name;
static final Random rng = new Random();
public DataNodeMetrics(String name, String sessionId) {
this.name = name;
registry.tag(SessionId, sessionId);
}
public static DataNodeMetrics create(Configuration conf, String dnName) {
String sessionId = conf.get(DFSConfigKeys.DFS_METRICS_SESSION_ID_KEY);
MetricsSystem ms = DefaultMetricsSystem.instance();
JvmMetrics.create("DataNode", sessionId, ms);
String name = "DataNodeActivity-" + (dnName.isEmpty() ? "UndefinedDataNodeName" + rng.nextInt() : dnName.replace(':', '-'));
return ms.register(name, null, new DataNodeMetrics(name, sessionId));
}
public String name() {
return name;
}
public void addHeartbeat(long latency) {
heartbeats.add(latency);
}
public void addBlockReport(long latency) {
blockReports.add(latency);
}
public void incrBlocksReplicated(int delta) {
blocksReplicated.incr(delta);
}
public void incrBlocksWritten() {
blocksWritten.incr();
}
public void incrBlocksRemoved(int delta) {
blocksRemoved.incr(delta);
}
public void incrBytesWritten(int delta) {
bytesWritten.incr(delta);
}
public void incrBlockVerificationFailures() {
blockVerificationFailures.incr();
}
public void incrBlocksVerified() {
blocksVerified.incr();
}
public void addReadBlockOp(long latency) {
readBlockOp.add(latency);
}
public void addWriteBlockOp(long latency) {
writeBlockOp.add(latency);
}
public void addReplaceBlockOp(long latency) {
replaceBlockOp.add(latency);
}
public void addCopyBlockOp(long latency) {
copyBlockOp.add(latency);
}
public void addBlockChecksumOp(long latency) {
blockChecksumOp.add(latency);
}
public void incrBytesRead(int delta) {
bytesRead.incr(delta);
}
public void incrBlocksRead() {
blocksRead.incr();
}
public void shutdown() {
DefaultMetricsSystem.shutdown();
}
public void incrWritesFromClient(boolean local) {
(local ? writesFromLocalClient : writesFromRemoteClient).incr();
}
public void incrReadsFromClient(boolean local) {
(local ? readsFromLocalClient : readsFromRemoteClient).incr();
}
public void incrVolumeFailures() {
volumeFailures.incr();
}
}
16
View Complete Implementation : SCMContainerPlacementMetrics.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
/**
* This clreplaced is for maintaining Topology aware container placement statistics.
*/
@Metrics(about = "SCM Container Placement Metrics", context = OzoneConsts.OZONE)
public clreplaced SCMContainerPlacementMetrics implements MetricsSource {
public static final String SOURCE_NAME = SCMContainerPlacementMetrics.clreplaced.getSimpleName();
private static final MetricsInfo RECORD_INFO = Interns.info(SOURCE_NAME, "SCM Container Placement Metrics");
private static MetricsRegistry registry;
// total datanode allocation request count
@Metric
private MutableCounterLong datanodeRequestCount;
// datanode allocation attempt count, including success, fallback and failed
@Metric
private MutableCounterLong datanodeChooseAttemptCount;
// datanode successful allocation count
@Metric
private MutableCounterLong datanodeChooseSuccessCount;
// datanode allocated with some allocation constrains compromised
@Metric
private MutableCounterLong datanodeChooseFallbackCount;
public SCMContainerPlacementMetrics() {
}
public static SCMContainerPlacementMetrics create() {
MetricsSystem ms = DefaultMetricsSystem.instance();
registry = new MetricsRegistry(RECORD_INFO);
return ms.register(SOURCE_NAME, "SCM Container Placement Metrics", new SCMContainerPlacementMetrics());
}
public void incrDatanodeRequestCount(long count) {
this.datanodeRequestCount.incr(count);
}
public void incrDatanodeChooseSuccessCount() {
this.datanodeChooseSuccessCount.incr(1);
}
public void incrDatanodeChooseFallbackCount() {
this.datanodeChooseFallbackCount.incr(1);
}
public void incrDatanodeChooseAttemptCount() {
this.datanodeChooseAttemptCount.incr(1);
}
public void unRegister() {
MetricsSystem ms = DefaultMetricsSystem.instance();
ms.unregisterSource(SOURCE_NAME);
}
@VisibleForTesting
public long getDatanodeRequestCount() {
return this.datanodeRequestCount.value();
}
@VisibleForTesting
public long getDatanodeChooseSuccessCount() {
return this.datanodeChooseSuccessCount.value();
}
@VisibleForTesting
public long getDatanodeChooseFallbackCount() {
return this.datanodeChooseFallbackCount.value();
}
@VisibleForTesting
public long getDatanodeChooseAttemptCount() {
return this.datanodeChooseAttemptCount.value();
}
@Override
public void getMetrics(MetricsCollector collector, boolean all) {
registry.snapshot(collector.addRecord(registry.info().name()), true);
}
}
16
View Complete Implementation : SCMPipelineMetrics.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
/**
* This clreplaced maintains Pipeline related metrics.
*/
@InterfaceAudience.Private
@Metrics(about = "SCM PipelineManager Metrics", context = OzoneConsts.OZONE)
public final clreplaced SCMPipelineMetrics implements MetricsSource {
private static final String SOURCE_NAME = SCMPipelineMetrics.clreplaced.getSimpleName();
private MetricsRegistry registry;
@Metric
private MutableCounterLong numPipelineAllocated;
@Metric
private MutableCounterLong numPipelineCreated;
@Metric
private MutableCounterLong numPipelineCreationFailed;
@Metric
private MutableCounterLong numPipelineDestroyed;
@Metric
private MutableCounterLong numPipelineDestroyFailed;
@Metric
private MutableCounterLong numPipelineReportProcessed;
@Metric
private MutableCounterLong numPipelineReportProcessingFailed;
private Map<PipelineID, MutableCounterLong> numBlocksAllocated;
/**
* Private constructor.
*/
private SCMPipelineMetrics() {
this.registry = new MetricsRegistry(SOURCE_NAME);
numBlocksAllocated = new ConcurrentHashMap<>();
}
/**
* Create and returns SCMPipelineMetrics instance.
*
* @return SCMPipelineMetrics
*/
public static SCMPipelineMetrics create() {
MetricsSystem ms = DefaultMetricsSystem.instance();
return ms.register(SOURCE_NAME, "SCM PipelineManager Metrics", new SCMPipelineMetrics());
}
/**
* Unregister the metrics instance.
*/
public void unRegister() {
MetricsSystem ms = DefaultMetricsSystem.instance();
ms.unregisterSource(SOURCE_NAME);
}
@Override
@SuppressWarnings("SuspiciousMethodCalls")
public void getMetrics(MetricsCollector collector, boolean all) {
MetricsRecordBuilder recordBuilder = collector.addRecord(SOURCE_NAME);
numPipelineAllocated.snapshot(recordBuilder, true);
numPipelineCreated.snapshot(recordBuilder, true);
numPipelineCreationFailed.snapshot(recordBuilder, true);
numPipelineDestroyed.snapshot(recordBuilder, true);
numPipelineDestroyFailed.snapshot(recordBuilder, true);
numPipelineReportProcessed.snapshot(recordBuilder, true);
numPipelineReportProcessingFailed.snapshot(recordBuilder, true);
numBlocksAllocated.forEach((pid, metric) -> metric.snapshot(recordBuilder, true));
}
void createPerPipelineMetrics(Pipeline pipeline) {
numBlocksAllocated.put(pipeline.getId(), new MutableCounterLong(Interns.info(getBlockAllocationMetricName(pipeline), "Number of blocks allocated in pipeline " + pipeline.getId()), 0L));
}
public static String getBlockAllocationMetricName(Pipeline pipeline) {
return "NumBlocksAllocated-" + pipeline.getType() + "-" + pipeline.getFactor() + "-" + pipeline.getId().getId();
}
void removePipelineMetrics(PipelineID pipelineID) {
numBlocksAllocated.remove(pipelineID);
}
/**
* Increments number of blocks allocated for the pipeline.
*/
void incNumBlocksAllocated(PipelineID pipelineID) {
Optional.of(numBlocksAllocated.get(pipelineID)).ifPresent(MutableCounterLong::incr);
}
/**
* Increments number of pipeline allocation count, including succeeded
* and failed.
*/
void incNumPipelineAllocated() {
numPipelineAllocated.incr();
}
/**
* Increments number of successful pipeline creation count.
*/
void incNumPipelineCreated() {
numPipelineCreated.incr();
}
/**
* Increments number of failed pipeline creation count.
*/
void incNumPipelineCreationFailed() {
numPipelineCreationFailed.incr();
}
/**
* Increments number of successful pipeline destroy count.
*/
void incNumPipelineDestroyed() {
numPipelineDestroyed.incr();
}
/**
* Increments number of failed pipeline destroy count.
*/
void incNumPipelineDestroyFailed() {
numPipelineDestroyFailed.incr();
}
/**
* Increments number of pipeline report processed count.
*/
void incNumPipelineReportProcessed() {
numPipelineReportProcessed.incr();
}
/**
* Increments number of pipeline report processing failed count.
*/
void incNumPipelineReportProcessingFailed() {
numPipelineReportProcessingFailed.incr();
}
}
16
View Complete Implementation : DataNodeMetrics.java
Copyright Apache License 2.0
Author : yncxcw
Copyright Apache License 2.0
Author : yncxcw
/**
* This clreplaced is for maintaining the various DataNode statistics
* and publishing them through the metrics interfaces.
* This also registers the JMX MBean for RPC.
* <p>
* This clreplaced has a number of metrics variables that are publicly accessible;
* these variables (objects) have methods to update their values;
* for example:
* <p> {@link #blocksRead}.inc()
*/
@InterfaceAudience.Private
@Metrics(about = "DataNode metrics", context = "dfs")
public clreplaced DataNodeMetrics {
@Metric
MutableCounterLong bytesWritten;
@Metric("Milliseconds spent writing")
MutableCounterLong totalWriteTime;
@Metric
MutableCounterLong bytesRead;
@Metric("Milliseconds spent reading")
MutableCounterLong totalReadTime;
@Metric
MutableCounterLong blocksWritten;
@Metric
MutableCounterLong blocksRead;
@Metric
MutableCounterLong blocksReplicated;
@Metric
MutableCounterLong blocksRemoved;
@Metric
MutableCounterLong blocksVerified;
@Metric
MutableCounterLong blockVerificationFailures;
@Metric
MutableCounterLong blocksCached;
@Metric
MutableCounterLong blocksUncached;
@Metric
MutableCounterLong readsFromLocalClient;
@Metric
MutableCounterLong readsFromRemoteClient;
@Metric
MutableCounterLong writesFromLocalClient;
@Metric
MutableCounterLong writesFromRemoteClient;
@Metric
MutableCounterLong blocksGetLocalPathInfo;
@Metric("Bytes read by remote client")
MutableCounterLong remoteBytesRead;
@Metric("Bytes written by remote client")
MutableCounterLong remoteBytesWritten;
// RamDisk metrics on read/write
@Metric
MutableCounterLong ramDiskBlocksWrite;
@Metric
MutableCounterLong ramDiskBlocksWriteFallback;
@Metric
MutableCounterLong ramDiskBytesWrite;
@Metric
MutableCounterLong ramDiskBlocksReadHits;
// RamDisk metrics on eviction
@Metric
MutableCounterLong ramDiskBlocksEvicted;
@Metric
MutableCounterLong ramDiskBlocksEvictedWithoutRead;
@Metric
MutableRate ramDiskBlocksEvictionWindowMs;
final MutableQuantiles[] ramDiskBlocksEvictionWindowMsQuantiles;
// RamDisk metrics on lazy persist
@Metric
MutableCounterLong ramDiskBlocksLazyPersisted;
@Metric
MutableCounterLong ramDiskBlocksDeletedBeforeLazyPersisted;
@Metric
MutableCounterLong ramDiskBytesLazyPersisted;
@Metric
MutableRate ramDiskBlocksLazyPersistWindowMs;
final MutableQuantiles[] ramDiskBlocksLazyPersistWindowMsQuantiles;
@Metric
MutableCounterLong fsyncCount;
@Metric
MutableCounterLong volumeFailures;
@Metric("Count of network errors on the datanode")
MutableCounterLong datanodeNetworkErrors;
@Metric
MutableRate readBlockOp;
@Metric
MutableRate writeBlockOp;
@Metric
MutableRate blockChecksumOp;
@Metric
MutableRate copyBlockOp;
@Metric
MutableRate replaceBlockOp;
@Metric
MutableRate heartbeats;
@Metric
MutableRate blockReports;
@Metric
MutableRate incrementalBlockReports;
@Metric
MutableRate cacheReports;
@Metric
MutableRate packetAckRoundTripTimeNanos;
final MutableQuantiles[] packetAckRoundTripTimeNanosQuantiles;
@Metric
MutableRate flushNanos;
final MutableQuantiles[] flushNanosQuantiles;
@Metric
MutableRate fsyncNanos;
final MutableQuantiles[] fsyncNanosQuantiles;
@Metric
MutableRate sendDataPacketBlockedOnNetworkNanos;
final MutableQuantiles[] sendDataPacketBlockedOnNetworkNanosQuantiles;
@Metric
MutableRate sendDataPacketTransferNanos;
final MutableQuantiles[] sendDataPacketTransferNanosQuantiles;
final MetricsRegistry registry = new MetricsRegistry("datanode");
final String name;
JvmMetrics jvmMetrics = null;
public DataNodeMetrics(String name, String sessionId, int[] intervals, final JvmMetrics jvmMetrics) {
this.name = name;
this.jvmMetrics = jvmMetrics;
registry.tag(SessionId, sessionId);
final int len = intervals.length;
packetAckRoundTripTimeNanosQuantiles = new MutableQuantiles[len];
flushNanosQuantiles = new MutableQuantiles[len];
fsyncNanosQuantiles = new MutableQuantiles[len];
sendDataPacketBlockedOnNetworkNanosQuantiles = new MutableQuantiles[len];
sendDataPacketTransferNanosQuantiles = new MutableQuantiles[len];
ramDiskBlocksEvictionWindowMsQuantiles = new MutableQuantiles[len];
ramDiskBlocksLazyPersistWindowMsQuantiles = new MutableQuantiles[len];
for (int i = 0; i < len; i++) {
int interval = intervals[i];
packetAckRoundTripTimeNanosQuantiles[i] = registry.newQuantiles("packetAckRoundTripTimeNanos" + interval + "s", "Packet Ack RTT in ns", "ops", "latency", interval);
flushNanosQuantiles[i] = registry.newQuantiles("flushNanos" + interval + "s", "Disk flush latency in ns", "ops", "latency", interval);
fsyncNanosQuantiles[i] = registry.newQuantiles("fsyncNanos" + interval + "s", "Disk fsync latency in ns", "ops", "latency", interval);
sendDataPacketBlockedOnNetworkNanosQuantiles[i] = registry.newQuantiles("sendDataPacketBlockedOnNetworkNanos" + interval + "s", "Time blocked on network while sending a packet in ns", "ops", "latency", interval);
sendDataPacketTransferNanosQuantiles[i] = registry.newQuantiles("sendDataPacketTransferNanos" + interval + "s", "Time reading from disk and writing to network while sending " + "a packet in ns", "ops", "latency", interval);
ramDiskBlocksEvictionWindowMsQuantiles[i] = registry.newQuantiles("ramDiskBlocksEvictionWindows" + interval + "s", "Time between the RamDisk block write and eviction in ms", "ops", "latency", interval);
ramDiskBlocksLazyPersistWindowMsQuantiles[i] = registry.newQuantiles("ramDiskBlocksLazyPersistWindows" + interval + "s", "Time between the RamDisk block write and disk persist in ms", "ops", "latency", interval);
}
}
public static DataNodeMetrics create(Configuration conf, String dnName) {
String sessionId = conf.get(DFSConfigKeys.DFS_METRICS_SESSION_ID_KEY);
MetricsSystem ms = DefaultMetricsSystem.instance();
JvmMetrics jm = JvmMetrics.create("DataNode", sessionId, ms);
String name = "DataNodeActivity-" + (dnName.isEmpty() ? "UndefinedDataNodeName" + DFSUtil.getRandom().nextInt() : dnName.replace(':', '-'));
// Percentile measurement is off by default, by watching no intervals
int[] intervals = conf.getInts(DFSConfigKeys.DFS_METRICS_PERCENTILES_INTERVALS_KEY);
return ms.register(name, null, new DataNodeMetrics(name, sessionId, intervals, jm));
}
public String name() {
return name;
}
public JvmMetrics getJvmMetrics() {
return jvmMetrics;
}
public void addHeartbeat(long latency) {
heartbeats.add(latency);
}
public void addBlockReport(long latency) {
blockReports.add(latency);
}
public void addIncrementalBlockReport(long latency) {
incrementalBlockReports.add(latency);
}
public void addCacheReport(long latency) {
cacheReports.add(latency);
}
public void incrBlocksReplicated(int delta) {
blocksReplicated.incr(delta);
}
public void incrBlocksWritten() {
blocksWritten.incr();
}
public void incrBlocksRemoved(int delta) {
blocksRemoved.incr(delta);
}
public void incrBytesWritten(int delta) {
bytesWritten.incr(delta);
}
public void incrBlockVerificationFailures() {
blockVerificationFailures.incr();
}
public void incrBlocksVerified() {
blocksVerified.incr();
}
public void incrBlocksCached(int delta) {
blocksCached.incr(delta);
}
public void incrBlocksUncached(int delta) {
blocksUncached.incr(delta);
}
public void addReadBlockOp(long latency) {
readBlockOp.add(latency);
}
public void addWriteBlockOp(long latency) {
writeBlockOp.add(latency);
}
public void addReplaceBlockOp(long latency) {
replaceBlockOp.add(latency);
}
public void addCopyBlockOp(long latency) {
copyBlockOp.add(latency);
}
public void addBlockChecksumOp(long latency) {
blockChecksumOp.add(latency);
}
public void incrBytesRead(int delta) {
bytesRead.incr(delta);
}
public void incrBlocksRead() {
blocksRead.incr();
}
public void incrFsyncCount() {
fsyncCount.incr();
}
public void incrTotalWriteTime(long timeTaken) {
totalWriteTime.incr(timeTaken);
}
public void incrTotalReadTime(long timeTaken) {
totalReadTime.incr(timeTaken);
}
public void addPacketAckRoundTripTimeNanos(long latencyNanos) {
packetAckRoundTripTimeNanos.add(latencyNanos);
for (MutableQuantiles q : packetAckRoundTripTimeNanosQuantiles) {
q.add(latencyNanos);
}
}
public void addFlushNanos(long latencyNanos) {
flushNanos.add(latencyNanos);
for (MutableQuantiles q : flushNanosQuantiles) {
q.add(latencyNanos);
}
}
public void addFsyncNanos(long latencyNanos) {
fsyncNanos.add(latencyNanos);
for (MutableQuantiles q : fsyncNanosQuantiles) {
q.add(latencyNanos);
}
}
public void shutdown() {
DefaultMetricsSystem.shutdown();
}
public void incrWritesFromClient(boolean local, long size) {
if (local) {
writesFromLocalClient.incr();
} else {
writesFromRemoteClient.incr();
remoteBytesWritten.incr(size);
}
}
public void incrReadsFromClient(boolean local, long size) {
if (local) {
readsFromLocalClient.incr();
} else {
readsFromRemoteClient.incr();
remoteBytesRead.incr(size);
}
}
public void incrVolumeFailures() {
volumeFailures.incr();
}
public void incrDatanodeNetworkErrors() {
datanodeNetworkErrors.incr();
}
/**
* Increment for getBlockLocalPathInfo calls
*/
public void incrBlocksGetLocalPathInfo() {
blocksGetLocalPathInfo.incr();
}
public void addSendDataPacketBlockedOnNetworkNanos(long latencyNanos) {
sendDataPacketBlockedOnNetworkNanos.add(latencyNanos);
for (MutableQuantiles q : sendDataPacketBlockedOnNetworkNanosQuantiles) {
q.add(latencyNanos);
}
}
public void addSendDataPacketTransferNanos(long latencyNanos) {
sendDataPacketTransferNanos.add(latencyNanos);
for (MutableQuantiles q : sendDataPacketTransferNanosQuantiles) {
q.add(latencyNanos);
}
}
public void incrRamDiskBlocksWrite() {
ramDiskBlocksWrite.incr();
}
public void incrRamDiskBlocksWriteFallback() {
ramDiskBlocksWriteFallback.incr();
}
public void addRamDiskBytesWrite(long bytes) {
ramDiskBytesWrite.incr(bytes);
}
public void incrRamDiskBlocksReadHits() {
ramDiskBlocksReadHits.incr();
}
public void incrRamDiskBlocksEvicted() {
ramDiskBlocksEvicted.incr();
}
public void incrRamDiskBlocksEvictedWithoutRead() {
ramDiskBlocksEvictedWithoutRead.incr();
}
public void addRamDiskBlocksEvictionWindowMs(long latencyMs) {
ramDiskBlocksEvictionWindowMs.add(latencyMs);
for (MutableQuantiles q : ramDiskBlocksEvictionWindowMsQuantiles) {
q.add(latencyMs);
}
}
public void incrRamDiskBlocksLazyPersisted() {
ramDiskBlocksLazyPersisted.incr();
}
public void incrRamDiskBlocksDeletedBeforeLazyPersisted() {
ramDiskBlocksDeletedBeforeLazyPersisted.incr();
}
public void incrRamDiskBytesLazyPersisted(long bytes) {
ramDiskBytesLazyPersisted.incr(bytes);
}
public void addRamDiskBlocksLazyPersistWindowMs(long latencyMs) {
ramDiskBlocksLazyPersistWindowMs.add(latencyMs);
for (MutableQuantiles q : ramDiskBlocksLazyPersistWindowMsQuantiles) {
q.add(latencyMs);
}
}
}
16
View Complete Implementation : NameNodeMetrics.java
Copyright Apache License 2.0
Author : yncxcw
Copyright Apache License 2.0
Author : yncxcw
/**
* This clreplaced is for maintaining the various NameNode activity statistics
* and publishing them through the metrics interfaces.
*/
@Metrics(name = "NameNodeActivity", about = "NameNode metrics", context = "dfs")
public clreplaced NameNodeMetrics {
final MetricsRegistry registry = new MetricsRegistry("namenode");
@Metric
MutableCounterLong createFileOps;
@Metric
MutableCounterLong filesCreated;
@Metric
MutableCounterLong filesAppended;
@Metric
MutableCounterLong getBlockLocations;
@Metric
MutableCounterLong filesRenamed;
@Metric
MutableCounterLong filesTruncated;
@Metric
MutableCounterLong getListingOps;
@Metric
MutableCounterLong deleteFileOps;
@Metric("Number of files/dirs deleted by delete or rename operations")
MutableCounterLong filesDeleted;
@Metric
MutableCounterLong fileInfoOps;
@Metric
MutableCounterLong addBlockOps;
@Metric
MutableCounterLong getAdditionalDatanodeOps;
@Metric
MutableCounterLong createSymlinkOps;
@Metric
MutableCounterLong getLinkTargetOps;
@Metric
MutableCounterLong filesInGetListingOps;
@Metric("Number of allowSnapshot operations")
MutableCounterLong allowSnapshotOps;
@Metric("Number of disallowSnapshot operations")
MutableCounterLong disallowSnapshotOps;
@Metric("Number of createSnapshot operations")
MutableCounterLong createSnapshotOps;
@Metric("Number of deleteSnapshot operations")
MutableCounterLong deleteSnapshotOps;
@Metric("Number of renameSnapshot operations")
MutableCounterLong renameSnapshotOps;
@Metric("Number of listSnapshottableDirectory operations")
MutableCounterLong listSnapshottableDirOps;
@Metric("Number of snapshotDiffReport operations")
MutableCounterLong snapshotDiffReportOps;
@Metric("Number of blockReceivedAndDeleted calls")
MutableCounterLong blockReceivedAndDeletedOps;
@Metric("Number of blockReports from individual storages")
MutableCounterLong storageBlockReportOps;
@Metric("Number of file system operations")
public long totalFileOps() {
return getBlockLocations.value() + createFileOps.value() + filesAppended.value() + addBlockOps.value() + getAdditionalDatanodeOps.value() + filesRenamed.value() + filesTruncated.value() + deleteFileOps.value() + getListingOps.value() + fileInfoOps.value() + getLinkTargetOps.value() + createSnapshotOps.value() + deleteSnapshotOps.value() + allowSnapshotOps.value() + disallowSnapshotOps.value() + renameSnapshotOps.value() + listSnapshottableDirOps.value() + createSymlinkOps.value() + snapshotDiffReportOps.value();
}
@Metric("Journal transactions")
MutableRate transactions;
@Metric("Journal syncs")
MutableRate syncs;
final MutableQuantiles[] syncsQuantiles;
@Metric("Journal transactions batched in sync")
MutableCounterLong transactionsBatchedInSync;
@Metric("Block report")
MutableRate blockReport;
final MutableQuantiles[] blockReportQuantiles;
@Metric("Cache report")
MutableRate cacheReport;
final MutableQuantiles[] cacheReportQuantiles;
@Metric("Duration in SafeMode at startup in msec")
MutableGaugeInt safeModeTime;
@Metric("Time loading FS Image at startup in msec")
MutableGaugeInt fsImageLoadTime;
@Metric("GetImageServlet getEdit")
MutableRate getEdit;
@Metric("GetImageServlet getImage")
MutableRate getImage;
@Metric("GetImageServlet putImage")
MutableRate putImage;
JvmMetrics jvmMetrics = null;
NameNodeMetrics(String processName, String sessionId, int[] intervals, final JvmMetrics jvmMetrics) {
this.jvmMetrics = jvmMetrics;
registry.tag(ProcessName, processName).tag(SessionId, sessionId);
final int len = intervals.length;
syncsQuantiles = new MutableQuantiles[len];
blockReportQuantiles = new MutableQuantiles[len];
cacheReportQuantiles = new MutableQuantiles[len];
for (int i = 0; i < len; i++) {
int interval = intervals[i];
syncsQuantiles[i] = registry.newQuantiles("syncs" + interval + "s", "Journal syncs", "ops", "latency", interval);
blockReportQuantiles[i] = registry.newQuantiles("blockReport" + interval + "s", "Block report", "ops", "latency", interval);
cacheReportQuantiles[i] = registry.newQuantiles("cacheReport" + interval + "s", "Cache report", "ops", "latency", interval);
}
}
public static NameNodeMetrics create(Configuration conf, NamenodeRole r) {
String sessionId = conf.get(DFSConfigKeys.DFS_METRICS_SESSION_ID_KEY);
String processName = r.toString();
MetricsSystem ms = DefaultMetricsSystem.instance();
JvmMetrics jm = JvmMetrics.create(processName, sessionId, ms);
// Percentile measurement is off by default, by watching no intervals
int[] intervals = conf.getInts(DFSConfigKeys.DFS_METRICS_PERCENTILES_INTERVALS_KEY);
return ms.register(new NameNodeMetrics(processName, sessionId, intervals, jm));
}
public JvmMetrics getJvmMetrics() {
return jvmMetrics;
}
public void shutdown() {
DefaultMetricsSystem.shutdown();
}
public void incrGetBlockLocations() {
getBlockLocations.incr();
}
public void incrFilesCreated() {
filesCreated.incr();
}
public void incrCreateFileOps() {
createFileOps.incr();
}
public void incrFilesAppended() {
filesAppended.incr();
}
public void incrAddBlockOps() {
addBlockOps.incr();
}
public void incrGetAdditionalDatanodeOps() {
getAdditionalDatanodeOps.incr();
}
public void incrFilesRenamed() {
filesRenamed.incr();
}
public void incrFilesTruncated() {
filesTruncated.incr();
}
public void incrFilesDeleted(long delta) {
filesDeleted.incr(delta);
}
public void incrDeleteFileOps() {
deleteFileOps.incr();
}
public void incrGetListingOps() {
getListingOps.incr();
}
public void incrFilesInGetListingOps(int delta) {
filesInGetListingOps.incr(delta);
}
public void incrFileInfoOps() {
fileInfoOps.incr();
}
public void incrCreateSymlinkOps() {
createSymlinkOps.incr();
}
public void incrGetLinkTargetOps() {
getLinkTargetOps.incr();
}
public void incrAllowSnapshotOps() {
allowSnapshotOps.incr();
}
public void incrDisAllowSnapshotOps() {
disallowSnapshotOps.incr();
}
public void incrCreateSnapshotOps() {
createSnapshotOps.incr();
}
public void incrDeleteSnapshotOps() {
deleteSnapshotOps.incr();
}
public void incrRenameSnapshotOps() {
renameSnapshotOps.incr();
}
public void incrListSnapshottableDirOps() {
listSnapshottableDirOps.incr();
}
public void incrSnapshotDiffReportOps() {
snapshotDiffReportOps.incr();
}
public void incrBlockReceivedAndDeletedOps() {
blockReceivedAndDeletedOps.incr();
}
public void incrStorageBlockReportOps() {
storageBlockReportOps.incr();
}
public void addTransaction(long latency) {
transactions.add(latency);
}
public void incrTransactionsBatchedInSync() {
transactionsBatchedInSync.incr();
}
public void addSync(long elapsed) {
syncs.add(elapsed);
for (MutableQuantiles q : syncsQuantiles) {
q.add(elapsed);
}
}
public void setFsImageLoadTime(long elapsed) {
fsImageLoadTime.set((int) elapsed);
}
public void addBlockReport(long latency) {
blockReport.add(latency);
for (MutableQuantiles q : blockReportQuantiles) {
q.add(latency);
}
}
public void addCacheBlockReport(long latency) {
cacheReport.add(latency);
for (MutableQuantiles q : cacheReportQuantiles) {
q.add(latency);
}
}
public void setSafeModeTime(long elapsed) {
safeModeTime.set((int) elapsed);
}
public void addGetEdit(long latency) {
getEdit.add(latency);
}
public void addGetImage(long latency) {
getImage.add(latency);
}
public void addPutImage(long latency) {
putImage.add(latency);
}
}
15
View Complete Implementation : MetricsSystemImpl.java
Copyright Apache License 2.0
Author : aliyun-beta
Copyright Apache License 2.0
Author : aliyun-beta
/**
* A base clreplaced for metrics system singletons
*/
@InterfaceAudience.Private
@Metrics(context = "metricssystem")
public clreplaced MetricsSystemImpl extends MetricsSystem implements MetricsSource {
static final Log LOG = LogFactory.getLog(MetricsSystemImpl.clreplaced);
static final String MS_NAME = "MetricsSystem";
static final String MS_STATS_NAME = MS_NAME + ",sub=Stats";
static final String MS_STATS_DESC = "Metrics system metrics";
static final String MS_CONTROL_NAME = MS_NAME + ",sub=Control";
static final String MS_INIT_MODE_KEY = "hadoop.metrics.init.mode";
enum InitMode {
NORMAL, STANDBY
}
private final Map<String, MetricsSourceAdapter> sources;
private final Map<String, MetricsSource> allSources;
private final Map<String, MetricsSinkAdapter> sinks;
private final Map<String, MetricsSink> allSinks;
// The callback list is used by register(Callback callback), while
// the callback map is used by register(String name, String desc, T sink)
private final List<Callback> callbacks;
private final Map<String, Callback> namedCallbacks;
private final MetricsCollectorImpl collector;
private final MetricsRegistry registry = new MetricsRegistry(MS_NAME);
@Metric({ "Snapshot", "Snapshot stats" })
MutableStat snapshotStat;
@Metric({ "Publish", "Publishing stats" })
MutableStat publishStat;
@Metric("Dropped updates by all sinks")
MutableCounterLong droppedPubAll;
private final List<MetricsTag> injectedTags;
// Things that are changed by init()/start()/stop()
private String prefix;
private MetricsFilter sourceFilter;
private MetricsConfig config;
private Map<String, MetricsConfig> sourceConfigs, sinkConfigs;
private boolean monitoring = false;
private Timer timer;
// milliseconds
private long period;
// number of timer invocations * period
private long logicalTime;
private ObjectName mbeanName;
private boolean publishSelfMetrics = true;
private MetricsSourceAdapter sysSource;
// for mini cluster mode
private int refCount = 0;
/**
* Construct the metrics system
* @param prefix for the system
*/
public MetricsSystemImpl(String prefix) {
this.prefix = prefix;
allSources = Maps.newHashMap();
sources = Maps.newLinkedHashMap();
allSinks = Maps.newHashMap();
sinks = Maps.newLinkedHashMap();
sourceConfigs = Maps.newHashMap();
sinkConfigs = Maps.newHashMap();
callbacks = Lists.newArrayList();
namedCallbacks = Maps.newHashMap();
injectedTags = Lists.newArrayList();
collector = new MetricsCollectorImpl();
if (prefix != null) {
// prefix could be null for default ctor, which requires init later
initSystemMBean();
}
}
/**
* Construct the system but not initializing (read config etc.) it.
*/
public MetricsSystemImpl() {
this(null);
}
/**
* Initialized the metrics system with a prefix.
* @param prefix the system will look for configs with the prefix
* @return the metrics system object itself
*/
@Override
public synchronized MetricsSystem init(String prefix) {
if (monitoring && !DefaultMetricsSystem.inMiniClusterMode()) {
LOG.warn(this.prefix + " metrics system already initialized!");
return this;
}
this.prefix = checkNotNull(prefix, "prefix");
++refCount;
if (monitoring) {
// in mini cluster mode
LOG.info(this.prefix + " metrics system started (again)");
return this;
}
switch(initMode()) {
case NORMAL:
try {
start();
} catch (MetricsConfigException e) {
// Configuration errors (e.g., typos) should not be fatal.
// We can always start the metrics system later via JMX.
LOG.warn("Metrics system not started: " + e.getMessage());
LOG.debug("Stacktrace: ", e);
}
break;
case STANDBY:
LOG.info(prefix + " metrics system started in standby mode");
}
initSystemMBean();
return this;
}
@Override
public synchronized void start() {
checkNotNull(prefix, "prefix");
if (monitoring) {
LOG.warn(prefix + " metrics system already started!", new MetricsException("Illegal start"));
return;
}
for (Callback cb : callbacks) cb.preStart();
for (Callback cb : namedCallbacks.values()) cb.preStart();
configure(prefix);
startTimer();
monitoring = true;
LOG.info(prefix + " metrics system started");
for (Callback cb : callbacks) cb.postStart();
for (Callback cb : namedCallbacks.values()) cb.postStart();
}
@Override
public synchronized void stop() {
if (!monitoring && !DefaultMetricsSystem.inMiniClusterMode()) {
LOG.warn(prefix + " metrics system not yet started!", new MetricsException("Illegal stop"));
return;
}
if (!monitoring) {
// in mini cluster mode
LOG.info(prefix + " metrics system stopped (again)");
return;
}
for (Callback cb : callbacks) cb.preStop();
for (Callback cb : namedCallbacks.values()) cb.preStop();
LOG.info("Stopping " + prefix + " metrics system...");
stopTimer();
stopSources();
stopSinks();
clearConfigs();
monitoring = false;
LOG.info(prefix + " metrics system stopped.");
for (Callback cb : callbacks) cb.postStop();
for (Callback cb : namedCallbacks.values()) cb.postStop();
}
@Override
public synchronized <T> T register(String name, String desc, T source) {
MetricsSourceBuilder sb = MetricsAnnotations.newSourceBuilder(source);
final MetricsSource s = sb.build();
MetricsInfo si = sb.info();
String name2 = name == null ? si.name() : name;
final String finalDesc = desc == null ? si.description() : desc;
final // be friendly to non-metrics tests
String finalName = DefaultMetricsSystem.sourceName(name2, !monitoring);
allSources.put(finalName, s);
LOG.debug(finalName + ", " + finalDesc);
if (monitoring) {
registerSource(finalName, finalDesc, s);
}
// We want to re-register the source to pick up new config when the
// metrics system restarts.
register(finalName, new AbstractCallback() {
@Override
public void postStart() {
registerSource(finalName, finalDesc, s);
}
});
return source;
}
@Override
public synchronized void unregisterSource(String name) {
if (sources.containsKey(name)) {
sources.get(name).stop();
sources.remove(name);
}
if (allSources.containsKey(name)) {
allSources.remove(name);
}
if (namedCallbacks.containsKey(name)) {
namedCallbacks.remove(name);
}
}
synchronized void registerSource(String name, String desc, MetricsSource source) {
checkNotNull(config, "config");
MetricsConfig conf = sourceConfigs.get(name);
MetricsSourceAdapter sa = new MetricsSourceAdapter(prefix, name, desc, source, injectedTags, period, conf != null ? conf : config.subset(SOURCE_KEY));
sources.put(name, sa);
sa.start();
LOG.debug("Registered source " + name);
}
@Override
public synchronized <T extends MetricsSink> T register(final String name, final String description, final T sink) {
LOG.debug(name + ", " + description);
if (allSinks.containsKey(name)) {
LOG.warn("Sink " + name + " already exists!");
return sink;
}
allSinks.put(name, sink);
if (config != null) {
registerSink(name, description, sink);
}
// We want to re-register the sink to pick up new config
// when the metrics system restarts.
register(name, new AbstractCallback() {
@Override
public void postStart() {
register(name, description, sink);
}
});
return sink;
}
synchronized void registerSink(String name, String desc, MetricsSink sink) {
checkNotNull(config, "config");
MetricsConfig conf = sinkConfigs.get(name);
MetricsSinkAdapter sa = conf != null ? newSink(name, desc, sink, conf) : newSink(name, desc, sink, config.subset(SINK_KEY));
sinks.put(name, sa);
sa.start();
LOG.info("Registered sink " + name);
}
@Override
public synchronized void register(final Callback callback) {
callbacks.add((Callback) getProxyForCallback(callback));
}
private synchronized void register(String name, final Callback callback) {
namedCallbacks.put(name, (Callback) getProxyForCallback(callback));
}
private Object getProxyForCallback(final Callback callback) {
return Proxy.newProxyInstance(callback.getClreplaced().getClreplacedLoader(), new Clreplaced<?>[] { Callback.clreplaced }, new InvocationHandler() {
@Override
public Object invoke(Object proxy, Method method, Object[] args) throws Throwable {
try {
return method.invoke(callback, args);
} catch (Exception e) {
// These are not considered fatal.
LOG.warn("Caught exception in callback " + method.getName(), e);
}
return null;
}
});
}
@Override
public synchronized void startMetricsMBeans() {
for (MetricsSourceAdapter sa : sources.values()) {
sa.startMBeans();
}
}
@Override
public synchronized void stopMetricsMBeans() {
for (MetricsSourceAdapter sa : sources.values()) {
sa.stopMBeans();
}
}
@Override
public synchronized String currentConfig() {
PropertiesConfiguration saver = new PropertiesConfiguration();
StringWriter writer = new StringWriter();
saver.copy(config);
try {
saver.save(writer);
} catch (Exception e) {
throw new MetricsConfigException("Error stringify config", e);
}
return writer.toString();
}
private synchronized void startTimer() {
if (timer != null) {
LOG.warn(prefix + " metrics system timer already started!");
return;
}
logicalTime = 0;
long millis = period;
timer = new Timer("Timer for '" + prefix + "' metrics system", true);
timer.scheduleAtFixedRate(new TimerTask() {
@Override
public void run() {
try {
onTimerEvent();
} catch (Exception e) {
LOG.warn("Error invoking metrics timer", e);
}
}
}, millis, millis);
LOG.info("Scheduled snapshot period at " + (period / 1000) + " second(s).");
}
synchronized void onTimerEvent() {
logicalTime += period;
if (sinks.size() > 0) {
publishMetrics(sampleMetrics(), false);
}
}
/**
* Requests an immediate publish of all metrics from sources to sinks.
*/
@Override
public synchronized void publishMetricsNow() {
if (sinks.size() > 0) {
publishMetrics(sampleMetrics(), true);
}
}
/**
* Sample all the sources for a snapshot of metrics/tags
* @return the metrics buffer containing the snapshot
*/
@VisibleForTesting
public synchronized MetricsBuffer sampleMetrics() {
collector.clear();
MetricsBufferBuilder bufferBuilder = new MetricsBufferBuilder();
for (Entry<String, MetricsSourceAdapter> entry : sources.entrySet()) {
if (sourceFilter == null || sourceFilter.accepts(entry.getKey())) {
snapshotMetrics(entry.getValue(), bufferBuilder);
}
}
if (publishSelfMetrics) {
snapshotMetrics(sysSource, bufferBuilder);
}
MetricsBuffer buffer = bufferBuilder.get();
return buffer;
}
private void snapshotMetrics(MetricsSourceAdapter sa, MetricsBufferBuilder bufferBuilder) {
long startTime = Time.now();
bufferBuilder.add(sa.name(), sa.getMetrics(collector, true));
collector.clear();
snapshotStat.add(Time.now() - startTime);
LOG.debug("Snapshotted source " + sa.name());
}
/**
* Publish a metrics snapshot to all the sinks
* @param buffer the metrics snapshot to publish
* @param immediate indicates that we should publish metrics immediately
* instead of using a separate thread.
*/
synchronized void publishMetrics(MetricsBuffer buffer, boolean immediate) {
int dropped = 0;
for (MetricsSinkAdapter sa : sinks.values()) {
long startTime = Time.now();
boolean result;
if (immediate) {
result = sa.putMetricsImmediate(buffer);
} else {
result = sa.putMetrics(buffer, logicalTime);
}
dropped += result ? 0 : 1;
publishStat.add(Time.now() - startTime);
}
droppedPubAll.incr(dropped);
}
private synchronized void stopTimer() {
if (timer == null) {
LOG.warn(prefix + " metrics system timer already stopped!");
return;
}
timer.cancel();
timer = null;
}
private synchronized void stopSources() {
for (Entry<String, MetricsSourceAdapter> entry : sources.entrySet()) {
MetricsSourceAdapter sa = entry.getValue();
LOG.debug("Stopping metrics source " + entry.getKey() + ": clreplaced=" + sa.source().getClreplaced());
sa.stop();
}
sysSource.stop();
sources.clear();
}
private synchronized void stopSinks() {
for (Entry<String, MetricsSinkAdapter> entry : sinks.entrySet()) {
MetricsSinkAdapter sa = entry.getValue();
LOG.debug("Stopping metrics sink " + entry.getKey() + ": clreplaced=" + sa.sink().getClreplaced());
sa.stop();
}
sinks.clear();
}
private synchronized void configure(String prefix) {
config = MetricsConfig.create(prefix);
configureSinks();
configureSources();
configureSystem();
}
private synchronized void configureSystem() {
injectedTags.add(Interns.tag(MsInfo.Hostname, getHostname()));
}
private synchronized void configureSinks() {
sinkConfigs = config.getInstanceConfigs(SINK_KEY);
long confPeriodMillis = 0;
for (Entry<String, MetricsConfig> entry : sinkConfigs.entrySet()) {
MetricsConfig conf = entry.getValue();
int sinkPeriod = conf.getInt(PERIOD_KEY, PERIOD_DEFAULT);
// Support configuring periodMillis for testing.
long sinkPeriodMillis = conf.getLong(PERIOD_MILLIS_KEY, sinkPeriod * 1000);
confPeriodMillis = confPeriodMillis == 0 ? sinkPeriodMillis : ArithmeticUtils.gcd(confPeriodMillis, sinkPeriodMillis);
String clsName = conf.getClreplacedName("");
// sink can be registered later on
if (clsName == null)
continue;
String sinkName = entry.getKey();
try {
MetricsSinkAdapter sa = newSink(sinkName, conf.getString(DESC_KEY, sinkName), conf);
sa.start();
sinks.put(sinkName, sa);
} catch (Exception e) {
LOG.warn("Error creating sink '" + sinkName + "'", e);
}
}
long periodSec = config.getInt(PERIOD_KEY, PERIOD_DEFAULT);
period = confPeriodMillis > 0 ? confPeriodMillis : config.getLong(PERIOD_MILLIS_KEY, periodSec * 1000);
}
static MetricsSinkAdapter newSink(String name, String desc, MetricsSink sink, MetricsConfig conf) {
return new MetricsSinkAdapter(name, desc, sink, conf.getString(CONTEXT_KEY), conf.getFilter(SOURCE_FILTER_KEY), conf.getFilter(RECORD_FILTER_KEY), conf.getFilter(METRIC_FILTER_KEY), conf.getInt(PERIOD_KEY, PERIOD_DEFAULT), conf.getInt(QUEUE_CAPACITY_KEY, QUEUE_CAPACITY_DEFAULT), conf.getInt(RETRY_DELAY_KEY, RETRY_DELAY_DEFAULT), conf.getFloat(RETRY_BACKOFF_KEY, RETRY_BACKOFF_DEFAULT), conf.getInt(RETRY_COUNT_KEY, RETRY_COUNT_DEFAULT));
}
static MetricsSinkAdapter newSink(String name, String desc, MetricsConfig conf) {
return newSink(name, desc, (MetricsSink) conf.getPlugin(""), conf);
}
private void configureSources() {
sourceFilter = config.getFilter(PREFIX_DEFAULT + SOURCE_FILTER_KEY);
sourceConfigs = config.getInstanceConfigs(SOURCE_KEY);
registerSystemSource();
}
private void clearConfigs() {
sinkConfigs.clear();
sourceConfigs.clear();
injectedTags.clear();
config = null;
}
static String getHostname() {
try {
return InetAddress.getLocalHost().getHostName();
} catch (Exception e) {
LOG.error("Error getting localhost name. Using 'localhost'...", e);
}
return "localhost";
}
private void registerSystemSource() {
MetricsConfig sysConf = sourceConfigs.get(MS_NAME);
sysSource = new MetricsSourceAdapter(prefix, MS_STATS_NAME, MS_STATS_DESC, MetricsAnnotations.makeSource(this), injectedTags, period, sysConf == null ? config.subset(SOURCE_KEY) : sysConf);
sysSource.start();
}
@Override
public synchronized void getMetrics(MetricsCollector builder, boolean all) {
MetricsRecordBuilder rb = builder.addRecord(MS_NAME).addGauge(MsInfo.NumActiveSources, sources.size()).addGauge(MsInfo.NumAllSources, allSources.size()).addGauge(MsInfo.NumActiveSinks, sinks.size()).addGauge(MsInfo.NumAllSinks, allSinks.size());
for (MetricsSinkAdapter sa : sinks.values()) {
sa.snapshot(rb, all);
}
registry.snapshot(rb, all);
}
private void initSystemMBean() {
checkNotNull(prefix, "prefix should not be null here!");
if (mbeanName == null) {
mbeanName = MBeans.register(prefix, MS_CONTROL_NAME, this);
}
}
@Override
public synchronized boolean shutdown() {
LOG.debug("refCount=" + refCount);
if (refCount <= 0) {
LOG.debug("Redundant shutdown", new Throwable());
// already shutdown
return true;
}
if (--refCount > 0)
return false;
if (monitoring) {
try {
stop();
} catch (Exception e) {
LOG.warn("Error stopping the metrics system", e);
}
}
allSources.clear();
allSinks.clear();
callbacks.clear();
namedCallbacks.clear();
if (mbeanName != null) {
MBeans.unregister(mbeanName);
mbeanName = null;
}
LOG.info(prefix + " metrics system shutdown complete.");
return true;
}
@Override
public MetricsSource getSource(String name) {
return allSources.get(name);
}
@VisibleForTesting
MetricsSourceAdapter getSourceAdapter(String name) {
return sources.get(name);
}
private InitMode initMode() {
LOG.debug("from system property: " + System.getProperty(MS_INIT_MODE_KEY));
LOG.debug("from environment variable: " + System.getenv(MS_INIT_MODE_KEY));
String m = System.getProperty(MS_INIT_MODE_KEY);
String m2 = m == null ? System.getenv(MS_INIT_MODE_KEY) : m;
return InitMode.valueOf(StringUtils.toUpperCase((m2 == null ? InitMode.NORMAL.name() : m2)));
}
}
15
View Complete Implementation : NameNodeMetrics.java
Copyright Apache License 2.0
Author : aliyun-beta
Copyright Apache License 2.0
Author : aliyun-beta
/**
* This clreplaced is for maintaining the various NameNode activity statistics
* and publishing them through the metrics interfaces.
*/
@Metrics(name = "NameNodeActivity", about = "NameNode metrics", context = "dfs")
public clreplaced NameNodeMetrics {
final MetricsRegistry registry = new MetricsRegistry("namenode");
@Metric
MutableCounterLong createFileOps;
@Metric
MutableCounterLong filesCreated;
@Metric
MutableCounterLong filesAppended;
@Metric
MutableCounterLong getBlockLocations;
@Metric
MutableCounterLong filesRenamed;
@Metric
MutableCounterLong filesTruncated;
@Metric
MutableCounterLong getListingOps;
@Metric
MutableCounterLong deleteFileOps;
@Metric("Number of files/dirs deleted by delete or rename operations")
MutableCounterLong filesDeleted;
@Metric
MutableCounterLong fileInfoOps;
@Metric
MutableCounterLong addBlockOps;
@Metric
MutableCounterLong getAdditionalDatanodeOps;
@Metric
MutableCounterLong createSymlinkOps;
@Metric
MutableCounterLong getLinkTargetOps;
@Metric
MutableCounterLong filesInGetListingOps;
@Metric("Number of allowSnapshot operations")
MutableCounterLong allowSnapshotOps;
@Metric("Number of disallowSnapshot operations")
MutableCounterLong disallowSnapshotOps;
@Metric("Number of createSnapshot operations")
MutableCounterLong createSnapshotOps;
@Metric("Number of deleteSnapshot operations")
MutableCounterLong deleteSnapshotOps;
@Metric("Number of renameSnapshot operations")
MutableCounterLong renameSnapshotOps;
@Metric("Number of listSnapshottableDirectory operations")
MutableCounterLong listSnapshottableDirOps;
@Metric("Number of snapshotDiffReport operations")
MutableCounterLong snapshotDiffReportOps;
@Metric("Number of blockReceivedAndDeleted calls")
MutableCounterLong blockReceivedAndDeletedOps;
@Metric("Number of blockReports from individual storages")
MutableCounterLong storageBlockReportOps;
@Metric("Number of blockReports and blockReceivedAndDeleted queued")
MutableGaugeInt blockOpsQueued;
@Metric("Number of blockReports and blockReceivedAndDeleted batch processed")
MutableCounterLong blockOpsBatched;
@Metric("Number of file system operations")
public long totalFileOps() {
return getBlockLocations.value() + createFileOps.value() + filesAppended.value() + addBlockOps.value() + getAdditionalDatanodeOps.value() + filesRenamed.value() + filesTruncated.value() + deleteFileOps.value() + getListingOps.value() + fileInfoOps.value() + getLinkTargetOps.value() + createSnapshotOps.value() + deleteSnapshotOps.value() + allowSnapshotOps.value() + disallowSnapshotOps.value() + renameSnapshotOps.value() + listSnapshottableDirOps.value() + createSymlinkOps.value() + snapshotDiffReportOps.value();
}
@Metric("Journal transactions")
MutableRate transactions;
@Metric("Journal syncs")
MutableRate syncs;
final MutableQuantiles[] syncsQuantiles;
@Metric("Journal transactions batched in sync")
MutableCounterLong transactionsBatchedInSync;
@Metric("Block report")
MutableRate blockReport;
final MutableQuantiles[] blockReportQuantiles;
@Metric("Cache report")
MutableRate cacheReport;
final MutableQuantiles[] cacheReportQuantiles;
@Metric("Duration in SafeMode at startup in msec")
MutableGaugeInt safeModeTime;
@Metric("Time loading FS Image at startup in msec")
MutableGaugeInt fsImageLoadTime;
@Metric("GetImageServlet getEdit")
MutableRate getEdit;
@Metric("GetImageServlet getImage")
MutableRate getImage;
@Metric("GetImageServlet putImage")
MutableRate putImage;
JvmMetrics jvmMetrics = null;
NameNodeMetrics(String processName, String sessionId, int[] intervals, final JvmMetrics jvmMetrics) {
this.jvmMetrics = jvmMetrics;
registry.tag(ProcessName, processName).tag(SessionId, sessionId);
final int len = intervals.length;
syncsQuantiles = new MutableQuantiles[len];
blockReportQuantiles = new MutableQuantiles[len];
cacheReportQuantiles = new MutableQuantiles[len];
for (int i = 0; i < len; i++) {
int interval = intervals[i];
syncsQuantiles[i] = registry.newQuantiles("syncs" + interval + "s", "Journal syncs", "ops", "latency", interval);
blockReportQuantiles[i] = registry.newQuantiles("blockReport" + interval + "s", "Block report", "ops", "latency", interval);
cacheReportQuantiles[i] = registry.newQuantiles("cacheReport" + interval + "s", "Cache report", "ops", "latency", interval);
}
}
public static NameNodeMetrics create(Configuration conf, NamenodeRole r) {
String sessionId = conf.get(DFSConfigKeys.DFS_METRICS_SESSION_ID_KEY);
String processName = r.toString();
MetricsSystem ms = DefaultMetricsSystem.instance();
JvmMetrics jm = JvmMetrics.create(processName, sessionId, ms);
// Percentile measurement is off by default, by watching no intervals
int[] intervals = conf.getInts(DFSConfigKeys.DFS_METRICS_PERCENTILES_INTERVALS_KEY);
return ms.register(new NameNodeMetrics(processName, sessionId, intervals, jm));
}
public JvmMetrics getJvmMetrics() {
return jvmMetrics;
}
public void shutdown() {
DefaultMetricsSystem.shutdown();
}
public void incrGetBlockLocations() {
getBlockLocations.incr();
}
public void incrFilesCreated() {
filesCreated.incr();
}
public void incrCreateFileOps() {
createFileOps.incr();
}
public void incrFilesAppended() {
filesAppended.incr();
}
public void incrAddBlockOps() {
addBlockOps.incr();
}
public void incrGetAdditionalDatanodeOps() {
getAdditionalDatanodeOps.incr();
}
public void incrFilesRenamed() {
filesRenamed.incr();
}
public void incrFilesTruncated() {
filesTruncated.incr();
}
public void incrFilesDeleted(long delta) {
filesDeleted.incr(delta);
}
public void incrDeleteFileOps() {
deleteFileOps.incr();
}
public void incrGetListingOps() {
getListingOps.incr();
}
public void incrFilesInGetListingOps(int delta) {
filesInGetListingOps.incr(delta);
}
public void incrFileInfoOps() {
fileInfoOps.incr();
}
public void incrCreateSymlinkOps() {
createSymlinkOps.incr();
}
public void incrGetLinkTargetOps() {
getLinkTargetOps.incr();
}
public void incrAllowSnapshotOps() {
allowSnapshotOps.incr();
}
public void incrDisAllowSnapshotOps() {
disallowSnapshotOps.incr();
}
public void incrCreateSnapshotOps() {
createSnapshotOps.incr();
}
public void incrDeleteSnapshotOps() {
deleteSnapshotOps.incr();
}
public void incrRenameSnapshotOps() {
renameSnapshotOps.incr();
}
public void incrListSnapshottableDirOps() {
listSnapshottableDirOps.incr();
}
public void incrSnapshotDiffReportOps() {
snapshotDiffReportOps.incr();
}
public void incrBlockReceivedAndDeletedOps() {
blockReceivedAndDeletedOps.incr();
}
public void incrStorageBlockReportOps() {
storageBlockReportOps.incr();
}
public void setBlockOpsQueued(int size) {
blockOpsQueued.set(size);
}
public void addBlockOpsBatched(int count) {
blockOpsBatched.incr(count);
}
public void addTransaction(long latency) {
transactions.add(latency);
}
public void incrTransactionsBatchedInSync() {
transactionsBatchedInSync.incr();
}
public void addSync(long elapsed) {
syncs.add(elapsed);
for (MutableQuantiles q : syncsQuantiles) {
q.add(elapsed);
}
}
public void setFsImageLoadTime(long elapsed) {
fsImageLoadTime.set((int) elapsed);
}
public void addBlockReport(long latency) {
blockReport.add(latency);
for (MutableQuantiles q : blockReportQuantiles) {
q.add(latency);
}
}
public void addCacheBlockReport(long latency) {
cacheReport.add(latency);
for (MutableQuantiles q : cacheReportQuantiles) {
q.add(latency);
}
}
public void setSafeModeTime(long elapsed) {
safeModeTime.set((int) elapsed);
}
public void addGetEdit(long latency) {
getEdit.add(latency);
}
public void addGetImage(long latency) {
getImage.add(latency);
}
public void addPutImage(long latency) {
putImage.add(latency);
}
}
15
View Complete Implementation : RpcMetrics.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
/**
* This clreplaced is for maintaining the various RPC statistics
* and publishing them through the metrics interfaces.
*/
@InterfaceAudience.Private
@Metrics(about = "Aggregate RPC metrics", context = "rpc")
public clreplaced RpcMetrics {
static final Logger LOG = LoggerFactory.getLogger(RpcMetrics.clreplaced);
final Server server;
final MetricsRegistry registry;
final String name;
final boolean rpcQuantileEnable;
/**
* The time unit used when storing/accessing time durations.
*/
public final static TimeUnit TIMEUNIT = TimeUnit.MILLISECONDS;
RpcMetrics(Server server, Configuration conf) {
String port = String.valueOf(server.getListenerAddress().getPort());
name = "RpcActivityForPort" + port;
this.server = server;
registry = new MetricsRegistry("rpc").tag("port", "RPC port", port).tag("serverName", "Name of the RPC server", server.getServerName());
int[] intervals = conf.getInts(CommonConfigurationKeys.RPC_METRICS_PERCENTILES_INTERVALS_KEY);
rpcQuantileEnable = (intervals.length > 0) && conf.getBoolean(CommonConfigurationKeys.RPC_METRICS_QUANTILE_ENABLE, CommonConfigurationKeys.RPC_METRICS_QUANTILE_ENABLE_DEFAULT);
if (rpcQuantileEnable) {
rpcQueueTimeQuantiles = new MutableQuantiles[intervals.length];
rpcLockWaitTimeQuantiles = new MutableQuantiles[intervals.length];
rpcProcessingTimeQuantiles = new MutableQuantiles[intervals.length];
deferredRpcProcessingTimeQuantiles = new MutableQuantiles[intervals.length];
for (int i = 0; i < intervals.length; i++) {
int interval = intervals[i];
rpcQueueTimeQuantiles[i] = registry.newQuantiles("rpcQueueTime" + interval + "s", "rpc queue time in " + TIMEUNIT, "ops", "latency", interval);
rpcLockWaitTimeQuantiles[i] = registry.newQuantiles("rpcLockWaitTime" + interval + "s", "rpc lock wait time in " + TIMEUNIT, "ops", "latency", interval);
rpcProcessingTimeQuantiles[i] = registry.newQuantiles("rpcProcessingTime" + interval + "s", "rpc processing time in " + TIMEUNIT, "ops", "latency", interval);
deferredRpcProcessingTimeQuantiles[i] = registry.newQuantiles("deferredRpcProcessingTime" + interval + "s", "deferred rpc processing time in " + TIMEUNIT, "ops", "latency", interval);
}
}
LOG.debug("Initialized " + registry);
}
public String name() {
return name;
}
public static RpcMetrics create(Server server, Configuration conf) {
RpcMetrics m = new RpcMetrics(server, conf);
return DefaultMetricsSystem.instance().register(m.name, null, m);
}
@Metric("Number of received bytes")
MutableCounterLong receivedBytes;
@Metric("Number of sent bytes")
MutableCounterLong sentBytes;
@Metric("Queue time")
MutableRate rpcQueueTime;
MutableQuantiles[] rpcQueueTimeQuantiles;
@Metric("Lock wait time")
MutableRate rpcLockWaitTime;
MutableQuantiles[] rpcLockWaitTimeQuantiles;
@Metric("Processing time")
MutableRate rpcProcessingTime;
MutableQuantiles[] rpcProcessingTimeQuantiles;
@Metric("Deferred Processing time")
MutableRate deferredRpcProcessingTime;
MutableQuantiles[] deferredRpcProcessingTimeQuantiles;
@Metric("Number of authentication failures")
MutableCounterLong rpcAuthenticationFailures;
@Metric("Number of authentication successes")
MutableCounterLong rpcAuthenticationSuccesses;
@Metric("Number of authorization failures")
MutableCounterLong rpcAuthorizationFailures;
@Metric("Number of authorization successes")
MutableCounterLong rpcAuthorizationSuccesses;
@Metric("Number of client backoff requests")
MutableCounterLong rpcClientBackoff;
@Metric("Number of Slow RPC calls")
MutableCounterLong rpcSlowCalls;
@Metric("Number of open connections")
public int numOpenConnections() {
return server.getNumOpenConnections();
}
@Metric("Number of open connections per user")
public String numOpenConnectionsPerUser() {
return server.getNumOpenConnectionsPerUser();
}
@Metric("Length of the call queue")
public int callQueueLength() {
return server.getCallQueueLen();
}
@Metric("Number of dropped connections")
public long numDroppedConnections() {
return server.getNumDroppedConnections();
}
// Public instrumentation methods that could be extracted to an
// abstract clreplaced if we decide to do custom instrumentation clreplacedes a la
// JobTrackerInstrumentation. The methods with //@Override comment are
// candidates for abstract methods in a abstract instrumentation clreplaced.
/**
* One authentication failure event
*/
// @Override
public void incrAuthenticationFailures() {
rpcAuthenticationFailures.incr();
}
/**
* One authentication success event
*/
// @Override
public void incrAuthenticationSuccesses() {
rpcAuthenticationSuccesses.incr();
}
/**
* One authorization success event
*/
// @Override
public void incrAuthorizationSuccesses() {
rpcAuthorizationSuccesses.incr();
}
/**
* One authorization failure event
*/
// @Override
public void incrAuthorizationFailures() {
rpcAuthorizationFailures.incr();
}
/**
* Shutdown the instrumentation for the process
*/
// @Override
public void shutdown() {
DefaultMetricsSystem.instance().unregisterSource(name);
}
/**
* Increment sent bytes by count
* @param count to increment
*/
// @Override
public void incrSentBytes(int count) {
sentBytes.incr(count);
}
/**
* Increment received bytes by count
* @param count to increment
*/
// @Override
public void incrReceivedBytes(int count) {
receivedBytes.incr(count);
}
/**
* Add an RPC queue time sample
* @param qTime the queue time
*/
public void addRpcQueueTime(long qTime) {
rpcQueueTime.add(qTime);
if (rpcQuantileEnable) {
for (MutableQuantiles q : rpcQueueTimeQuantiles) {
q.add(qTime);
}
}
}
public void addRpcLockWaitTime(long waitTime) {
rpcLockWaitTime.add(waitTime);
if (rpcQuantileEnable) {
for (MutableQuantiles q : rpcLockWaitTimeQuantiles) {
q.add(waitTime);
}
}
}
/**
* Add an RPC processing time sample
* @param processingTime the processing time
*/
public void addRpcProcessingTime(long processingTime) {
rpcProcessingTime.add(processingTime);
if (rpcQuantileEnable) {
for (MutableQuantiles q : rpcProcessingTimeQuantiles) {
q.add(processingTime);
}
}
}
public void addDeferredRpcProcessingTime(long processingTime) {
deferredRpcProcessingTime.add(processingTime);
if (rpcQuantileEnable) {
for (MutableQuantiles q : deferredRpcProcessingTimeQuantiles) {
q.add(processingTime);
}
}
}
/**
* One client backoff event
*/
// @Override
public void incrClientBackoff() {
rpcClientBackoff.incr();
}
/**
* Increments the Slow RPC counter.
*/
public void incrSlowRpc() {
rpcSlowCalls.incr();
}
/**
* Returns a MutableRate Counter.
* @return Mutable Rate
*/
public MutableRate getRpcProcessingTime() {
return rpcProcessingTime;
}
/**
* Returns the number of samples that we have seen so far.
* @return long
*/
public long getProcessingSampleCount() {
return rpcProcessingTime.lastStat().numSamples();
}
/**
* Returns mean of RPC Processing Times.
* @return double
*/
public double getProcessingMean() {
return rpcProcessingTime.lastStat().mean();
}
/**
* Return Standard Deviation of the Processing Time.
* @return double
*/
public double getProcessingStdDev() {
return rpcProcessingTime.lastStat().stddev();
}
/**
* Returns the number of slow calls.
* @return long
*/
public long getRpcSlowCalls() {
return rpcSlowCalls.value();
}
public MutableRate getDeferredRpcProcessingTime() {
return deferredRpcProcessingTime;
}
public long getDeferredRpcProcessingSampleCount() {
return deferredRpcProcessingTime.lastStat().numSamples();
}
public double getDeferredRpcProcessingMean() {
return deferredRpcProcessingTime.lastStat().mean();
}
public double getDeferredRpcProcessingStdDev() {
return deferredRpcProcessingTime.lastStat().stddev();
}
@VisibleForTesting
public MetricsTag getTag(String tagName) {
return registry.getTag(tagName);
}
}
15
View Complete Implementation : ITestS3AMetrics.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
@Test
public void testMetricsRegister() throws IOException, InterruptedException {
S3AFileSystem fs = getFileSystem();
Path dest = path("testMetricsRegister");
ContractTestUtils.touch(fs, dest);
MutableCounterLong fileCreated = (MutableCounterLong) fs.getInstrumentation().getRegistry().get(Statistic.FILES_CREATED.getSymbol());
replacedertEquals("Metrics system should report single file created event", 1, fileCreated.value());
}
15
View Complete Implementation : ITestS3AMetrics.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
@Test
public void testStreamStatistics() throws IOException {
S3AFileSystem fs = getFileSystem();
Path file = path("testStreamStatistics");
byte[] data = "abcdefghijklmnopqrstuvwxyz".getBytes();
ContractTestUtils.createFile(fs, file, false, data);
try (InputStream inputStream = fs.open(file)) {
while (inputStream.read(data) != -1) {
LOG.debug("Read batch of data from input stream...");
}
}
MutableCounterLong read = (MutableCounterLong) fs.getInstrumentation().getRegistry().get(Statistic.STREAM_SEEK_BYTES_READ.getSymbol());
replacedertEquals("Stream statistics were not merged", 26, read.value());
}
15
View Complete Implementation : TestEntityGroupFSTimelineStore.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
@Test
public void testSummaryRead() throws Exception {
// Load data
EnreplacedyGroupFSTimelineStore.AppLogs appLogs = store.new AppLogs(mainTestAppId, mainTestAppDirPath, AppState.COMPLETED);
MutableCounterLong summaryLogEnreplacedyRead = store.metrics.getGetEnreplacedyToSummaryOps();
long numEnreplacedyReadBefore = summaryLogEnreplacedyRead.value();
TimelineDataManager tdm = PluginStoreTestUtils.getTdmWithStore(config, store);
appLogs.scanForLogs();
appLogs.parseSummaryLogs(tdm);
// Verify single enreplacedy read
PluginStoreTestUtils.verifyTestEnreplacedies(tdm);
// Verify multiple enreplacedies read
TimelineEnreplacedies enreplacedies = tdm.getEnreplacedies("type_1", null, null, null, null, null, null, null, EnumSet.allOf(TimelineReader.Field.clreplaced), UserGroupInformation.getLoginUser());
replacedertThat(enreplacedies.getEnreplacedies()).hreplacedize(1);
for (TimelineEnreplacedy enreplacedy : enreplacedies.getEnreplacedies()) {
replacedertEquals((Long) 123L, enreplacedy.getStartTime());
}
// Verify metrics
replacedertEquals(numEnreplacedyReadBefore + 5L, summaryLogEnreplacedyRead.value());
}
15
View Complete Implementation : NameNodeMetrics.java
Copyright Apache License 2.0
Author : apache
Copyright Apache License 2.0
Author : apache
/**
* This clreplaced is for maintaining the various NameNode activity statistics
* and publishing them through the metrics interfaces.
*/
@Metrics(name = "NameNodeActivity", about = "NameNode metrics", context = "dfs")
public clreplaced NameNodeMetrics {
final MetricsRegistry registry = new MetricsRegistry("namenode");
@Metric
MutableCounterLong createFileOps;
@Metric
MutableCounterLong filesCreated;
@Metric
MutableCounterLong filesAppended;
@Metric
MutableCounterLong getBlockLocations;
@Metric
MutableCounterLong filesRenamed;
@Metric
MutableCounterLong getListingOps;
@Metric
MutableCounterLong deleteFileOps;
@Metric("Number of files/dirs deleted by delete or rename operations")
MutableCounterLong filesDeleted;
@Metric
MutableCounterLong fileInfoOps;
@Metric
MutableCounterLong addBlockOps;
@Metric
MutableCounterLong getAdditionalDatanodeOps;
@Metric
MutableCounterLong createSymlinkOps;
@Metric
MutableCounterLong getLinkTargetOps;
@Metric
MutableCounterLong filesInGetListingOps;
@Metric("Journal transactions")
MutableRate transactions;
@Metric("Journal syncs")
MutableRate syncs;
@Metric("Journal transactions batched in sync")
MutableCounterLong transactionsBatchedInSync;
@Metric("Block report")
MutableRate blockReport;
@Metric("Duration in SafeMode at startup")
MutableGaugeInt safeModeTime;
@Metric("Time loading FS Image at startup")
MutableGaugeInt fsImageLoadTime;
NameNodeMetrics(String processName, String sessionId) {
registry.tag(ProcessName, processName).tag(SessionId, sessionId);
}
public static NameNodeMetrics create(Configuration conf, NamenodeRole r) {
String sessionId = conf.get(DFSConfigKeys.DFS_METRICS_SESSION_ID_KEY);
String processName = r.toString();
MetricsSystem ms = DefaultMetricsSystem.instance();
JvmMetrics.create(processName, sessionId, ms);
return ms.register(new NameNodeMetrics(processName, sessionId));
}
public void shutdown() {
DefaultMetricsSystem.shutdown();
}
public void incrGetBlockLocations() {
getBlockLocations.incr();
}
public void incrFilesCreated() {
filesCreated.incr();
}
public void incrCreateFileOps() {
createFileOps.incr();
}
public void incrFilesAppended() {
filesAppended.incr();
}
public void incrAddBlockOps() {
addBlockOps.incr();
}
public void incrGetAdditionalDatanodeOps() {
getAdditionalDatanodeOps.incr();
}
public void incrFilesRenamed() {
filesRenamed.incr();
}
public void incrFilesDeleted(int delta) {
filesDeleted.incr(delta);
}
public void incrDeleteFileOps() {
deleteFileOps.incr();
}
public void incrGetListingOps() {
getListingOps.incr();
}
public void incrFilesInGetListingOps(int delta) {
filesInGetListingOps.incr(delta);
}
public void incrFileInfoOps() {
fileInfoOps.incr();
}
public void incrCreateSymlinkOps() {
createSymlinkOps.incr();
}
public void incrGetLinkTargetOps() {
getLinkTargetOps.incr();
}
public void addTransaction(long latency) {
transactions.add(latency);
}
public void incrTransactionsBatchedInSync() {
transactionsBatchedInSync.incr();
}
public void addSync(long elapsed) {
syncs.add(elapsed);
}
public void setFsImageLoadTime(long elapsed) {
fsImageLoadTime.set((int) elapsed);
}
public void addBlockReport(long latency) {
blockReport.add(latency);
}
public void setSafeModeTime(long elapsed) {
safeModeTime.set((int) elapsed);
}
}
15
View Complete Implementation : MetricsRegionSourceImpl.java
Copyright Apache License 2.0
Author : fengchen8086
Copyright Apache License 2.0
Author : fengchen8086
@InterfaceAudience.Private
public clreplaced MetricsRegionSourceImpl implements MetricsRegionSource {
private static final Log LOG = LogFactory.getLog(MetricsRegionSourceImpl.clreplaced);
private AtomicBoolean closed = new AtomicBoolean(false);
// Non-final so that we can null out the wrapper
// This is just paranoia. We really really don't want to
// leak a whole region by way of keeping the
// regionWrapper around too long.
private MetricsRegionWrapper regionWrapper;
private final MetricsRegionAggregateSourceImpl agg;
private final DynamicMetricsRegistry registry;
private final String regionNamePrefix;
private final String regionPutKey;
private final String regionDeleteKey;
private final String regionGetKey;
private final String regionIncrementKey;
private final String regionAppendKey;
private final String regionScanNextKey;
private final MutableCounterLong regionPut;
private final MutableCounterLong regionDelete;
private final MutableCounterLong regionIncrement;
private final MutableCounterLong regionAppend;
private final MutableHistogram regionGet;
private final MutableHistogram regionScanNext;
private final int hashCode;
public MetricsRegionSourceImpl(MetricsRegionWrapper regionWrapper, MetricsRegionAggregateSourceImpl aggregate) {
this.regionWrapper = regionWrapper;
agg = aggregate;
agg.register(this);
LOG.debug("Creating new MetricsRegionSourceImpl for table " + regionWrapper.getTableName() + " " + regionWrapper.getRegionName());
registry = agg.getMetricsRegistry();
regionNamePrefix = "Namespace_" + regionWrapper.getNamespace() + "_table_" + regionWrapper.getTableName() + "_region_" + regionWrapper.getRegionName() + "_metric_";
String suffix = "Count";
regionPutKey = regionNamePrefix + MetricsRegionServerSource.MUTATE_KEY + suffix;
regionPut = registry.getLongCounter(regionPutKey, 0L);
regionDeleteKey = regionNamePrefix + MetricsRegionServerSource.DELETE_KEY + suffix;
regionDelete = registry.getLongCounter(regionDeleteKey, 0L);
regionIncrementKey = regionNamePrefix + MetricsRegionServerSource.INCREMENT_KEY + suffix;
regionIncrement = registry.getLongCounter(regionIncrementKey, 0L);
regionAppendKey = regionNamePrefix + MetricsRegionServerSource.APPEND_KEY + suffix;
regionAppend = registry.getLongCounter(regionAppendKey, 0L);
regionGetKey = regionNamePrefix + MetricsRegionServerSource.GET_KEY;
regionGet = registry.newTimeHistogram(regionGetKey);
regionScanNextKey = regionNamePrefix + MetricsRegionServerSource.SCAN_NEXT_KEY;
regionScanNext = registry.newTimeHistogram(regionScanNextKey);
hashCode = regionWrapper.getRegionHashCode();
}
@Override
public void close() {
boolean wasClosed = closed.getAndSet(true);
// Has someone else already closed this for us?
if (wasClosed) {
return;
}
// Before removing the metrics remove this region from the aggregate region bean.
// This should mean that it's unlikely that snapshot and close happen at the same time.
agg.deregister(this);
// While it's un-likely that snapshot and close happen at the same time it's still possible.
// So grab the lock to ensure that all calls to snapshot are done before we remove the metrics
synchronized (this) {
if (LOG.isTraceEnabled()) {
LOG.trace("Removing region Metrics: " + regionWrapper.getRegionName());
}
registry.removeMetric(regionPutKey);
registry.removeMetric(regionDeleteKey);
registry.removeMetric(regionIncrementKey);
registry.removeMetric(regionAppendKey);
registry.removeMetric(regionGetKey);
registry.removeMetric(regionScanNextKey);
registry.removeHistogramMetrics(regionGetKey);
registry.removeHistogramMetrics(regionScanNextKey);
regionWrapper = null;
}
}
@Override
public void updatePut() {
regionPut.incr();
}
@Override
public void updateDelete() {
regionDelete.incr();
}
@Override
public void updateGet(long getSize) {
regionGet.add(getSize);
}
@Override
public void updateScan(long scanSize) {
regionScanNext.add(scanSize);
}
@Override
public void updateIncrement() {
regionIncrement.incr();
}
@Override
public void updateAppend() {
regionAppend.incr();
}
@Override
public MetricsRegionAggregateSource getAggregateSource() {
return agg;
}
@Override
public int compareTo(MetricsRegionSource source) {
if (!(source instanceof MetricsRegionSourceImpl)) {
return -1;
}
MetricsRegionSourceImpl impl = (MetricsRegionSourceImpl) source;
if (impl == null) {
return -1;
}
return Long.compare(hashCode, impl.hashCode);
}
void snapshot(MetricsRecordBuilder mrb, boolean ignored) {
// If there is a close that started be double extra sure
// that we're not getting any locks and not putting data
// into the metrics that should be removed. So early out
// before even getting the lock.
if (closed.get()) {
return;
}
// Grab the read
// This ensures that removes of the metrics
// can't happen while we are putting them back in.
synchronized (this) {
// It's possible that a close happened between checking
// the closed variable and getting the lock.
if (closed.get()) {
return;
}
mrb.addGauge(Interns.info(regionNamePrefix + MetricsRegionServerSource.STORE_COUNT, MetricsRegionServerSource.STORE_COUNT_DESC), this.regionWrapper.getNumStores());
mrb.addGauge(Interns.info(regionNamePrefix + MetricsRegionServerSource.STOREFILE_COUNT, MetricsRegionServerSource.STOREFILE_COUNT_DESC), this.regionWrapper.getNumStoreFiles());
mrb.addGauge(Interns.info(regionNamePrefix + MetricsRegionServerSource.MEMSTORE_SIZE, MetricsRegionServerSource.MEMSTORE_SIZE_DESC), this.regionWrapper.getMemstoreSize());
mrb.addGauge(Interns.info(regionNamePrefix + MetricsRegionServerSource.STOREFILE_SIZE, MetricsRegionServerSource.STOREFILE_SIZE_DESC), this.regionWrapper.getStoreFileSize());
mrb.addCounter(Interns.info(regionNamePrefix + MetricsRegionSource.COMPACTIONS_COMPLETED_COUNT, MetricsRegionSource.COMPACTIONS_COMPLETED_DESC), this.regionWrapper.getNumCompactionsCompleted());
mrb.addCounter(Interns.info(regionNamePrefix + MetricsRegionSource.NUM_BYTES_COMPACTED_COUNT, MetricsRegionSource.NUM_BYTES_COMPACTED_DESC), this.regionWrapper.getNumBytesCompacted());
mrb.addCounter(Interns.info(regionNamePrefix + MetricsRegionSource.NUM_FILES_COMPACTED_COUNT, MetricsRegionSource.NUM_FILES_COMPACTED_DESC), this.regionWrapper.getNumFilesCompacted());
mrb.addCounter(Interns.info(regionNamePrefix + MetricsRegionServerSource.READ_REQUEST_COUNT, MetricsRegionServerSource.READ_REQUEST_COUNT_DESC), this.regionWrapper.getReadRequestCount());
mrb.addCounter(Interns.info(regionNamePrefix + MetricsRegionServerSource.WRITE_REQUEST_COUNT, MetricsRegionServerSource.WRITE_REQUEST_COUNT_DESC), this.regionWrapper.getWriteRequestCount());
mrb.addCounter(Interns.info(regionNamePrefix + MetricsRegionSource.REPLICA_ID, MetricsRegionSource.REPLICA_ID_DESC), this.regionWrapper.getReplicaId());
}
}
@Override
public int hashCode() {
return hashCode;
}
@Override
public boolean equals(Object obj) {
return obj == this || (obj instanceof MetricsRegionSourceImpl && compareTo((MetricsRegionSourceImpl) obj) == 0);
}
}
15
View Complete Implementation : QueueMetrics.java
Copyright Apache License 2.0
Author : yncxcw
Copyright Apache License 2.0
Author : yncxcw
@InterfaceAudience.Private
@Metrics(context = "yarn")
public clreplaced QueueMetrics implements MetricsSource {
@Metric("# of apps submitted")
MutableCounterInt appsSubmitted;
@Metric("# of running apps")
MutableGaugeInt appsRunning;
@Metric("# of pending apps")
MutableGaugeInt appsPending;
@Metric("# of apps completed")
MutableCounterInt appsCompleted;
@Metric("# of apps killed")
MutableCounterInt appsKilled;
@Metric("# of apps failed")
MutableCounterInt appsFailed;
@Metric("Allocated memory in MB")
MutableGaugeInt allocatedMB;
@Metric("Allocated CPU in virtual cores")
MutableGaugeInt allocatedVCores;
@Metric("# of allocated containers")
MutableGaugeInt allocatedContainers;
@Metric("Aggregate # of allocated containers")
MutableCounterLong aggregateContainersAllocated;
@Metric("Aggregate # of released containers")
MutableCounterLong aggregateContainersReleased;
@Metric("Available memory in MB")
MutableGaugeInt availableMB;
@Metric("Available CPU in virtual cores")
MutableGaugeInt availableVCores;
@Metric("Pending memory allocation in MB")
MutableGaugeInt pendingMB;
@Metric("Pending CPU allocation in virtual cores")
MutableGaugeInt pendingVCores;
@Metric("# of pending containers")
MutableGaugeInt pendingContainers;
@Metric("# of reserved memory in MB")
MutableGaugeInt reservedMB;
@Metric("Reserved CPU in virtual cores")
MutableGaugeInt reservedVCores;
@Metric("# of reserved containers")
MutableGaugeInt reservedContainers;
@Metric("# of active users")
MutableGaugeInt activeUsers;
@Metric("# of active applications")
MutableGaugeInt activeApplications;
private final MutableGaugeInt[] runningTime;
private TimeBucketMetrics<ApplicationId> runBuckets;
static final Logger LOG = LoggerFactory.getLogger(QueueMetrics.clreplaced);
static final MetricsInfo RECORD_INFO = info("QueueMetrics", "Metrics for the resource scheduler");
protected static final MetricsInfo QUEUE_INFO = info("Queue", "Metrics by queue");
static final MetricsInfo USER_INFO = info("User", "Metrics by user");
static final Splitter Q_SPLITTER = Splitter.on('.').omitEmptyStrings().trimResults();
final MetricsRegistry registry;
final String queueName;
final QueueMetrics parent;
final MetricsSystem metricsSystem;
private final Map<String, QueueMetrics> users;
private final Configuration conf;
protected QueueMetrics(MetricsSystem ms, String queueName, Queue parent, boolean enableUserMetrics, Configuration conf) {
registry = new MetricsRegistry(RECORD_INFO);
this.queueName = queueName;
this.parent = parent != null ? parent.getMetrics() : null;
this.users = enableUserMetrics ? new HashMap<String, QueueMetrics>() : null;
metricsSystem = ms;
this.conf = conf;
runningTime = buildBuckets(conf);
}
protected QueueMetrics tag(MetricsInfo info, String value) {
registry.tag(info, value);
return this;
}
protected static StringBuilder sourceName(String queueName) {
StringBuilder sb = new StringBuilder(RECORD_INFO.name());
int i = 0;
for (String node : Q_SPLITTER.split(queueName)) {
sb.append(",q").append(i++).append('=').append(node);
}
return sb;
}
public synchronized static QueueMetrics forQueue(String queueName, Queue parent, boolean enableUserMetrics, Configuration conf) {
return forQueue(DefaultMetricsSystem.instance(), queueName, parent, enableUserMetrics, conf);
}
/**
* Helper method to clear cache.
*/
@Private
public synchronized static void clearQueueMetrics() {
queueMetrics.clear();
}
/**
* Simple metrics cache to help prevent re-registrations.
*/
protected final static Map<String, QueueMetrics> queueMetrics = new HashMap<String, QueueMetrics>();
public synchronized static QueueMetrics forQueue(MetricsSystem ms, String queueName, Queue parent, boolean enableUserMetrics, Configuration conf) {
QueueMetrics metrics = queueMetrics.get(queueName);
if (metrics == null) {
metrics = new QueueMetrics(ms, queueName, parent, enableUserMetrics, conf).tag(QUEUE_INFO, queueName);
// Register with the MetricsSystems
if (ms != null) {
metrics = ms.register(sourceName(queueName).toString(), "Metrics for queue: " + queueName, metrics);
}
queueMetrics.put(queueName, metrics);
}
return metrics;
}
public synchronized QueueMetrics getUserMetrics(String userName) {
if (users == null) {
return null;
}
QueueMetrics metrics = users.get(userName);
if (metrics == null) {
metrics = new QueueMetrics(metricsSystem, queueName, null, false, conf);
users.put(userName, metrics);
metricsSystem.register(sourceName(queueName).append(",user=").append(userName).toString(), "Metrics for user '" + userName + "' in queue '" + queueName + "'", metrics.tag(QUEUE_INFO, queueName).tag(USER_INFO, userName));
}
return metrics;
}
private ArrayList<Integer> parseInts(String value) {
ArrayList<Integer> result = new ArrayList<Integer>();
for (String s : value.split(",")) {
result.add(Integer.parseInt(s.trim()));
}
return result;
}
private MutableGaugeInt[] buildBuckets(Configuration conf) {
ArrayList<Integer> buckets = parseInts(conf.get(YarnConfiguration.RM_METRICS_RUNTIME_BUCKETS, YarnConfiguration.DEFAULT_RM_METRICS_RUNTIME_BUCKETS));
MutableGaugeInt[] result = new MutableGaugeInt[buckets.size() + 1];
result[0] = registry.newGauge("running_0", "", 0);
long[] cuts = new long[buckets.size()];
for (int i = 0; i < buckets.size(); ++i) {
result[i + 1] = registry.newGauge("running_" + buckets.get(i), "", 0);
// covert from min to ms
cuts[i] = buckets.get(i) * 1000L * 60;
}
this.runBuckets = new TimeBucketMetrics<ApplicationId>(cuts);
return result;
}
private void updateRunningTime() {
int[] counts = runBuckets.getBucketCounts(System.currentTimeMillis());
for (int i = 0; i < counts.length; ++i) {
runningTime[i].set(counts[i]);
}
}
public void getMetrics(MetricsCollector collector, boolean all) {
updateRunningTime();
registry.snapshot(collector.addRecord(registry.info()), all);
}
public void submitApp(String user) {
appsSubmitted.incr();
QueueMetrics userMetrics = getUserMetrics(user);
if (userMetrics != null) {
userMetrics.submitApp(user);
}
if (parent != null) {
parent.submitApp(user);
}
}
public void submitAppAttempt(String user) {
appsPending.incr();
QueueMetrics userMetrics = getUserMetrics(user);
if (userMetrics != null) {
userMetrics.submitAppAttempt(user);
}
if (parent != null) {
parent.submitAppAttempt(user);
}
}
public void runAppAttempt(ApplicationId appId, String user) {
runBuckets.add(appId, System.currentTimeMillis());
appsRunning.incr();
appsPending.decr();
QueueMetrics userMetrics = getUserMetrics(user);
if (userMetrics != null) {
userMetrics.runAppAttempt(appId, user);
}
if (parent != null) {
parent.runAppAttempt(appId, user);
}
}
public void finishAppAttempt(ApplicationId appId, boolean isPending, String user) {
runBuckets.remove(appId);
if (isPending) {
appsPending.decr();
} else {
appsRunning.decr();
}
QueueMetrics userMetrics = getUserMetrics(user);
if (userMetrics != null) {
userMetrics.finishAppAttempt(appId, isPending, user);
}
if (parent != null) {
parent.finishAppAttempt(appId, isPending, user);
}
}
public void finishApp(String user, RMAppState rmAppFinalState) {
switch(rmAppFinalState) {
case KILLED:
appsKilled.incr();
break;
case FAILED:
appsFailed.incr();
break;
default:
appsCompleted.incr();
break;
}
QueueMetrics userMetrics = getUserMetrics(user);
if (userMetrics != null) {
userMetrics.finishApp(user, rmAppFinalState);
}
if (parent != null) {
parent.finishApp(user, rmAppFinalState);
}
}
public void moveAppFrom(AppSchedulingInfo app) {
if (app.isPending()) {
appsPending.decr();
} else {
appsRunning.decr();
}
QueueMetrics userMetrics = getUserMetrics(app.getUser());
if (userMetrics != null) {
userMetrics.moveAppFrom(app);
}
if (parent != null) {
parent.moveAppFrom(app);
}
}
public void moveAppTo(AppSchedulingInfo app) {
if (app.isPending()) {
appsPending.incr();
} else {
appsRunning.incr();
}
QueueMetrics userMetrics = getUserMetrics(app.getUser());
if (userMetrics != null) {
userMetrics.moveAppTo(app);
}
if (parent != null) {
parent.moveAppTo(app);
}
}
/**
* Set available resources. To be called by scheduler periodically as
* resources become available.
* @param limit resource limit
*/
public void setAvailableResourcesToQueue(Resource limit) {
availableMB.set(limit.getMemory());
availableVCores.set(limit.getVirtualCores());
}
/**
* Set available resources. To be called by scheduler periodically as
* resources become available.
* @param user
* @param limit resource limit
*/
public void setAvailableResourcesToUser(String user, Resource limit) {
QueueMetrics userMetrics = getUserMetrics(user);
if (userMetrics != null) {
userMetrics.setAvailableResourcesToQueue(limit);
}
}
/**
* Increment pending resource metrics
* @param user
* @param containers
* @param res the TOTAL delta of resources note this is different from
* the other APIs which use per container resource
*/
public void incrPendingResources(String user, int containers, Resource res) {
_incrPendingResources(containers, res);
QueueMetrics userMetrics = getUserMetrics(user);
if (userMetrics != null) {
userMetrics.incrPendingResources(user, containers, res);
}
if (parent != null) {
parent.incrPendingResources(user, containers, res);
}
}
private void _incrPendingResources(int containers, Resource res) {
pendingContainers.incr(containers);
pendingMB.incr(res.getMemory() * containers);
pendingVCores.incr(res.getVirtualCores() * containers);
}
public void decrPendingResources(String user, int containers, Resource res) {
_decrPendingResources(containers, res);
QueueMetrics userMetrics = getUserMetrics(user);
if (userMetrics != null) {
userMetrics.decrPendingResources(user, containers, res);
}
if (parent != null) {
parent.decrPendingResources(user, containers, res);
}
}
private void _decrPendingResources(int containers, Resource res) {
pendingContainers.decr(containers);
pendingMB.decr(res.getMemory() * containers);
pendingVCores.decr(res.getVirtualCores() * containers);
}
public void allocateResources(String user, int containers, Resource res, boolean decrPending) {
allocatedContainers.incr(containers);
aggregateContainersAllocated.incr(containers);
allocatedMB.incr(res.getMemory() * containers);
allocatedVCores.incr(res.getVirtualCores() * containers);
if (decrPending) {
_decrPendingResources(containers, res);
}
QueueMetrics userMetrics = getUserMetrics(user);
if (userMetrics != null) {
userMetrics.allocateResources(user, containers, res, decrPending);
}
if (parent != null) {
parent.allocateResources(user, containers, res, decrPending);
}
}
public void releaseResources(String user, int containers, Resource res) {
allocatedContainers.decr(containers);
aggregateContainersReleased.incr(containers);
allocatedMB.decr(res.getMemory() * containers);
allocatedVCores.decr(res.getVirtualCores() * containers);
QueueMetrics userMetrics = getUserMetrics(user);
if (userMetrics != null) {
userMetrics.releaseResources(user, containers, res);
}
if (parent != null) {
parent.releaseResources(user, containers, res);
}
}
public void reserveResource(String user, Resource res) {
reservedContainers.incr();
reservedMB.incr(res.getMemory());
reservedVCores.incr(res.getVirtualCores());
QueueMetrics userMetrics = getUserMetrics(user);
if (userMetrics != null) {
userMetrics.reserveResource(user, res);
}
if (parent != null) {
parent.reserveResource(user, res);
}
}
public void unreserveResource(String user, Resource res) {
reservedContainers.decr();
reservedMB.decr(res.getMemory());
reservedVCores.decr(res.getVirtualCores());
QueueMetrics userMetrics = getUserMetrics(user);
if (userMetrics != null) {
userMetrics.unreserveResource(user, res);
}
if (parent != null) {
parent.unreserveResource(user, res);
}
}
public void incrActiveUsers() {
activeUsers.incr();
}
public void decrActiveUsers() {
activeUsers.decr();
}
public void activateApp(String user) {
activeApplications.incr();
QueueMetrics userMetrics = getUserMetrics(user);
if (userMetrics != null) {
userMetrics.activateApp(user);
}
if (parent != null) {
parent.activateApp(user);
}
}
public void deactivateApp(String user) {
activeApplications.decr();
QueueMetrics userMetrics = getUserMetrics(user);
if (userMetrics != null) {
userMetrics.deactivateApp(user);
}
if (parent != null) {
parent.deactivateApp(user);
}
}
public int getAppsSubmitted() {
return appsSubmitted.value();
}
public int getAppsRunning() {
return appsRunning.value();
}
public int getAppsPending() {
return appsPending.value();
}
public int getAppsCompleted() {
return appsCompleted.value();
}
public int getAppsKilled() {
return appsKilled.value();
}
public int getAppsFailed() {
return appsFailed.value();
}
public Resource getAllocatedResources() {
return BuilderUtils.newResource(allocatedMB.value(), allocatedVCores.value());
}
public int getAllocatedMB() {
return allocatedMB.value();
}
public int getAllocatedVirtualCores() {
return allocatedVCores.value();
}
public int getAllocatedContainers() {
return allocatedContainers.value();
}
public int getAvailableMB() {
return availableMB.value();
}
public int getAvailableVirtualCores() {
return availableVCores.value();
}
public int getPendingMB() {
return pendingMB.value();
}
public int getPendingVirtualCores() {
return pendingVCores.value();
}
public int getPendingContainers() {
return pendingContainers.value();
}
public int getReservedMB() {
return reservedMB.value();
}
public int getReservedVirtualCores() {
return reservedVCores.value();
}
public int getReservedContainers() {
return reservedContainers.value();
}
public int getActiveUsers() {
return activeUsers.value();
}
public int getActiveApps() {
return activeApplications.value();
}
public MetricsSystem getMetricsSystem() {
return metricsSystem;
}
}
14
View Complete Implementation : RetryCacheMetrics.java
Copyright Apache License 2.0
Author : aliyun-beta
Copyright Apache License 2.0
Author : aliyun-beta
/**
* This clreplaced is for maintaining the various RetryCache-related statistics
* and publishing them through the metrics interfaces.
*/
@InterfaceAudience.Private
@Metrics(about = "Aggregate RetryCache metrics", context = "rpc")
public clreplaced RetryCacheMetrics {
static final Log LOG = LogFactory.getLog(RetryCacheMetrics.clreplaced);
final MetricsRegistry registry;
final String name;
RetryCacheMetrics(RetryCache retryCache) {
name = "RetryCache." + retryCache.getCacheName();
registry = new MetricsRegistry(name);
if (LOG.isDebugEnabled()) {
LOG.debug("Initialized " + registry);
}
}
public String getName() {
return name;
}
public static RetryCacheMetrics create(RetryCache cache) {
RetryCacheMetrics m = new RetryCacheMetrics(cache);
return DefaultMetricsSystem.instance().register(m.name, null, m);
}
@Metric("Number of RetryCache hit")
MutableCounterLong cacheHit;
@Metric("Number of RetryCache cleared")
MutableCounterLong cacheCleared;
@Metric("Number of RetryCache updated")
MutableCounterLong cacheUpdated;
/**
* One cache hit event
*/
public void incrCacheHit() {
cacheHit.incr();
}
/**
* One cache cleared
*/
public void incrCacheCleared() {
cacheCleared.incr();
}
/**
* One cache updated
*/
public void incrCacheUpdated() {
cacheUpdated.incr();
}
public long getCacheHit() {
return cacheHit.value();
}
public long getCacheCleared() {
return cacheCleared.value();
}
public long getCacheUpdated() {
return cacheUpdated.value();
}
}
14
View Complete Implementation : JournalMetrics.java
Copyright Apache License 2.0
Author : aliyun-beta
Copyright Apache License 2.0
Author : aliyun-beta
/**
* The server-side metrics for a journal from the JournalNode's
* perspective.
*/
@Metrics(about = "Journal metrics", context = "dfs")
clreplaced JournalMetrics {
final MetricsRegistry registry = new MetricsRegistry("JournalNode");
@Metric("Number of batches written since startup")
MutableCounterLong batchesWritten;
@Metric("Number of txns written since startup")
MutableCounterLong txnsWritten;
@Metric("Number of bytes written since startup")
MutableCounterLong bytesWritten;
@Metric("Number of batches written where this node was lagging")
MutableCounterLong batchesWrittenWhileLagging;
private final int[] QUANTILE_INTERVALS = new int[] { // 1m
1 * 60, // 5m
5 * 60, // 1h
60 * 60 };
final MutableQuantiles[] syncsQuantiles;
private final Journal journal;
JournalMetrics(Journal journal) {
this.journal = journal;
syncsQuantiles = new MutableQuantiles[QUANTILE_INTERVALS.length];
for (int i = 0; i < syncsQuantiles.length; i++) {
int interval = QUANTILE_INTERVALS[i];
syncsQuantiles[i] = registry.newQuantiles("syncs" + interval + "s", "Journal sync time", "ops", "latencyMicros", interval);
}
}
public static JournalMetrics create(Journal j) {
JournalMetrics m = new JournalMetrics(j);
return DefaultMetricsSystem.instance().register(m.getName(), null, m);
}
String getName() {
return "Journal-" + journal.getJournalId();
}
@Metric("Current writer's epoch")
public long getLastWriterEpoch() {
try {
return journal.getLastWriterEpoch();
} catch (IOException e) {
return -1L;
}
}
@Metric("Last accepted epoch")
public long getLastPromisedEpoch() {
try {
return journal.getLastPromisedEpoch();
} catch (IOException e) {
return -1L;
}
}
@Metric("The highest txid stored on this JN")
public long getLastWrittenTxId() {
return journal.getHighestWrittenTxId();
}
@Metric("Number of transactions that this JN is lagging")
public long getCurrentLagTxns() {
try {
return journal.getCurrentLagTxns();
} catch (IOException e) {
return -1L;
}
}
@Metric("The timestamp of last successfully written transaction")
public long getLastJournalTimestamp() {
return journal.getLastJournalTimestamp();
}
void addSync(long us) {
for (MutableQuantiles q : syncsQuantiles) {
q.add(us);
}
}
}