From ce96297c22275308f9244e937ef1a8cd7855bb69 Mon Sep 17 00:00:00 2001 From: Danilo Penna Queiroz Date: Tue, 27 May 2014 19:03:07 -0300 Subject: [PATCH 01/48] Make it work for ES 1.0.1 --- pom.xml | 2 +- .../service/statsd/StatsdReporter.java | 335 +++++++++--------- .../service/statsd/StatsdService.java | 224 ++++++------ 3 files changed, 284 insertions(+), 277 deletions(-) diff --git a/pom.xml b/pom.xml index 951fceb..09db620 100644 --- a/pom.xml +++ b/pom.xml @@ -12,7 +12,7 @@ https://github.com/swoop-inc/elasticsearch-statsd-plugin/ - 0.90.3 + 1.0.1 1.3 diff --git a/src/main/java/org/elasticsearch/service/statsd/StatsdReporter.java b/src/main/java/org/elasticsearch/service/statsd/StatsdReporter.java index 58a1919..22a99b0 100644 --- a/src/main/java/org/elasticsearch/service/statsd/StatsdReporter.java +++ b/src/main/java/org/elasticsearch/service/statsd/StatsdReporter.java @@ -1,6 +1,8 @@ package org.elasticsearch.service.statsd; -import com.timgroup.statsd.StatsDClient; +import java.util.Iterator; +import java.util.List; +import java.util.Map; import org.elasticsearch.action.admin.cluster.node.stats.NodeStats; import org.elasticsearch.common.logging.ESLogger; @@ -21,15 +23,14 @@ import org.elasticsearch.indices.NodeIndicesStats; import org.elasticsearch.monitor.fs.FsStats; import org.elasticsearch.monitor.jvm.JvmStats; +import org.elasticsearch.monitor.jvm.JvmStats.GarbageCollector; import org.elasticsearch.monitor.network.NetworkStats; import org.elasticsearch.monitor.os.OsStats; import org.elasticsearch.monitor.process.ProcessStats; import org.elasticsearch.threadpool.ThreadPoolStats; import org.elasticsearch.transport.TransportStats; -import java.util.Iterator; -import java.util.List; -import java.util.Map; +import com.timgroup.statsd.StatsDClient; public class StatsdReporter { @@ -54,169 +55,175 @@ public StatsdReporter(NodeIndicesStats nodeIndicesStats, List indexS public void run() { try { - sendNodeIndicesStats(); - sendIndexShardStats(); - sendNodeStats(); + this.sendNodeIndicesStats(); + this.sendIndexShardStats(); + this.sendNodeStats(); } catch (Exception e) { - logException(e); + this.logException(e); } } private void sendNodeStats() { - sendNodeFsStats(nodeStats.getFs()); - sendNodeHttpStats(nodeStats.getHttp()); - sendNodeJvmStats(nodeStats.getJvm()); - sendNodeNetworkStats(nodeStats.getNetwork()); - sendNodeOsStats(nodeStats.getOs()); - sendNodeProcessStats(nodeStats.getProcess()); - sendNodeTransportStats(nodeStats.getTransport()); - sendNodeThreadPoolStats(nodeStats.getThreadPool()); + this.sendNodeFsStats(this.nodeStats.getFs()); + this.sendNodeHttpStats(this.nodeStats.getHttp()); + this.sendNodeJvmStats(this.nodeStats.getJvm()); + this.sendNodeNetworkStats(this.nodeStats.getNetwork()); + this.sendNodeOsStats(this.nodeStats.getOs()); + this.sendNodeProcessStats(this.nodeStats.getProcess()); + this.sendNodeTransportStats(this.nodeStats.getTransport()); + this.sendNodeThreadPoolStats(this.nodeStats.getThreadPool()); } private void sendNodeThreadPoolStats(ThreadPoolStats threadPoolStats) { - String type = buildMetricName("node.threadpool"); + String type = this.buildMetricName("node.threadpool"); Iterator statsIterator = threadPoolStats.iterator(); while (statsIterator.hasNext()) { ThreadPoolStats.Stats stats = statsIterator.next(); String id = type + "." + stats.getName(); - sendGauge(id, "threads", stats.getThreads()); - sendGauge(id, "queue", stats.getQueue()); - sendGauge(id, "active", stats.getActive()); - sendGauge(id, "rejected", stats.getRejected()); - sendGauge(id, "largest", stats.getLargest()); - sendGauge(id, "completed", stats.getCompleted()); + this.sendGauge(id, "threads", stats.getThreads()); + this.sendGauge(id, "queue", stats.getQueue()); + this.sendGauge(id, "active", stats.getActive()); + this.sendGauge(id, "rejected", stats.getRejected()); + this.sendGauge(id, "largest", stats.getLargest()); + this.sendGauge(id, "completed", stats.getCompleted()); } } private void sendNodeTransportStats(TransportStats transportStats) { - String type = buildMetricName("node.transport"); - sendGauge(type, "serverOpen", transportStats.serverOpen()); - sendCount(type, "rxCount", transportStats.rxCount()); - sendCount(type, "rxSizeBytes", transportStats.rxSize().bytes()); - sendCount(type, "txCount", transportStats.txCount()); - sendCount(type, "txSizeBytes", transportStats.txSize().bytes()); + String type = this.buildMetricName("node.transport"); + this.sendGauge(type, "serverOpen", transportStats.serverOpen()); + this.sendCount(type, "rxCount", transportStats.rxCount()); + this.sendCount(type, "rxSizeBytes", transportStats.rxSize().bytes()); + this.sendCount(type, "txCount", transportStats.txCount()); + this.sendCount(type, "txSizeBytes", transportStats.txSize().bytes()); } private void sendNodeProcessStats(ProcessStats processStats) { - String type = buildMetricName("node.process"); + String type = this.buildMetricName("node.process"); - sendGauge(type, "openFileDescriptors", processStats.openFileDescriptors()); + this.sendGauge(type, "openFileDescriptors", processStats.openFileDescriptors()); if (processStats.cpu() != null) { - sendGauge(type + ".cpu", "percent", processStats.cpu().percent()); - sendGauge(type + ".cpu", "sysSeconds", processStats.cpu().sys().seconds()); - sendGauge(type + ".cpu", "totalSeconds", processStats.cpu().total().seconds()); - sendGauge(type + ".cpu", "userSeconds", processStats.cpu().user().seconds()); + this.sendGauge(type + ".cpu", "percent", processStats.cpu().percent()); + this.sendGauge(type + ".cpu", "sysSeconds", processStats.cpu().sys().seconds()); + this.sendGauge(type + ".cpu", "totalSeconds", processStats.cpu().total().seconds()); + this.sendGauge(type + ".cpu", "userSeconds", processStats.cpu().user().seconds()); } if (processStats.mem() != null) { - sendGauge(type + ".mem", "totalVirtual", processStats.mem().totalVirtual().bytes()); - sendGauge(type + ".mem", "resident", processStats.mem().resident().bytes()); - sendGauge(type + ".mem", "share", processStats.mem().share().bytes()); + this.sendGauge(type + ".mem", "totalVirtual", processStats.mem().totalVirtual().bytes()); + this.sendGauge(type + ".mem", "resident", processStats.mem().resident().bytes()); + this.sendGauge(type + ".mem", "share", processStats.mem().share().bytes()); } } private void sendNodeOsStats(OsStats osStats) { - String type = buildMetricName("node.os"); + String type = this.buildMetricName("node.os"); if (osStats.cpu() != null) { - sendGauge(type + ".cpu", "sys", osStats.cpu().sys()); - sendGauge(type + ".cpu", "idle", osStats.cpu().idle()); - sendGauge(type + ".cpu", "user", osStats.cpu().user()); + this.sendGauge(type + ".cpu", "sys", osStats.cpu().sys()); + this.sendGauge(type + ".cpu", "idle", osStats.cpu().idle()); + this.sendGauge(type + ".cpu", "user", osStats.cpu().user()); } if (osStats.mem() != null) { - sendGauge(type + ".mem", "freeBytes", osStats.mem().free().bytes()); - sendGauge(type + ".mem", "usedBytes", osStats.mem().used().bytes()); - sendGauge(type + ".mem", "freePercent", osStats.mem().freePercent()); - sendGauge(type + ".mem", "usedPercent", osStats.mem().usedPercent()); - sendGauge(type + ".mem", "actualFreeBytes", osStats.mem().actualFree().bytes()); - sendGauge(type + ".mem", "actualUsedBytes", osStats.mem().actualUsed().bytes()); + this.sendGauge(type + ".mem", "freeBytes", osStats.mem().free().bytes()); + this.sendGauge(type + ".mem", "usedBytes", osStats.mem().used().bytes()); + this.sendGauge(type + ".mem", "freePercent", osStats.mem().freePercent()); + this.sendGauge(type + ".mem", "usedPercent", osStats.mem().usedPercent()); + this.sendGauge(type + ".mem", "actualFreeBytes", osStats.mem().actualFree().bytes()); + this.sendGauge(type + ".mem", "actualUsedBytes", osStats.mem().actualUsed().bytes()); } if (osStats.swap() != null) { - sendGauge(type + ".swap", "freeBytes", osStats.swap().free().bytes()); - sendGauge(type + ".swap", "usedBytes", osStats.swap().used().bytes()); + this.sendGauge(type + ".swap", "freeBytes", osStats.swap().free().bytes()); + this.sendGauge(type + ".swap", "usedBytes", osStats.swap().used().bytes()); } } private void sendNodeNetworkStats(NetworkStats networkStats) { - String type = buildMetricName("node.network.tcp"); + String type = this.buildMetricName("node.network.tcp"); NetworkStats.Tcp tcp = networkStats.tcp(); // might be null, if sigar isnt loaded if (tcp != null) { - sendGauge(type, "activeOpens", tcp.activeOpens()); - sendGauge(type, "passiveOpens", tcp.passiveOpens()); - sendGauge(type, "attemptFails", tcp.attemptFails()); - sendGauge(type, "estabResets", tcp.estabResets()); - sendGauge(type, "currEstab", tcp.currEstab()); - sendGauge(type, "inSegs", tcp.inSegs()); - sendGauge(type, "outSegs", tcp.outSegs()); - sendGauge(type, "retransSegs", tcp.retransSegs()); - sendGauge(type, "inErrs", tcp.inErrs()); - sendGauge(type, "outRsts", tcp.outRsts()); + this.sendGauge(type, "activeOpens", tcp.activeOpens()); + this.sendGauge(type, "passiveOpens", tcp.passiveOpens()); + this.sendGauge(type, "attemptFails", tcp.attemptFails()); + this.sendGauge(type, "estabResets", tcp.estabResets()); + this.sendGauge(type, "currEstab", tcp.currEstab()); + this.sendGauge(type, "inSegs", tcp.inSegs()); + this.sendGauge(type, "outSegs", tcp.outSegs()); + this.sendGauge(type, "retransSegs", tcp.retransSegs()); + this.sendGauge(type, "inErrs", tcp.inErrs()); + this.sendGauge(type, "outRsts", tcp.outRsts()); } } private void sendNodeJvmStats(JvmStats jvmStats) { - String type = buildMetricName("node.jvm"); - sendGauge(type, "uptime", jvmStats.uptime().seconds()); + String type = this.buildMetricName("node.jvm"); + this.sendGauge(type, "uptime", jvmStats.uptime().seconds()); // mem - sendGauge(type + ".mem", "heapCommitted", jvmStats.mem().heapCommitted().bytes()); - sendGauge(type + ".mem", "heapUsed", jvmStats.mem().heapUsed().bytes()); - sendGauge(type + ".mem", "nonHeapCommitted", jvmStats.mem().nonHeapCommitted().bytes()); - sendGauge(type + ".mem", "nonHeapUsed", jvmStats.mem().nonHeapUsed().bytes()); + this.sendGauge(type + ".mem", "heapCommitted", jvmStats.mem().heapCommitted().bytes()); + this.sendGauge(type + ".mem", "heapUsed", jvmStats.mem().heapUsed().bytes()); + this.sendGauge(type + ".mem", "nonHeapCommitted", jvmStats.mem().nonHeapCommitted().bytes()); + this.sendGauge(type + ".mem", "nonHeapUsed", jvmStats.mem().nonHeapUsed().bytes()); Iterator memoryPoolIterator = jvmStats.mem().iterator(); while (memoryPoolIterator.hasNext()) { JvmStats.MemoryPool memoryPool = memoryPoolIterator.next(); String memoryPoolType = type + ".mem.pool." + memoryPool.name(); - sendGauge(memoryPoolType, "max", memoryPool.max().bytes()); - sendGauge(memoryPoolType, "used", memoryPool.used().bytes()); - sendGauge(memoryPoolType, "peakUsed", memoryPool.peakUsed().bytes()); - sendGauge(memoryPoolType, "peakMax", memoryPool.peakMax().bytes()); + this.sendGauge(memoryPoolType, "max", memoryPool.max().bytes()); + this.sendGauge(memoryPoolType, "used", memoryPool.used().bytes()); + this.sendGauge(memoryPoolType, "peakUsed", memoryPool.peakUsed().bytes()); + this.sendGauge(memoryPoolType, "peakMax", memoryPool.peakMax().bytes()); } // threads - sendGauge(type + ".threads", "count", jvmStats.threads().count()); - sendGauge(type + ".threads", "peakCount", jvmStats.threads().peakCount()); + this.sendGauge(type + ".threads", "count", jvmStats.threads().count()); + this.sendGauge(type + ".threads", "peakCount", jvmStats.threads().peakCount()); // garbage collectors - sendCount(type + ".gc", "collectionCount", jvmStats.gc().collectionCount()); - sendTime(type + ".gc", "collectionTimeSeconds", jvmStats.gc().collectionTime().seconds()); + long gcCounter = 0; + long gcTimes = 0; + for (GarbageCollector gc : jvmStats.gc()) { + gcCounter += gc.collectionCount(); + gcTimes += gc.collectionTime().getSeconds(); + } + this.sendCount(type + ".gc", "collectionCount", gcCounter); + this.sendTime(type + ".gc", "collectionTimeSeconds", gcTimes); for (JvmStats.GarbageCollector collector : jvmStats.gc().collectors()) { String id = type + ".gc." + collector.name(); - sendCount(id, "collectionCount", collector.collectionCount()); - sendTime(id, "collectionTimeSeconds", collector.collectionTime().seconds()); + this.sendCount(id, "collectionCount", collector.collectionCount()); + this.sendTime(id, "collectionTimeSeconds", collector.collectionTime().seconds()); JvmStats.GarbageCollector.LastGc lastGc = collector.lastGc(); String lastGcType = type + ".lastGc"; if (lastGc != null) { - sendTime(lastGcType, "time", lastGc.endTime() - lastGc.startTime()); - sendGauge(lastGcType, "max", lastGc.max().bytes()); - sendGauge(lastGcType, "beforeUsed", lastGc.beforeUsed().bytes()); - sendGauge(lastGcType, "afterUsed", lastGc.afterUsed().bytes()); - sendGauge(lastGcType, "durationSeconds", lastGc.duration().seconds()); + this.sendTime(lastGcType, "time", lastGc.endTime() - lastGc.startTime()); + this.sendGauge(lastGcType, "max", lastGc.max().bytes()); + this.sendGauge(lastGcType, "beforeUsed", lastGc.beforeUsed().bytes()); + this.sendGauge(lastGcType, "afterUsed", lastGc.afterUsed().bytes()); + this.sendGauge(lastGcType, "durationSeconds", lastGc.duration().seconds()); } } } private void sendNodeHttpStats(HttpStats httpStats) { - String type = buildMetricName("node.http"); - sendGauge(type, "serverOpen", httpStats.getServerOpen()); - sendGauge(type, "totalOpen", httpStats.getTotalOpen()); + String type = this.buildMetricName("node.http"); + this.sendGauge(type, "serverOpen", httpStats.getServerOpen()); + this.sendGauge(type, "totalOpen", httpStats.getTotalOpen()); } private void sendNodeFsStats(FsStats fs) @@ -224,174 +231,174 @@ private void sendNodeFsStats(FsStats fs) Iterator infoIterator = fs.iterator(); int i = 0; while (infoIterator.hasNext()) { - String type = buildMetricName("node.fs") + i; + String type = this.buildMetricName("node.fs") + i; FsStats.Info info = infoIterator.next(); - sendGauge(type, "available", info.getAvailable().bytes()); - sendGauge(type, "total", info.getTotal().bytes()); - sendGauge(type, "free", info.getFree().bytes()); - sendCount(type, "diskReads", info.getDiskReads()); - sendCount(type, "diskReadsInBytes", info.getDiskReadSizeInBytes()); - sendCount(type, "diskWrites", info.getDiskWrites()); - sendCount(type, "diskWritesInBytes", info.getDiskWriteSizeInBytes()); - sendGauge(type, "diskQueue", (long) info.getDiskQueue()); - sendGauge(type, "diskService", (long) info.getDiskServiceTime()); + this.sendGauge(type, "available", info.getAvailable().bytes()); + this.sendGauge(type, "total", info.getTotal().bytes()); + this.sendGauge(type, "free", info.getFree().bytes()); + this.sendCount(type, "diskReads", info.getDiskReads()); + this.sendCount(type, "diskReadsInBytes", info.getDiskReadSizeInBytes()); + this.sendCount(type, "diskWrites", info.getDiskWrites()); + this.sendCount(type, "diskWritesInBytes", info.getDiskWriteSizeInBytes()); + this.sendGauge(type, "diskQueue", (long) info.getDiskQueue()); + this.sendGauge(type, "diskService", (long) info.getDiskServiceTime()); i++; } } private void sendIndexShardStats() { - for (IndexShard indexShard : indexShards) { - String type = buildMetricName("indexes.") + indexShard.shardId().index().name() + ".id." + indexShard.shardId().id(); - sendIndexShardStats(type, indexShard); + for (IndexShard indexShard : this.indexShards) { + String type = this.buildMetricName("indexes.") + indexShard.shardId().index().name() + ".id." + indexShard.shardId().id(); + this.sendIndexShardStats(type, indexShard); } } private void sendIndexShardStats(String type, IndexShard indexShard) { - sendSearchStats(type + ".search", indexShard.searchStats()); - sendGetStats(type + ".get", indexShard.getStats()); - sendDocsStats(type + ".docs", indexShard.docStats()); - sendRefreshStats(type + ".refresh", indexShard.refreshStats()); - sendIndexingStats(type + ".indexing", indexShard.indexingStats("_all")); - sendMergeStats(type + ".merge", indexShard.mergeStats()); - sendWarmerStats(type + ".warmer", indexShard.warmerStats()); - sendStoreStats(type + ".store", indexShard.storeStats()); + this.sendSearchStats(type + ".search", indexShard.searchStats()); + this.sendGetStats(type + ".get", indexShard.getStats()); + this.sendDocsStats(type + ".docs", indexShard.docStats()); + this.sendRefreshStats(type + ".refresh", indexShard.refreshStats()); + this.sendIndexingStats(type + ".indexing", indexShard.indexingStats("_all")); + this.sendMergeStats(type + ".merge", indexShard.mergeStats()); + this.sendWarmerStats(type + ".warmer", indexShard.warmerStats()); + this.sendStoreStats(type + ".store", indexShard.storeStats()); } private void sendStoreStats(String type, StoreStats storeStats) { - sendGauge(type, "sizeInBytes", storeStats.sizeInBytes()); - sendGauge(type, "throttleTimeInNanos", storeStats.throttleTime().getNanos()); + this.sendGauge(type, "sizeInBytes", storeStats.sizeInBytes()); + this.sendGauge(type, "throttleTimeInNanos", storeStats.throttleTime().getNanos()); } private void sendWarmerStats(String type, WarmerStats warmerStats) { - sendGauge(type, "current", warmerStats.current()); - sendGauge(type, "total", warmerStats.total()); - sendTime(type, "totalTimeInMillis", warmerStats.totalTimeInMillis()); + this.sendGauge(type, "current", warmerStats.current()); + this.sendGauge(type, "total", warmerStats.total()); + this.sendTime(type, "totalTimeInMillis", warmerStats.totalTimeInMillis()); } private void sendMergeStats(String type, MergeStats mergeStats) { - sendGauge(type, "total", mergeStats.getTotal()); - sendTime(type, "totalTimeInMillis", mergeStats.getTotalTimeInMillis()); - sendGauge(type, "totalNumDocs", mergeStats.getTotalNumDocs()); - sendGauge(type, "current", mergeStats.getCurrent()); - sendGauge(type, "currentNumDocs", mergeStats.getCurrentNumDocs()); - sendGauge(type, "currentSizeInBytes", mergeStats.getCurrentSizeInBytes()); + this.sendGauge(type, "total", mergeStats.getTotal()); + this.sendTime(type, "totalTimeInMillis", mergeStats.getTotalTimeInMillis()); + this.sendGauge(type, "totalNumDocs", mergeStats.getTotalNumDocs()); + this.sendGauge(type, "current", mergeStats.getCurrent()); + this.sendGauge(type, "currentNumDocs", mergeStats.getCurrentNumDocs()); + this.sendGauge(type, "currentSizeInBytes", mergeStats.getCurrentSizeInBytes()); } private void sendNodeIndicesStats() { - String type = buildMetricName("node"); - sendFilterCacheStats(type + ".filtercache", nodeIndicesStats.getFilterCache()); - sendIdCacheStats(type + ".idcache", nodeIndicesStats.getIdCache()); - sendDocsStats(type + ".docs", nodeIndicesStats.getDocs()); - sendFlushStats(type + ".flush", nodeIndicesStats.getFlush()); - sendGetStats(type + ".get", nodeIndicesStats.getGet()); - sendIndexingStats(type + ".indexing", nodeIndicesStats.getIndexing()); - sendRefreshStats(type + ".refresh", nodeIndicesStats.getRefresh()); - sendSearchStats(type + ".search", nodeIndicesStats.getSearch()); + String type = this.buildMetricName("node"); + this.sendFilterCacheStats(type + ".filtercache", this.nodeIndicesStats.getFilterCache()); + this.sendIdCacheStats(type + ".idcache", this.nodeIndicesStats.getIdCache()); + this.sendDocsStats(type + ".docs", this.nodeIndicesStats.getDocs()); + this.sendFlushStats(type + ".flush", this.nodeIndicesStats.getFlush()); + this.sendGetStats(type + ".get", this.nodeIndicesStats.getGet()); + this.sendIndexingStats(type + ".indexing", this.nodeIndicesStats.getIndexing()); + this.sendRefreshStats(type + ".refresh", this.nodeIndicesStats.getRefresh()); + this.sendSearchStats(type + ".search", this.nodeIndicesStats.getSearch()); } private void sendSearchStats(String type, SearchStats searchStats) { SearchStats.Stats totalSearchStats = searchStats.getTotal(); - sendSearchStatsStats(type + "._all", totalSearchStats); + this.sendSearchStatsStats(type + "._all", totalSearchStats); if (searchStats.getGroupStats() != null) { for (Map.Entry statsEntry : searchStats.getGroupStats().entrySet()) { - sendSearchStatsStats(type + "." + statsEntry.getKey(), statsEntry.getValue()); + this.sendSearchStatsStats(type + "." + statsEntry.getKey(), statsEntry.getValue()); } } } private void sendSearchStatsStats(String group, SearchStats.Stats searchStats) { - String type = buildMetricName("search.stats.") + group; - sendCount(type, "queryCount", searchStats.getQueryCount()); - sendCount(type, "queryTimeInMillis", searchStats.getQueryTimeInMillis()); - sendGauge(type, "queryCurrent", searchStats.getQueryCurrent()); - sendCount(type, "fetchCount", searchStats.getFetchCount()); - sendCount(type, "fetchTimeInMillis", searchStats.getFetchTimeInMillis()); - sendGauge(type, "fetchCurrent", searchStats.getFetchCurrent()); + String type = this.buildMetricName("search.stats.") + group; + this.sendCount(type, "queryCount", searchStats.getQueryCount()); + this.sendCount(type, "queryTimeInMillis", searchStats.getQueryTimeInMillis()); + this.sendGauge(type, "queryCurrent", searchStats.getQueryCurrent()); + this.sendCount(type, "fetchCount", searchStats.getFetchCount()); + this.sendCount(type, "fetchTimeInMillis", searchStats.getFetchTimeInMillis()); + this.sendGauge(type, "fetchCurrent", searchStats.getFetchCurrent()); } private void sendRefreshStats(String type, RefreshStats refreshStats) { - sendCount(type, "total", refreshStats.getTotal()); - sendCount(type, "totalTimeInMillis", refreshStats.getTotalTimeInMillis()); + this.sendCount(type, "total", refreshStats.getTotal()); + this.sendCount(type, "totalTimeInMillis", refreshStats.getTotalTimeInMillis()); } private void sendIndexingStats(String type, IndexingStats indexingStats) { IndexingStats.Stats totalStats = indexingStats.getTotal(); - sendStats(type + "._all", totalStats); + this.sendStats(type + "._all", totalStats); Map typeStats = indexingStats.getTypeStats(); if (typeStats != null) { for (Map.Entry statsEntry : typeStats.entrySet()) { - sendStats(type + "." + statsEntry.getKey(), statsEntry.getValue()); + this.sendStats(type + "." + statsEntry.getKey(), statsEntry.getValue()); } } } private void sendStats(String type, IndexingStats.Stats stats) { - sendCount(type, "indexCount", stats.getIndexCount()); - sendCount(type, "indexTimeInMillis", stats.getIndexTimeInMillis()); - sendGauge(type, "indexCurrent", stats.getIndexCount()); - sendCount(type, "deleteCount", stats.getDeleteCount()); - sendCount(type, "deleteTimeInMillis", stats.getDeleteTimeInMillis()); - sendGauge(type, "deleteCurrent", stats.getDeleteCurrent()); + this.sendCount(type, "indexCount", stats.getIndexCount()); + this.sendCount(type, "indexTimeInMillis", stats.getIndexTimeInMillis()); + this.sendGauge(type, "indexCurrent", stats.getIndexCount()); + this.sendCount(type, "deleteCount", stats.getDeleteCount()); + this.sendCount(type, "deleteTimeInMillis", stats.getDeleteTimeInMillis()); + this.sendGauge(type, "deleteCurrent", stats.getDeleteCurrent()); } private void sendGetStats(String type, GetStats getStats) { - sendCount(type, "existsCount", getStats.getExistsCount()); - sendCount(type, "existsTimeInMillis", getStats.getExistsTimeInMillis()); - sendCount(type, "missingCount", getStats.getMissingCount()); - sendCount(type, "missingTimeInMillis", getStats.getMissingTimeInMillis()); - sendGauge(type, "current", getStats.current()); + this.sendCount(type, "existsCount", getStats.getExistsCount()); + this.sendCount(type, "existsTimeInMillis", getStats.getExistsTimeInMillis()); + this.sendCount(type, "missingCount", getStats.getMissingCount()); + this.sendCount(type, "missingTimeInMillis", getStats.getMissingTimeInMillis()); + this.sendGauge(type, "current", getStats.current()); } private void sendFlushStats(String type, FlushStats flush) { - sendCount(type, "total", flush.getTotal()); - sendCount(type, "totalTimeInMillis", flush.getTotalTimeInMillis()); + this.sendCount(type, "total", flush.getTotal()); + this.sendCount(type, "totalTimeInMillis", flush.getTotalTimeInMillis()); } private void sendDocsStats(String name, DocsStats docsStats) { - sendCount(name, "count", docsStats.getCount()); - sendCount(name, "deleted", docsStats.getDeleted()); + this.sendCount(name, "count", docsStats.getCount()); + this.sendCount(name, "deleted", docsStats.getDeleted()); } private void sendIdCacheStats(String name, IdCacheStats idCache) { - sendGauge(name, "memorySizeInBytes", idCache.getMemorySizeInBytes()); + this.sendGauge(name, "memorySizeInBytes", idCache.getMemorySizeInBytes()); } private void sendFilterCacheStats(String name, FilterCacheStats filterCache) { - sendGauge(name, "memorySizeInBytes", filterCache.getMemorySizeInBytes()); - sendGauge(name, "evictions", filterCache.getEvictions()); + this.sendGauge(name, "memorySizeInBytes", filterCache.getMemorySizeInBytes()); + this.sendGauge(name, "evictions", filterCache.getEvictions()); } protected void sendGauge(String name, String valueName, long value) { - statsdClient.gauge(join(name, valueName), (int) value); + this.statsdClient.gauge(this.join(name, valueName), (int) value); } protected void sendCount(String name, String valueName, long value) { - statsdClient.count(join(name, valueName), (int) value); + this.statsdClient.count(this.join(name, valueName), (int) value); } protected void sendTime(String name, String valueName, long value) { - statsdClient.time(join(name, valueName), (int) value); + this.statsdClient.time(this.join(name, valueName), (int) value); } protected String sanitizeString(String s) @@ -401,17 +408,19 @@ protected String sanitizeString(String s) protected String buildMetricName(String name) { - return sanitizeString(name); + return this.sanitizeString(name); } private String join(String... parts) { - if (parts == null) return null; + if (parts == null) { + return null; + } StringBuilder builder = new StringBuilder(); for (int i = 0; i < parts.length; i++) { builder.append(parts[i]); if (i < parts.length - 1) { - builder.append(joiner); + builder.append(this.joiner); } } return builder.toString(); diff --git a/src/main/java/org/elasticsearch/service/statsd/StatsdService.java b/src/main/java/org/elasticsearch/service/statsd/StatsdService.java index e1917df..025d5f3 100644 --- a/src/main/java/org/elasticsearch/service/statsd/StatsdService.java +++ b/src/main/java/org/elasticsearch/service/statsd/StatsdService.java @@ -1,9 +1,8 @@ package org.elasticsearch.service.statsd; -import com.timgroup.statsd.NonBlockingStatsDClient; -import com.timgroup.statsd.StatsDClient; +import java.util.List; -import org.elasticsearch.ElasticSearchException; +import org.elasticsearch.ElasticsearchException; import org.elasticsearch.action.admin.cluster.node.stats.NodeStats; import org.elasticsearch.action.admin.indices.stats.CommonStatsFlags; import org.elasticsearch.cluster.ClusterService; @@ -21,115 +20,114 @@ import org.elasticsearch.indices.NodeIndicesStats; import org.elasticsearch.node.service.NodeService; -import java.util.List; +import com.timgroup.statsd.NonBlockingStatsDClient; +import com.timgroup.statsd.StatsDClient; -public class StatsdService extends AbstractLifecycleComponent -{ - - private final ClusterService clusterService; - private final IndicesService indicesService; - private NodeService nodeService; - private final String statsdHost; - private final Integer statsdPort; - private final TimeValue statsdRefreshInternal; - private final String statsdPrefix; - private final StatsDClient statsdClient; - - private volatile Thread statsdReporterThread; - private volatile boolean closed; - - @Inject - public StatsdService(Settings settings, ClusterService clusterService, IndicesService indicesService, - NodeService nodeService) - { - super(settings); - this.clusterService = clusterService; - this.indicesService = indicesService; - this.nodeService = nodeService; - this.statsdRefreshInternal = settings.getAsTime("metrics.statsd.every", TimeValue.timeValueMinutes(1)); - this.statsdHost = settings.get("metrics.statsd.host"); - this.statsdPort = settings.getAsInt("metrics.statsd.port", 8125); - this.statsdPrefix = settings.get("metrics.statsd.prefix", "elasticsearch" + "." + settings.get("cluster.name")); - this.statsdClient = new NonBlockingStatsDClient(statsdPrefix, statsdHost, statsdPort); - } - - @Override - protected void doStart() throws ElasticSearchException - { - if (statsdHost != null && statsdHost.length() > 0) { - statsdReporterThread = EsExecutors.daemonThreadFactory(settings, "statsd_reporter").newThread( - new StatsdReporterThread()); - statsdReporterThread.start(); - logger.info("Statsd reporting triggered every [{}] to host [{}:{}] with metric prefix [{}]", - statsdRefreshInternal, statsdHost, statsdPort, statsdPrefix); - } - else { - logger.error("Statsd reporting disabled, no statsd host configured"); - } - } - - @Override - protected void doStop() throws ElasticSearchException - { - if (closed) { - return; - } - if (statsdReporterThread != null) { - statsdReporterThread.interrupt(); - } - closed = true; - logger.info("Statsd reporter stopped"); - } - - @Override - protected void doClose() throws ElasticSearchException - { - } - - public class StatsdReporterThread implements Runnable - { - - public void run() - { - while (!closed) { - DiscoveryNode node = clusterService.localNode(); - boolean isClusterStarted = clusterService.lifecycleState().equals(Lifecycle.State.STARTED); - - if (isClusterStarted && node != null && node.isMasterNode()) { - NodeIndicesStats nodeIndicesStats = indicesService.stats(false); - CommonStatsFlags commonStatsFlags = new CommonStatsFlags().clear(); - NodeStats nodeStats = nodeService.stats(commonStatsFlags, true, true, true, true, true, true, true, true); - List indexShards = getIndexShards(indicesService); - - StatsdReporter statsdReporter = new StatsdReporter(nodeIndicesStats, indexShards, nodeStats, statsdClient); - statsdReporter.run(); - } - else { - if (node != null) { - logger.debug("[{}]/[{}] is not master node, not triggering update", node.getId(), node.getName()); - } - } - - try { - Thread.sleep(statsdRefreshInternal.millis()); - } - catch (InterruptedException e1) { - continue; - } - } - } - - private List getIndexShards(IndicesService indicesService) - { - List indexShards = Lists.newArrayList(); - String[] indices = indicesService.indices().toArray(new String[] {}); - for (String indexName : indices) { - IndexService indexService = indicesService.indexServiceSafe(indexName); - for (int shardId : indexService.shardIds()) { - indexShards.add(indexService.shard(shardId)); - } - } - return indexShards; - } - } +public class StatsdService extends AbstractLifecycleComponent { + + private final ClusterService clusterService; + private final IndicesService indicesService; + private NodeService nodeService; + private final String statsdHost; + private final Integer statsdPort; + private final TimeValue statsdRefreshInternal; + private final String statsdPrefix; + private final StatsDClient statsdClient; + + private volatile Thread statsdReporterThread; + private volatile boolean closed; + + @Inject + public StatsdService(Settings settings, ClusterService clusterService, IndicesService indicesService, + NodeService nodeService) { + super(settings); + this.clusterService = clusterService; + this.indicesService = indicesService; + this.nodeService = nodeService; + this.statsdRefreshInternal = settings + .getAsTime("metrics.statsd.every", TimeValue.timeValueMinutes(1)); + this.statsdHost = settings.get("metrics.statsd.host"); + this.statsdPort = settings.getAsInt("metrics.statsd.port", 8125); + this.statsdPrefix = settings.get("metrics.statsd.prefix", + "elasticsearch" + "." + settings.get("cluster.name")); + this.statsdClient = new NonBlockingStatsDClient(this.statsdPrefix, this.statsdHost, this.statsdPort); + } + + @Override + protected void doStart() throws ElasticsearchException { + if (this.statsdHost != null && this.statsdHost.length() > 0) { + this.statsdReporterThread = EsExecutors.daemonThreadFactory(this.settings, "statsd_reporter") + .newThread(new StatsdReporterThread()); + this.statsdReporterThread.start(); + this.logger.info("Statsd reporting triggered every [{}] to host [{}:{}] with metric prefix [{}]", + this.statsdRefreshInternal, this.statsdHost, this.statsdPort, this.statsdPrefix); + } else { + this.logger.error("Statsd reporting disabled, no statsd host configured"); + } + } + + @Override + protected void doStop() throws ElasticsearchException { + if (this.closed) { + return; + } + if (this.statsdReporterThread != null) { + this.statsdReporterThread.interrupt(); + } + this.closed = true; + this.logger.info("Statsd reporter stopped"); + } + + @Override + protected void doClose() throws ElasticsearchException { + } + + public class StatsdReporterThread implements Runnable { + + @Override + public void run() { + while (!StatsdService.this.closed) { + DiscoveryNode node = StatsdService.this.clusterService.localNode(); + boolean isClusterStarted = StatsdService.this.clusterService.lifecycleState() + .equals(Lifecycle.State.STARTED); + + if (isClusterStarted && node != null && node.isMasterNode()) { + NodeIndicesStats nodeIndicesStats = StatsdService.this.indicesService.stats(false); + CommonStatsFlags commonStatsFlags = new CommonStatsFlags().clear(); + NodeStats nodeStats = StatsdService.this.nodeService.stats(commonStatsFlags, true, true, + true, true, true, true, true, + true, false); + List indexShards = this.getIndexShards(StatsdService.this.indicesService); + + StatsdReporter statsdReporter = new StatsdReporter(nodeIndicesStats, indexShards, + nodeStats, StatsdService.this.statsdClient); + statsdReporter.run(); + } else { + if (node != null) { + StatsdService.this.logger + .debug("[{}]/[{}] is not master node, not triggering update", node.getId(), + node.getName()); + } + } + + try { + Thread.sleep(StatsdService.this.statsdRefreshInternal.millis()); + } catch (InterruptedException e1) { + continue; + } + } + } + + private List getIndexShards(IndicesService indicesService) { + List indexShards = Lists.newArrayList(); + String[] indices = indicesService.indices().toArray(new String[] {}); + for (String indexName : indices) { + IndexService indexService = indicesService.indexServiceSafe(indexName); + for (int shardId : indexService.shardIds()) { + indexShards.add(indexService.shard(shardId)); + } + } + return indexShards; + } + } } From e52e56dce0b00402f6700d2e5a5aea16419e23df Mon Sep 17 00:00:00 2001 From: Xiao Yu Date: Tue, 5 Aug 2014 17:41:58 -0400 Subject: [PATCH 02/48] Version bump and define encoding --- pom.xml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 09db620..005e17f 100644 --- a/pom.xml +++ b/pom.xml @@ -6,14 +6,15 @@ 4.0.0 de.spinscale.elasticsearch elasticsearch-statsd - 0.2-SNAPSHOT + 0.3-SNAPSHOT jar - Statsd monitoring plugin for Elasticsearch + StatsD monitoring plugin for Elasticsearch https://github.com/swoop-inc/elasticsearch-statsd-plugin/ 1.0.1 1.3 + UTF-8 From eff4ff221364f8dee88918bccf3141a13b7963fc Mon Sep 17 00:00:00 2001 From: Xiao Yu Date: Tue, 5 Aug 2014 19:11:57 -0400 Subject: [PATCH 03/48] Cleanup code and capitalization --- .../plugin/statsd/StatsdPlugin.java | 2 +- .../service/statsd/StatsdService.java | 38 ++++++++++++------- 2 files changed, 25 insertions(+), 15 deletions(-) diff --git a/src/main/java/org/elasticsearch/plugin/statsd/StatsdPlugin.java b/src/main/java/org/elasticsearch/plugin/statsd/StatsdPlugin.java index d8ef2eb..029e90d 100644 --- a/src/main/java/org/elasticsearch/plugin/statsd/StatsdPlugin.java +++ b/src/main/java/org/elasticsearch/plugin/statsd/StatsdPlugin.java @@ -17,7 +17,7 @@ public String name() public String description() { - return "Statsd Monitoring Plugin"; + return "StatsD Monitoring Plugin"; } @SuppressWarnings("rawtypes") diff --git a/src/main/java/org/elasticsearch/service/statsd/StatsdService.java b/src/main/java/org/elasticsearch/service/statsd/StatsdService.java index 025d5f3..076c224 100644 --- a/src/main/java/org/elasticsearch/service/statsd/StatsdService.java +++ b/src/main/java/org/elasticsearch/service/statsd/StatsdService.java @@ -38,31 +38,41 @@ public class StatsdService extends AbstractLifecycleComponent { private volatile boolean closed; @Inject - public StatsdService(Settings settings, ClusterService clusterService, IndicesService indicesService, - NodeService nodeService) { + public StatsdService(Settings settings, ClusterService clusterService, IndicesService indicesService, NodeService nodeService) { super(settings); this.clusterService = clusterService; this.indicesService = indicesService; this.nodeService = nodeService; - this.statsdRefreshInternal = settings - .getAsTime("metrics.statsd.every", TimeValue.timeValueMinutes(1)); - this.statsdHost = settings.get("metrics.statsd.host"); - this.statsdPort = settings.getAsInt("metrics.statsd.port", 8125); - this.statsdPrefix = settings.get("metrics.statsd.prefix", - "elasticsearch" + "." + settings.get("cluster.name")); + this.statsdRefreshInternal = settings.getAsTime( + "metrics.statsd.every", TimeValue.timeValueMinutes(1) + ); + this.statsdHost = settings.get( + "metrics.statsd.host" + ); + this.statsdPort = settings.getAsInt( + "metrics.statsd.port", 8125 + ); + this.statsdPrefix = settings.get( + "metrics.statsd.prefix", "elasticsearch" + "." + settings.get("cluster.name") + ); this.statsdClient = new NonBlockingStatsDClient(this.statsdPrefix, this.statsdHost, this.statsdPort); } @Override protected void doStart() throws ElasticsearchException { if (this.statsdHost != null && this.statsdHost.length() > 0) { - this.statsdReporterThread = EsExecutors.daemonThreadFactory(this.settings, "statsd_reporter") - .newThread(new StatsdReporterThread()); + this.statsdReporterThread = EsExecutors + .daemonThreadFactory(this.settings, "statsd_reporter") + .newThread(new StatsdReporterThread()); this.statsdReporterThread.start(); - this.logger.info("Statsd reporting triggered every [{}] to host [{}:{}] with metric prefix [{}]", - this.statsdRefreshInternal, this.statsdHost, this.statsdPort, this.statsdPrefix); + this.logger.info( + "StatsD reporting triggered every [{}] to host [{}:{}] with metric prefix [{}]", + this.statsdRefreshInternal, this.statsdHost, this.statsdPort, this.statsdPrefix + ); } else { - this.logger.error("Statsd reporting disabled, no statsd host configured"); + this.logger.error( + "StatsD reporting disabled, no statsd host configured" + ); } } @@ -75,7 +85,7 @@ protected void doStop() throws ElasticsearchException { this.statsdReporterThread.interrupt(); } this.closed = true; - this.logger.info("Statsd reporter stopped"); + this.logger.info("StatsD reporter stopped"); } @Override From 89a9c6158d00578da51de0ddae76ae3bc5830464 Mon Sep 17 00:00:00 2001 From: Xiao Yu Date: Tue, 5 Aug 2014 19:15:07 -0400 Subject: [PATCH 04/48] Add config to skip reporting of shard info --- .../service/statsd/StatsdService.java | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/src/main/java/org/elasticsearch/service/statsd/StatsdService.java b/src/main/java/org/elasticsearch/service/statsd/StatsdService.java index 076c224..1c50a8c 100644 --- a/src/main/java/org/elasticsearch/service/statsd/StatsdService.java +++ b/src/main/java/org/elasticsearch/service/statsd/StatsdService.java @@ -32,6 +32,7 @@ public class StatsdService extends AbstractLifecycleComponent { private final Integer statsdPort; private final TimeValue statsdRefreshInternal; private final String statsdPrefix; + private final Boolean statsdReportShards; private final StatsDClient statsdClient; private volatile Thread statsdReporterThread; @@ -55,6 +56,9 @@ public StatsdService(Settings settings, ClusterService clusterService, IndicesSe this.statsdPrefix = settings.get( "metrics.statsd.prefix", "elasticsearch" + "." + settings.get("cluster.name") ); + this.statsdReportShards = settings.getAsBoolean( + "metrics.statsd.report_shards", true + ); this.statsdClient = new NonBlockingStatsDClient(this.statsdPrefix, this.statsdHost, this.statsdPort); } @@ -130,13 +134,17 @@ public void run() { private List getIndexShards(IndicesService indicesService) { List indexShards = Lists.newArrayList(); - String[] indices = indicesService.indices().toArray(new String[] {}); - for (String indexName : indices) { - IndexService indexService = indicesService.indexServiceSafe(indexName); - for (int shardId : indexService.shardIds()) { - indexShards.add(indexService.shard(shardId)); + + if ( StatsdService.this.statsdReportShards ) { + String[] indices = indicesService.indices().toArray(new String[] {}); + for (String indexName : indices) { + IndexService indexService = indicesService.indexServiceSafe(indexName); + for (int shardId : indexService.shardIds()) { + indexShards.add(indexService.shard(shardId)); + } } } + return indexShards; } } From 77a384d65c6b52abb5c2634c50dd52e26d8eff6f Mon Sep 17 00:00:00 2001 From: Xiao Yu Date: Wed, 6 Aug 2014 00:01:28 -0400 Subject: [PATCH 05/48] Fix Whitespace -- spaces to tabs --- .../service/statsd/StatsdReporter.java | 18 +- .../service/statsd/StatsdService.java | 246 +++++++++--------- 2 files changed, 132 insertions(+), 132 deletions(-) diff --git a/src/main/java/org/elasticsearch/service/statsd/StatsdReporter.java b/src/main/java/org/elasticsearch/service/statsd/StatsdReporter.java index 22a99b0..c02c0d2 100644 --- a/src/main/java/org/elasticsearch/service/statsd/StatsdReporter.java +++ b/src/main/java/org/elasticsearch/service/statsd/StatsdReporter.java @@ -195,13 +195,13 @@ private void sendNodeJvmStats(JvmStats jvmStats) // garbage collectors long gcCounter = 0; - long gcTimes = 0; - for (GarbageCollector gc : jvmStats.gc()) { - gcCounter += gc.collectionCount(); - gcTimes += gc.collectionTime().getSeconds(); - } - this.sendCount(type + ".gc", "collectionCount", gcCounter); - this.sendTime(type + ".gc", "collectionTimeSeconds", gcTimes); + long gcTimes = 0; + for (GarbageCollector gc : jvmStats.gc()) { + gcCounter += gc.collectionCount(); + gcTimes += gc.collectionTime().getSeconds(); + } + this.sendCount(type + ".gc", "collectionCount", gcCounter); + this.sendTime(type + ".gc", "collectionTimeSeconds", gcTimes); for (JvmStats.GarbageCollector collector : jvmStats.gc().collectors()) { String id = type + ".gc." + collector.name(); this.sendCount(id, "collectionCount", collector.collectionCount()); @@ -414,8 +414,8 @@ protected String buildMetricName(String name) private String join(String... parts) { if (parts == null) { - return null; - } + return null; + } StringBuilder builder = new StringBuilder(); for (int i = 0; i < parts.length; i++) { builder.append(parts[i]); diff --git a/src/main/java/org/elasticsearch/service/statsd/StatsdService.java b/src/main/java/org/elasticsearch/service/statsd/StatsdService.java index 1c50a8c..a53fb7c 100644 --- a/src/main/java/org/elasticsearch/service/statsd/StatsdService.java +++ b/src/main/java/org/elasticsearch/service/statsd/StatsdService.java @@ -25,127 +25,127 @@ public class StatsdService extends AbstractLifecycleComponent { - private final ClusterService clusterService; - private final IndicesService indicesService; - private NodeService nodeService; - private final String statsdHost; - private final Integer statsdPort; - private final TimeValue statsdRefreshInternal; - private final String statsdPrefix; - private final Boolean statsdReportShards; - private final StatsDClient statsdClient; - - private volatile Thread statsdReporterThread; - private volatile boolean closed; - - @Inject - public StatsdService(Settings settings, ClusterService clusterService, IndicesService indicesService, NodeService nodeService) { - super(settings); - this.clusterService = clusterService; - this.indicesService = indicesService; - this.nodeService = nodeService; - this.statsdRefreshInternal = settings.getAsTime( - "metrics.statsd.every", TimeValue.timeValueMinutes(1) - ); - this.statsdHost = settings.get( - "metrics.statsd.host" - ); - this.statsdPort = settings.getAsInt( - "metrics.statsd.port", 8125 - ); - this.statsdPrefix = settings.get( - "metrics.statsd.prefix", "elasticsearch" + "." + settings.get("cluster.name") - ); - this.statsdReportShards = settings.getAsBoolean( - "metrics.statsd.report_shards", true - ); - this.statsdClient = new NonBlockingStatsDClient(this.statsdPrefix, this.statsdHost, this.statsdPort); - } - - @Override - protected void doStart() throws ElasticsearchException { - if (this.statsdHost != null && this.statsdHost.length() > 0) { - this.statsdReporterThread = EsExecutors - .daemonThreadFactory(this.settings, "statsd_reporter") - .newThread(new StatsdReporterThread()); - this.statsdReporterThread.start(); - this.logger.info( - "StatsD reporting triggered every [{}] to host [{}:{}] with metric prefix [{}]", - this.statsdRefreshInternal, this.statsdHost, this.statsdPort, this.statsdPrefix - ); - } else { - this.logger.error( - "StatsD reporting disabled, no statsd host configured" - ); - } - } - - @Override - protected void doStop() throws ElasticsearchException { - if (this.closed) { - return; - } - if (this.statsdReporterThread != null) { - this.statsdReporterThread.interrupt(); - } - this.closed = true; - this.logger.info("StatsD reporter stopped"); - } - - @Override - protected void doClose() throws ElasticsearchException { - } - - public class StatsdReporterThread implements Runnable { - - @Override - public void run() { - while (!StatsdService.this.closed) { - DiscoveryNode node = StatsdService.this.clusterService.localNode(); - boolean isClusterStarted = StatsdService.this.clusterService.lifecycleState() - .equals(Lifecycle.State.STARTED); - - if (isClusterStarted && node != null && node.isMasterNode()) { - NodeIndicesStats nodeIndicesStats = StatsdService.this.indicesService.stats(false); - CommonStatsFlags commonStatsFlags = new CommonStatsFlags().clear(); - NodeStats nodeStats = StatsdService.this.nodeService.stats(commonStatsFlags, true, true, - true, true, true, true, true, - true, false); - List indexShards = this.getIndexShards(StatsdService.this.indicesService); - - StatsdReporter statsdReporter = new StatsdReporter(nodeIndicesStats, indexShards, - nodeStats, StatsdService.this.statsdClient); - statsdReporter.run(); - } else { - if (node != null) { - StatsdService.this.logger - .debug("[{}]/[{}] is not master node, not triggering update", node.getId(), - node.getName()); - } - } - - try { - Thread.sleep(StatsdService.this.statsdRefreshInternal.millis()); - } catch (InterruptedException e1) { - continue; - } - } - } - - private List getIndexShards(IndicesService indicesService) { - List indexShards = Lists.newArrayList(); - - if ( StatsdService.this.statsdReportShards ) { - String[] indices = indicesService.indices().toArray(new String[] {}); - for (String indexName : indices) { - IndexService indexService = indicesService.indexServiceSafe(indexName); - for (int shardId : indexService.shardIds()) { - indexShards.add(indexService.shard(shardId)); - } - } - } - - return indexShards; - } - } + private final ClusterService clusterService; + private final IndicesService indicesService; + private NodeService nodeService; + private final String statsdHost; + private final Integer statsdPort; + private final TimeValue statsdRefreshInternal; + private final String statsdPrefix; + private final Boolean statsdReportShards; + private final StatsDClient statsdClient; + + private volatile Thread statsdReporterThread; + private volatile boolean closed; + + @Inject + public StatsdService(Settings settings, ClusterService clusterService, IndicesService indicesService, NodeService nodeService) { + super(settings); + this.clusterService = clusterService; + this.indicesService = indicesService; + this.nodeService = nodeService; + this.statsdRefreshInternal = settings.getAsTime( + "metrics.statsd.every", TimeValue.timeValueMinutes(1) + ); + this.statsdHost = settings.get( + "metrics.statsd.host" + ); + this.statsdPort = settings.getAsInt( + "metrics.statsd.port", 8125 + ); + this.statsdPrefix = settings.get( + "metrics.statsd.prefix", "elasticsearch" + "." + settings.get("cluster.name") + ); + this.statsdReportShards = settings.getAsBoolean( + "metrics.statsd.report_shards", true + ); + this.statsdClient = new NonBlockingStatsDClient(this.statsdPrefix, this.statsdHost, this.statsdPort); + } + + @Override + protected void doStart() throws ElasticsearchException { + if (this.statsdHost != null && this.statsdHost.length() > 0) { + this.statsdReporterThread = EsExecutors + .daemonThreadFactory(this.settings, "statsd_reporter") + .newThread(new StatsdReporterThread()); + this.statsdReporterThread.start(); + this.logger.info( + "StatsD reporting triggered every [{}] to host [{}:{}] with metric prefix [{}]", + this.statsdRefreshInternal, this.statsdHost, this.statsdPort, this.statsdPrefix + ); + } else { + this.logger.error( + "StatsD reporting disabled, no statsd host configured" + ); + } + } + + @Override + protected void doStop() throws ElasticsearchException { + if (this.closed) { + return; + } + if (this.statsdReporterThread != null) { + this.statsdReporterThread.interrupt(); + } + this.closed = true; + this.logger.info("StatsD reporter stopped"); + } + + @Override + protected void doClose() throws ElasticsearchException { + } + + public class StatsdReporterThread implements Runnable { + + @Override + public void run() { + while (!StatsdService.this.closed) { + DiscoveryNode node = StatsdService.this.clusterService.localNode(); + boolean isClusterStarted = StatsdService.this.clusterService.lifecycleState() + .equals(Lifecycle.State.STARTED); + + if (isClusterStarted && node != null && node.isMasterNode()) { + NodeIndicesStats nodeIndicesStats = StatsdService.this.indicesService.stats(false); + CommonStatsFlags commonStatsFlags = new CommonStatsFlags().clear(); + NodeStats nodeStats = StatsdService.this.nodeService.stats(commonStatsFlags, true, true, + true, true, true, true, true, + true, false); + List indexShards = this.getIndexShards(StatsdService.this.indicesService); + + StatsdReporter statsdReporter = new StatsdReporter(nodeIndicesStats, indexShards, + nodeStats, StatsdService.this.statsdClient); + statsdReporter.run(); + } else { + if (node != null) { + StatsdService.this.logger + .debug("[{}]/[{}] is not master node, not triggering update", node.getId(), + node.getName()); + } + } + + try { + Thread.sleep(StatsdService.this.statsdRefreshInternal.millis()); + } catch (InterruptedException e1) { + continue; + } + } + } + + private List getIndexShards(IndicesService indicesService) { + List indexShards = Lists.newArrayList(); + + if ( StatsdService.this.statsdReportShards ) { + String[] indices = indicesService.indices().toArray(new String[] {}); + for (String indexName : indices) { + IndexService indexService = indicesService.indexServiceSafe(indexName); + for (int shardId : indexService.shardIds()) { + indexShards.add(indexService.shard(shardId)); + } + } + } + + return indexShards; + } + } } From d474c1610336f16057411538a99d62804fedbf09 Mon Sep 17 00:00:00 2001 From: Xiao Yu Date: Tue, 5 Aug 2014 23:55:03 -0400 Subject: [PATCH 06/48] Add comments to args list, cleanup debug output --- .../service/statsd/StatsdService.java | 55 ++++++++++++------- 1 file changed, 36 insertions(+), 19 deletions(-) diff --git a/src/main/java/org/elasticsearch/service/statsd/StatsdService.java b/src/main/java/org/elasticsearch/service/statsd/StatsdService.java index a53fb7c..c01377a 100644 --- a/src/main/java/org/elasticsearch/service/statsd/StatsdService.java +++ b/src/main/java/org/elasticsearch/service/statsd/StatsdService.java @@ -102,25 +102,42 @@ public class StatsdReporterThread implements Runnable { public void run() { while (!StatsdService.this.closed) { DiscoveryNode node = StatsdService.this.clusterService.localNode(); - boolean isClusterStarted = StatsdService.this.clusterService.lifecycleState() - .equals(Lifecycle.State.STARTED); - - if (isClusterStarted && node != null && node.isMasterNode()) { - NodeIndicesStats nodeIndicesStats = StatsdService.this.indicesService.stats(false); - CommonStatsFlags commonStatsFlags = new CommonStatsFlags().clear(); - NodeStats nodeStats = StatsdService.this.nodeService.stats(commonStatsFlags, true, true, - true, true, true, true, true, - true, false); - List indexShards = this.getIndexShards(StatsdService.this.indicesService); - - StatsdReporter statsdReporter = new StatsdReporter(nodeIndicesStats, indexShards, - nodeStats, StatsdService.this.statsdClient); - statsdReporter.run(); - } else { - if (node != null) { - StatsdService.this.logger - .debug("[{}]/[{}] is not master node, not triggering update", node.getId(), - node.getName()); + boolean isClusterStarted = StatsdService.this.clusterService + .lifecycleState() + .equals(Lifecycle.State.STARTED); + + if (node != null && isClusterStarted) { + // Master Node sends cluster wide stats + if (node.isMasterNode()) { + NodeIndicesStats nodeIndicesStats = StatsdService.this.indicesService.stats( + false // includePrevious + ); + NodeStats nodeStats = StatsdService.this.nodeService.stats( + new CommonStatsFlags().clear(), // indices + true, // os + true, // process + true, // jvm + true, // threadPool + true, // network + true, // fs + true, // transport + true, // http + false // circuitBreaker + ); + List indexShards = this.getIndexShards(StatsdService.this.indicesService); + + StatsdReporter statsdReporter = new StatsdReporter( + nodeIndicesStats, + indexShards, + nodeStats, + StatsdService.this.statsdClient + ); + statsdReporter.run(); + } else { + StatsdService.this.logger.debug( + "[{}]/[{}] is not master node, not triggering update", + node.getId(), node.getName() + ); } } From 0cdc88412fb75e4fb338f5c9d82422b455b6b826 Mon Sep 17 00:00:00 2001 From: Xiao Yu Date: Wed, 6 Aug 2014 11:44:15 -0400 Subject: [PATCH 07/48] Refactor reporters --- .../service/statsd/StatsdReporter.java | 409 +----------------- .../service/statsd/StatsdReporterIndices.java | 134 ++++++ .../StatsdReporterNodeIndicesStats.java | 126 ++++++ .../statsd/StatsdReporterNodeStats.java | 210 +++++++++ .../service/statsd/StatsdService.java | 58 ++- 5 files changed, 522 insertions(+), 415 deletions(-) create mode 100644 src/main/java/org/elasticsearch/service/statsd/StatsdReporterIndices.java create mode 100644 src/main/java/org/elasticsearch/service/statsd/StatsdReporterNodeIndicesStats.java create mode 100644 src/main/java/org/elasticsearch/service/statsd/StatsdReporterNodeStats.java diff --git a/src/main/java/org/elasticsearch/service/statsd/StatsdReporter.java b/src/main/java/org/elasticsearch/service/statsd/StatsdReporter.java index c02c0d2..bec514e 100644 --- a/src/main/java/org/elasticsearch/service/statsd/StatsdReporter.java +++ b/src/main/java/org/elasticsearch/service/statsd/StatsdReporter.java @@ -1,418 +1,44 @@ package org.elasticsearch.service.statsd; -import java.util.Iterator; -import java.util.List; -import java.util.Map; - -import org.elasticsearch.action.admin.cluster.node.stats.NodeStats; import org.elasticsearch.common.logging.ESLogger; import org.elasticsearch.common.logging.ESLoggerFactory; -import org.elasticsearch.http.HttpStats; -import org.elasticsearch.index.cache.filter.FilterCacheStats; -import org.elasticsearch.index.cache.id.IdCacheStats; -import org.elasticsearch.index.flush.FlushStats; -import org.elasticsearch.index.get.GetStats; -import org.elasticsearch.index.indexing.IndexingStats; -import org.elasticsearch.index.merge.MergeStats; -import org.elasticsearch.index.refresh.RefreshStats; -import org.elasticsearch.index.search.stats.SearchStats; -import org.elasticsearch.index.shard.DocsStats; -import org.elasticsearch.index.shard.service.IndexShard; -import org.elasticsearch.index.store.StoreStats; -import org.elasticsearch.index.warmer.WarmerStats; -import org.elasticsearch.indices.NodeIndicesStats; -import org.elasticsearch.monitor.fs.FsStats; -import org.elasticsearch.monitor.jvm.JvmStats; -import org.elasticsearch.monitor.jvm.JvmStats.GarbageCollector; -import org.elasticsearch.monitor.network.NetworkStats; -import org.elasticsearch.monitor.os.OsStats; -import org.elasticsearch.monitor.process.ProcessStats; -import org.elasticsearch.threadpool.ThreadPoolStats; -import org.elasticsearch.transport.TransportStats; import com.timgroup.statsd.StatsDClient; -public class StatsdReporter { - - private static final String DEFAULT_JOINER = "."; - private static final ESLogger logger = ESLoggerFactory.getLogger(StatsdReporter.class.getName()); - private List indexShards; - private NodeStats nodeStats; - private final NodeIndicesStats nodeIndicesStats; - private final StatsDClient statsdClient; - private String joiner = DEFAULT_JOINER; +public abstract class StatsdReporter { - public StatsdReporter(NodeIndicesStats nodeIndicesStats, List indexShards, NodeStats nodeStats, - StatsDClient statsdClient) - { + private static final String DEFAULT_JOINER = "."; + private static final ESLogger logger = ESLoggerFactory.getLogger(StatsdReporter.class.getName()); + private StatsDClient statsdClient; - this.indexShards = indexShards; - this.nodeStats = nodeStats; - this.nodeIndicesStats = nodeIndicesStats; + public StatsdReporter setStatsDClient(StatsDClient statsdClient) { this.statsdClient = statsdClient; + return this; } - public void run() - { - try { - this.sendNodeIndicesStats(); - this.sendIndexShardStats(); - this.sendNodeStats(); - } - catch (Exception e) { - this.logException(e); - } - } - - private void sendNodeStats() - { - this.sendNodeFsStats(this.nodeStats.getFs()); - this.sendNodeHttpStats(this.nodeStats.getHttp()); - this.sendNodeJvmStats(this.nodeStats.getJvm()); - this.sendNodeNetworkStats(this.nodeStats.getNetwork()); - this.sendNodeOsStats(this.nodeStats.getOs()); - this.sendNodeProcessStats(this.nodeStats.getProcess()); - this.sendNodeTransportStats(this.nodeStats.getTransport()); - this.sendNodeThreadPoolStats(this.nodeStats.getThreadPool()); - } - - private void sendNodeThreadPoolStats(ThreadPoolStats threadPoolStats) - { - String type = this.buildMetricName("node.threadpool"); - Iterator statsIterator = threadPoolStats.iterator(); - while (statsIterator.hasNext()) { - ThreadPoolStats.Stats stats = statsIterator.next(); - String id = type + "." + stats.getName(); - - this.sendGauge(id, "threads", stats.getThreads()); - this.sendGauge(id, "queue", stats.getQueue()); - this.sendGauge(id, "active", stats.getActive()); - this.sendGauge(id, "rejected", stats.getRejected()); - this.sendGauge(id, "largest", stats.getLargest()); - this.sendGauge(id, "completed", stats.getCompleted()); - } - } - - private void sendNodeTransportStats(TransportStats transportStats) - { - String type = this.buildMetricName("node.transport"); - this.sendGauge(type, "serverOpen", transportStats.serverOpen()); - this.sendCount(type, "rxCount", transportStats.rxCount()); - this.sendCount(type, "rxSizeBytes", transportStats.rxSize().bytes()); - this.sendCount(type, "txCount", transportStats.txCount()); - this.sendCount(type, "txSizeBytes", transportStats.txSize().bytes()); - } - - private void sendNodeProcessStats(ProcessStats processStats) - { - String type = this.buildMetricName("node.process"); - - this.sendGauge(type, "openFileDescriptors", processStats.openFileDescriptors()); - if (processStats.cpu() != null) { - this.sendGauge(type + ".cpu", "percent", processStats.cpu().percent()); - this.sendGauge(type + ".cpu", "sysSeconds", processStats.cpu().sys().seconds()); - this.sendGauge(type + ".cpu", "totalSeconds", processStats.cpu().total().seconds()); - this.sendGauge(type + ".cpu", "userSeconds", processStats.cpu().user().seconds()); - } - - if (processStats.mem() != null) { - this.sendGauge(type + ".mem", "totalVirtual", processStats.mem().totalVirtual().bytes()); - this.sendGauge(type + ".mem", "resident", processStats.mem().resident().bytes()); - this.sendGauge(type + ".mem", "share", processStats.mem().share().bytes()); - } - } - - private void sendNodeOsStats(OsStats osStats) - { - String type = this.buildMetricName("node.os"); - - if (osStats.cpu() != null) { - this.sendGauge(type + ".cpu", "sys", osStats.cpu().sys()); - this.sendGauge(type + ".cpu", "idle", osStats.cpu().idle()); - this.sendGauge(type + ".cpu", "user", osStats.cpu().user()); - } - - if (osStats.mem() != null) { - this.sendGauge(type + ".mem", "freeBytes", osStats.mem().free().bytes()); - this.sendGauge(type + ".mem", "usedBytes", osStats.mem().used().bytes()); - this.sendGauge(type + ".mem", "freePercent", osStats.mem().freePercent()); - this.sendGauge(type + ".mem", "usedPercent", osStats.mem().usedPercent()); - this.sendGauge(type + ".mem", "actualFreeBytes", osStats.mem().actualFree().bytes()); - this.sendGauge(type + ".mem", "actualUsedBytes", osStats.mem().actualUsed().bytes()); - } - - if (osStats.swap() != null) { - this.sendGauge(type + ".swap", "freeBytes", osStats.swap().free().bytes()); - this.sendGauge(type + ".swap", "usedBytes", osStats.swap().used().bytes()); - } - } - - private void sendNodeNetworkStats(NetworkStats networkStats) - { - String type = this.buildMetricName("node.network.tcp"); - NetworkStats.Tcp tcp = networkStats.tcp(); - - // might be null, if sigar isnt loaded - if (tcp != null) { - this.sendGauge(type, "activeOpens", tcp.activeOpens()); - this.sendGauge(type, "passiveOpens", tcp.passiveOpens()); - this.sendGauge(type, "attemptFails", tcp.attemptFails()); - this.sendGauge(type, "estabResets", tcp.estabResets()); - this.sendGauge(type, "currEstab", tcp.currEstab()); - this.sendGauge(type, "inSegs", tcp.inSegs()); - this.sendGauge(type, "outSegs", tcp.outSegs()); - this.sendGauge(type, "retransSegs", tcp.retransSegs()); - this.sendGauge(type, "inErrs", tcp.inErrs()); - this.sendGauge(type, "outRsts", tcp.outRsts()); - } - } - - private void sendNodeJvmStats(JvmStats jvmStats) - { - String type = this.buildMetricName("node.jvm"); - this.sendGauge(type, "uptime", jvmStats.uptime().seconds()); - - // mem - this.sendGauge(type + ".mem", "heapCommitted", jvmStats.mem().heapCommitted().bytes()); - this.sendGauge(type + ".mem", "heapUsed", jvmStats.mem().heapUsed().bytes()); - this.sendGauge(type + ".mem", "nonHeapCommitted", jvmStats.mem().nonHeapCommitted().bytes()); - this.sendGauge(type + ".mem", "nonHeapUsed", jvmStats.mem().nonHeapUsed().bytes()); - - Iterator memoryPoolIterator = jvmStats.mem().iterator(); - while (memoryPoolIterator.hasNext()) { - JvmStats.MemoryPool memoryPool = memoryPoolIterator.next(); - String memoryPoolType = type + ".mem.pool." + memoryPool.name(); - - this.sendGauge(memoryPoolType, "max", memoryPool.max().bytes()); - this.sendGauge(memoryPoolType, "used", memoryPool.used().bytes()); - this.sendGauge(memoryPoolType, "peakUsed", memoryPool.peakUsed().bytes()); - this.sendGauge(memoryPoolType, "peakMax", memoryPool.peakMax().bytes()); - } - - // threads - this.sendGauge(type + ".threads", "count", jvmStats.threads().count()); - this.sendGauge(type + ".threads", "peakCount", jvmStats.threads().peakCount()); - - // garbage collectors - long gcCounter = 0; - long gcTimes = 0; - for (GarbageCollector gc : jvmStats.gc()) { - gcCounter += gc.collectionCount(); - gcTimes += gc.collectionTime().getSeconds(); - } - this.sendCount(type + ".gc", "collectionCount", gcCounter); - this.sendTime(type + ".gc", "collectionTimeSeconds", gcTimes); - for (JvmStats.GarbageCollector collector : jvmStats.gc().collectors()) { - String id = type + ".gc." + collector.name(); - this.sendCount(id, "collectionCount", collector.collectionCount()); - this.sendTime(id, "collectionTimeSeconds", collector.collectionTime().seconds()); - - JvmStats.GarbageCollector.LastGc lastGc = collector.lastGc(); - String lastGcType = type + ".lastGc"; - if (lastGc != null) { - this.sendTime(lastGcType, "time", lastGc.endTime() - lastGc.startTime()); - this.sendGauge(lastGcType, "max", lastGc.max().bytes()); - this.sendGauge(lastGcType, "beforeUsed", lastGc.beforeUsed().bytes()); - this.sendGauge(lastGcType, "afterUsed", lastGc.afterUsed().bytes()); - this.sendGauge(lastGcType, "durationSeconds", lastGc.duration().seconds()); - } - } - } - - private void sendNodeHttpStats(HttpStats httpStats) - { - String type = this.buildMetricName("node.http"); - this.sendGauge(type, "serverOpen", httpStats.getServerOpen()); - this.sendGauge(type, "totalOpen", httpStats.getTotalOpen()); - } - - private void sendNodeFsStats(FsStats fs) - { - Iterator infoIterator = fs.iterator(); - int i = 0; - while (infoIterator.hasNext()) { - String type = this.buildMetricName("node.fs") + i; - FsStats.Info info = infoIterator.next(); - this.sendGauge(type, "available", info.getAvailable().bytes()); - this.sendGauge(type, "total", info.getTotal().bytes()); - this.sendGauge(type, "free", info.getFree().bytes()); - this.sendCount(type, "diskReads", info.getDiskReads()); - this.sendCount(type, "diskReadsInBytes", info.getDiskReadSizeInBytes()); - this.sendCount(type, "diskWrites", info.getDiskWrites()); - this.sendCount(type, "diskWritesInBytes", info.getDiskWriteSizeInBytes()); - this.sendGauge(type, "diskQueue", (long) info.getDiskQueue()); - this.sendGauge(type, "diskService", (long) info.getDiskServiceTime()); - i++; - } - } - - private void sendIndexShardStats() - { - for (IndexShard indexShard : this.indexShards) { - String type = this.buildMetricName("indexes.") + indexShard.shardId().index().name() + ".id." + indexShard.shardId().id(); - this.sendIndexShardStats(type, indexShard); - } - } - - private void sendIndexShardStats(String type, IndexShard indexShard) - { - this.sendSearchStats(type + ".search", indexShard.searchStats()); - this.sendGetStats(type + ".get", indexShard.getStats()); - this.sendDocsStats(type + ".docs", indexShard.docStats()); - this.sendRefreshStats(type + ".refresh", indexShard.refreshStats()); - this.sendIndexingStats(type + ".indexing", indexShard.indexingStats("_all")); - this.sendMergeStats(type + ".merge", indexShard.mergeStats()); - this.sendWarmerStats(type + ".warmer", indexShard.warmerStats()); - this.sendStoreStats(type + ".store", indexShard.storeStats()); - } - - private void sendStoreStats(String type, StoreStats storeStats) - { - this.sendGauge(type, "sizeInBytes", storeStats.sizeInBytes()); - this.sendGauge(type, "throttleTimeInNanos", storeStats.throttleTime().getNanos()); - } - - private void sendWarmerStats(String type, WarmerStats warmerStats) - { - this.sendGauge(type, "current", warmerStats.current()); - this.sendGauge(type, "total", warmerStats.total()); - this.sendTime(type, "totalTimeInMillis", warmerStats.totalTimeInMillis()); - } - - private void sendMergeStats(String type, MergeStats mergeStats) - { - this.sendGauge(type, "total", mergeStats.getTotal()); - this.sendTime(type, "totalTimeInMillis", mergeStats.getTotalTimeInMillis()); - this.sendGauge(type, "totalNumDocs", mergeStats.getTotalNumDocs()); - this.sendGauge(type, "current", mergeStats.getCurrent()); - this.sendGauge(type, "currentNumDocs", mergeStats.getCurrentNumDocs()); - this.sendGauge(type, "currentSizeInBytes", mergeStats.getCurrentSizeInBytes()); - } - - private void sendNodeIndicesStats() - { - String type = this.buildMetricName("node"); - this.sendFilterCacheStats(type + ".filtercache", this.nodeIndicesStats.getFilterCache()); - this.sendIdCacheStats(type + ".idcache", this.nodeIndicesStats.getIdCache()); - this.sendDocsStats(type + ".docs", this.nodeIndicesStats.getDocs()); - this.sendFlushStats(type + ".flush", this.nodeIndicesStats.getFlush()); - this.sendGetStats(type + ".get", this.nodeIndicesStats.getGet()); - this.sendIndexingStats(type + ".indexing", this.nodeIndicesStats.getIndexing()); - this.sendRefreshStats(type + ".refresh", this.nodeIndicesStats.getRefresh()); - this.sendSearchStats(type + ".search", this.nodeIndicesStats.getSearch()); - } - - private void sendSearchStats(String type, SearchStats searchStats) - { - SearchStats.Stats totalSearchStats = searchStats.getTotal(); - this.sendSearchStatsStats(type + "._all", totalSearchStats); - - if (searchStats.getGroupStats() != null) { - for (Map.Entry statsEntry : searchStats.getGroupStats().entrySet()) { - this.sendSearchStatsStats(type + "." + statsEntry.getKey(), statsEntry.getValue()); - } - } - } - - private void sendSearchStatsStats(String group, SearchStats.Stats searchStats) - { - String type = this.buildMetricName("search.stats.") + group; - this.sendCount(type, "queryCount", searchStats.getQueryCount()); - this.sendCount(type, "queryTimeInMillis", searchStats.getQueryTimeInMillis()); - this.sendGauge(type, "queryCurrent", searchStats.getQueryCurrent()); - this.sendCount(type, "fetchCount", searchStats.getFetchCount()); - this.sendCount(type, "fetchTimeInMillis", searchStats.getFetchTimeInMillis()); - this.sendGauge(type, "fetchCurrent", searchStats.getFetchCurrent()); - } - - private void sendRefreshStats(String type, RefreshStats refreshStats) - { - this.sendCount(type, "total", refreshStats.getTotal()); - this.sendCount(type, "totalTimeInMillis", refreshStats.getTotalTimeInMillis()); - } - - private void sendIndexingStats(String type, IndexingStats indexingStats) - { - IndexingStats.Stats totalStats = indexingStats.getTotal(); - this.sendStats(type + "._all", totalStats); - - Map typeStats = indexingStats.getTypeStats(); - if (typeStats != null) { - for (Map.Entry statsEntry : typeStats.entrySet()) { - this.sendStats(type + "." + statsEntry.getKey(), statsEntry.getValue()); - } - } - } - - private void sendStats(String type, IndexingStats.Stats stats) - { - this.sendCount(type, "indexCount", stats.getIndexCount()); - this.sendCount(type, "indexTimeInMillis", stats.getIndexTimeInMillis()); - this.sendGauge(type, "indexCurrent", stats.getIndexCount()); - this.sendCount(type, "deleteCount", stats.getDeleteCount()); - this.sendCount(type, "deleteTimeInMillis", stats.getDeleteTimeInMillis()); - this.sendGauge(type, "deleteCurrent", stats.getDeleteCurrent()); - } - - private void sendGetStats(String type, GetStats getStats) - { - this.sendCount(type, "existsCount", getStats.getExistsCount()); - this.sendCount(type, "existsTimeInMillis", getStats.getExistsTimeInMillis()); - this.sendCount(type, "missingCount", getStats.getMissingCount()); - this.sendCount(type, "missingTimeInMillis", getStats.getMissingTimeInMillis()); - this.sendGauge(type, "current", getStats.current()); - } - - private void sendFlushStats(String type, FlushStats flush) - { - this.sendCount(type, "total", flush.getTotal()); - this.sendCount(type, "totalTimeInMillis", flush.getTotalTimeInMillis()); - } - - private void sendDocsStats(String name, DocsStats docsStats) - { - this.sendCount(name, "count", docsStats.getCount()); - this.sendCount(name, "deleted", docsStats.getDeleted()); - } - - private void sendIdCacheStats(String name, IdCacheStats idCache) - { - this.sendGauge(name, "memorySizeInBytes", idCache.getMemorySizeInBytes()); - } - - private void sendFilterCacheStats(String name, FilterCacheStats filterCache) - { - this.sendGauge(name, "memorySizeInBytes", filterCache.getMemorySizeInBytes()); - this.sendGauge(name, "evictions", filterCache.getEvictions()); - } + public abstract void run(); - protected void sendGauge(String name, String valueName, long value) - { + protected void sendGauge(String name, String valueName, long value) { this.statsdClient.gauge(this.join(name, valueName), (int) value); } - protected void sendCount(String name, String valueName, long value) - { + protected void sendCount(String name, String valueName, long value) { this.statsdClient.count(this.join(name, valueName), (int) value); } - protected void sendTime(String name, String valueName, long value) - { + protected void sendTime(String name, String valueName, long value) { this.statsdClient.time(this.join(name, valueName), (int) value); } - protected String sanitizeString(String s) - { + protected String sanitizeString(String s) { return s.replace(' ', '-'); } - protected String buildMetricName(String name) - { + protected String buildMetricName(String name) { return this.sanitizeString(name); } - private String join(String... parts) - { + private String join(String... parts) { if (parts == null) { return null; } @@ -420,19 +46,18 @@ private String join(String... parts) for (int i = 0; i < parts.length; i++) { builder.append(parts[i]); if (i < parts.length - 1) { - builder.append(this.joiner); + builder.append(this.DEFAULT_JOINER); } } return builder.toString(); } - private void logException(Exception e) - { + protected void logException(Exception e) { if (logger.isDebugEnabled()) { - logger.debug("Error writing to Statsd", e); + logger.debug("Error writing to StatsD", e); } else { - logger.warn("Error writing to Statsd: {}", e.getMessage()); + logger.warn("Error writing to StatsD: {}", e.getMessage()); } } } diff --git a/src/main/java/org/elasticsearch/service/statsd/StatsdReporterIndices.java b/src/main/java/org/elasticsearch/service/statsd/StatsdReporterIndices.java new file mode 100644 index 0000000..ebfc4ad --- /dev/null +++ b/src/main/java/org/elasticsearch/service/statsd/StatsdReporterIndices.java @@ -0,0 +1,134 @@ +package org.elasticsearch.service.statsd; + +import java.util.Iterator; +import java.util.List; +import java.util.Map; + +import org.elasticsearch.index.shard.service.IndexShard; +import org.elasticsearch.index.get.GetStats; +import org.elasticsearch.index.indexing.IndexingStats; +import org.elasticsearch.index.merge.MergeStats; +import org.elasticsearch.index.refresh.RefreshStats; +import org.elasticsearch.index.search.stats.SearchStats; +import org.elasticsearch.index.shard.DocsStats; +import org.elasticsearch.index.store.StoreStats; +import org.elasticsearch.index.warmer.WarmerStats; + +public class StatsdReporterIndices extends StatsdReporter { + + private final List indexShards; + + public StatsdReporterIndices(List indexShards) { + this.indexShards = indexShards; + } + + public void run() + { + try { + for (IndexShard indexShard : this.indexShards) { + String type = this.buildMetricName("indexes.") + indexShard.shardId().index().name() + ".id." + indexShard.shardId().id(); + this.sendSearchStats(type + ".search", indexShard.searchStats()); + this.sendGetStats(type + ".get", indexShard.getStats()); + this.sendDocsStats(type + ".docs", indexShard.docStats()); + this.sendRefreshStats(type + ".refresh", indexShard.refreshStats()); + this.sendIndexingStats(type + ".indexing", indexShard.indexingStats("_all")); + this.sendMergeStats(type + ".merge", indexShard.mergeStats()); + this.sendWarmerStats(type + ".warmer", indexShard.warmerStats()); + this.sendStoreStats(type + ".store", indexShard.storeStats()); + } + } + catch (Exception e) { + this.logException(e); + } + } + + private void sendStoreStats(String type, StoreStats storeStats) + { + this.sendGauge(type, "sizeInBytes", storeStats.sizeInBytes()); + this.sendGauge(type, "throttleTimeInNanos", storeStats.throttleTime().getNanos()); + } + + private void sendWarmerStats(String type, WarmerStats warmerStats) + { + this.sendGauge(type, "current", warmerStats.current()); + this.sendGauge(type, "total", warmerStats.total()); + this.sendTime(type, "totalTimeInMillis", warmerStats.totalTimeInMillis()); + } + + private void sendMergeStats(String type, MergeStats mergeStats) + { + this.sendGauge(type, "total", mergeStats.getTotal()); + this.sendTime(type, "totalTimeInMillis", mergeStats.getTotalTimeInMillis()); + this.sendGauge(type, "totalNumDocs", mergeStats.getTotalNumDocs()); + this.sendGauge(type, "current", mergeStats.getCurrent()); + this.sendGauge(type, "currentNumDocs", mergeStats.getCurrentNumDocs()); + this.sendGauge(type, "currentSizeInBytes", mergeStats.getCurrentSizeInBytes()); + } + + private void sendSearchStats(String type, SearchStats searchStats) + { + SearchStats.Stats totalSearchStats = searchStats.getTotal(); + this.sendSearchStatsStats(type + "._all", totalSearchStats); + + if (searchStats.getGroupStats() != null) { + for (Map.Entry statsEntry : searchStats.getGroupStats().entrySet()) { + this.sendSearchStatsStats(type + "." + statsEntry.getKey(), statsEntry.getValue()); + } + } + } + + private void sendSearchStatsStats(String group, SearchStats.Stats searchStats) + { + String type = this.buildMetricName("search.stats.") + group; + this.sendCount(type, "queryCount", searchStats.getQueryCount()); + this.sendCount(type, "queryTimeInMillis", searchStats.getQueryTimeInMillis()); + this.sendGauge(type, "queryCurrent", searchStats.getQueryCurrent()); + this.sendCount(type, "fetchCount", searchStats.getFetchCount()); + this.sendCount(type, "fetchTimeInMillis", searchStats.getFetchTimeInMillis()); + this.sendGauge(type, "fetchCurrent", searchStats.getFetchCurrent()); + } + + private void sendRefreshStats(String type, RefreshStats refreshStats) + { + this.sendCount(type, "total", refreshStats.getTotal()); + this.sendCount(type, "totalTimeInMillis", refreshStats.getTotalTimeInMillis()); + } + + private void sendIndexingStats(String type, IndexingStats indexingStats) + { + IndexingStats.Stats totalStats = indexingStats.getTotal(); + this.sendStats(type + "._all", totalStats); + + Map typeStats = indexingStats.getTypeStats(); + if (typeStats != null) { + for (Map.Entry statsEntry : typeStats.entrySet()) { + this.sendStats(type + "." + statsEntry.getKey(), statsEntry.getValue()); + } + } + } + + private void sendStats(String type, IndexingStats.Stats stats) + { + this.sendCount(type, "indexCount", stats.getIndexCount()); + this.sendCount(type, "indexTimeInMillis", stats.getIndexTimeInMillis()); + this.sendGauge(type, "indexCurrent", stats.getIndexCount()); + this.sendCount(type, "deleteCount", stats.getDeleteCount()); + this.sendCount(type, "deleteTimeInMillis", stats.getDeleteTimeInMillis()); + this.sendGauge(type, "deleteCurrent", stats.getDeleteCurrent()); + } + + private void sendGetStats(String type, GetStats getStats) + { + this.sendCount(type, "existsCount", getStats.getExistsCount()); + this.sendCount(type, "existsTimeInMillis", getStats.getExistsTimeInMillis()); + this.sendCount(type, "missingCount", getStats.getMissingCount()); + this.sendCount(type, "missingTimeInMillis", getStats.getMissingTimeInMillis()); + this.sendGauge(type, "current", getStats.current()); + } + + private void sendDocsStats(String name, DocsStats docsStats) + { + this.sendCount(name, "count", docsStats.getCount()); + this.sendCount(name, "deleted", docsStats.getDeleted()); + } +} diff --git a/src/main/java/org/elasticsearch/service/statsd/StatsdReporterNodeIndicesStats.java b/src/main/java/org/elasticsearch/service/statsd/StatsdReporterNodeIndicesStats.java new file mode 100644 index 0000000..b4e4ac9 --- /dev/null +++ b/src/main/java/org/elasticsearch/service/statsd/StatsdReporterNodeIndicesStats.java @@ -0,0 +1,126 @@ +package org.elasticsearch.service.statsd; + +import java.util.Iterator; +import java.util.List; +import java.util.Map; + +import org.elasticsearch.indices.NodeIndicesStats; +import org.elasticsearch.index.cache.filter.FilterCacheStats; +import org.elasticsearch.index.cache.id.IdCacheStats; +import org.elasticsearch.index.flush.FlushStats; +import org.elasticsearch.index.get.GetStats; +import org.elasticsearch.index.indexing.IndexingStats; +import org.elasticsearch.index.refresh.RefreshStats; +import org.elasticsearch.index.search.stats.SearchStats; +import org.elasticsearch.index.shard.DocsStats; + +public class StatsdReporterNodeIndicesStats extends StatsdReporter { + + private final NodeIndicesStats nodeIndicesStats; + + public StatsdReporterNodeIndicesStats(NodeIndicesStats nodeIndicesStats) { + this.nodeIndicesStats = nodeIndicesStats; + } + + public void run() + { + try { + String type = this.buildMetricName("node"); + this.sendFilterCacheStats(type + ".filtercache", this.nodeIndicesStats.getFilterCache()); + this.sendIdCacheStats(type + ".idcache", this.nodeIndicesStats.getIdCache()); + this.sendDocsStats(type + ".docs", this.nodeIndicesStats.getDocs()); + this.sendFlushStats(type + ".flush", this.nodeIndicesStats.getFlush()); + this.sendGetStats(type + ".get", this.nodeIndicesStats.getGet()); + this.sendIndexingStats(type + ".indexing", this.nodeIndicesStats.getIndexing()); + this.sendRefreshStats(type + ".refresh", this.nodeIndicesStats.getRefresh()); + this.sendSearchStats(type + ".search", this.nodeIndicesStats.getSearch()); + } + catch (Exception e) { + this.logException(e); + } + } + + private void sendSearchStats(String type, SearchStats searchStats) + { + SearchStats.Stats totalSearchStats = searchStats.getTotal(); + this.sendSearchStatsStats(type + "._all", totalSearchStats); + + if (searchStats.getGroupStats() != null) { + for (Map.Entry statsEntry : searchStats.getGroupStats().entrySet()) { + this.sendSearchStatsStats(type + "." + statsEntry.getKey(), statsEntry.getValue()); + } + } + } + + private void sendSearchStatsStats(String group, SearchStats.Stats searchStats) + { + String type = this.buildMetricName("search.stats.") + group; + this.sendCount(type, "queryCount", searchStats.getQueryCount()); + this.sendCount(type, "queryTimeInMillis", searchStats.getQueryTimeInMillis()); + this.sendGauge(type, "queryCurrent", searchStats.getQueryCurrent()); + this.sendCount(type, "fetchCount", searchStats.getFetchCount()); + this.sendCount(type, "fetchTimeInMillis", searchStats.getFetchTimeInMillis()); + this.sendGauge(type, "fetchCurrent", searchStats.getFetchCurrent()); + } + + private void sendRefreshStats(String type, RefreshStats refreshStats) + { + this.sendCount(type, "total", refreshStats.getTotal()); + this.sendCount(type, "totalTimeInMillis", refreshStats.getTotalTimeInMillis()); + } + + private void sendIndexingStats(String type, IndexingStats indexingStats) + { + IndexingStats.Stats totalStats = indexingStats.getTotal(); + this.sendStats(type + "._all", totalStats); + + Map typeStats = indexingStats.getTypeStats(); + if (typeStats != null) { + for (Map.Entry statsEntry : typeStats.entrySet()) { + this.sendStats(type + "." + statsEntry.getKey(), statsEntry.getValue()); + } + } + } + + private void sendStats(String type, IndexingStats.Stats stats) + { + this.sendCount(type, "indexCount", stats.getIndexCount()); + this.sendCount(type, "indexTimeInMillis", stats.getIndexTimeInMillis()); + this.sendGauge(type, "indexCurrent", stats.getIndexCount()); + this.sendCount(type, "deleteCount", stats.getDeleteCount()); + this.sendCount(type, "deleteTimeInMillis", stats.getDeleteTimeInMillis()); + this.sendGauge(type, "deleteCurrent", stats.getDeleteCurrent()); + } + + private void sendGetStats(String type, GetStats getStats) + { + this.sendCount(type, "existsCount", getStats.getExistsCount()); + this.sendCount(type, "existsTimeInMillis", getStats.getExistsTimeInMillis()); + this.sendCount(type, "missingCount", getStats.getMissingCount()); + this.sendCount(type, "missingTimeInMillis", getStats.getMissingTimeInMillis()); + this.sendGauge(type, "current", getStats.current()); + } + + private void sendFlushStats(String type, FlushStats flush) + { + this.sendCount(type, "total", flush.getTotal()); + this.sendCount(type, "totalTimeInMillis", flush.getTotalTimeInMillis()); + } + + private void sendDocsStats(String name, DocsStats docsStats) + { + this.sendCount(name, "count", docsStats.getCount()); + this.sendCount(name, "deleted", docsStats.getDeleted()); + } + + private void sendIdCacheStats(String name, IdCacheStats idCache) + { + this.sendGauge(name, "memorySizeInBytes", idCache.getMemorySizeInBytes()); + } + + private void sendFilterCacheStats(String name, FilterCacheStats filterCache) + { + this.sendGauge(name, "memorySizeInBytes", filterCache.getMemorySizeInBytes()); + this.sendGauge(name, "evictions", filterCache.getEvictions()); + } +} diff --git a/src/main/java/org/elasticsearch/service/statsd/StatsdReporterNodeStats.java b/src/main/java/org/elasticsearch/service/statsd/StatsdReporterNodeStats.java new file mode 100644 index 0000000..b292333 --- /dev/null +++ b/src/main/java/org/elasticsearch/service/statsd/StatsdReporterNodeStats.java @@ -0,0 +1,210 @@ +package org.elasticsearch.service.statsd; + +import java.util.Iterator; +import java.util.List; +import java.util.Map; + +import org.elasticsearch.action.admin.cluster.node.stats.NodeStats; +import org.elasticsearch.monitor.fs.FsStats; +import org.elasticsearch.monitor.jvm.JvmStats; +import org.elasticsearch.monitor.jvm.JvmStats.GarbageCollector; +import org.elasticsearch.monitor.network.NetworkStats; +import org.elasticsearch.monitor.os.OsStats; +import org.elasticsearch.monitor.process.ProcessStats; +import org.elasticsearch.http.HttpStats; +import org.elasticsearch.transport.TransportStats; +import org.elasticsearch.threadpool.ThreadPoolStats; + +public class StatsdReporterNodeStats extends StatsdReporter { + + private final NodeStats nodeStats; + + public StatsdReporterNodeStats(NodeStats nodeStats) { + this.nodeStats = nodeStats; + } + + public void run() { + try { + this.sendNodeFsStats(this.nodeStats.getFs()); + this.sendNodeJvmStats(this.nodeStats.getJvm()); + this.sendNodeNetworkStats(this.nodeStats.getNetwork()); + this.sendNodeOsStats(this.nodeStats.getOs()); + this.sendNodeProcessStats(this.nodeStats.getProcess()); + this.sendNodeHttpStats(this.nodeStats.getHttp()); + this.sendNodeTransportStats(this.nodeStats.getTransport()); + this.sendNodeThreadPoolStats(this.nodeStats.getThreadPool()); + } catch (Exception e) { + this.logException(e); + } + } + + private void sendNodeThreadPoolStats(ThreadPoolStats threadPoolStats) + { + String type = this.buildMetricName("node.threadpool"); + Iterator statsIterator = threadPoolStats.iterator(); + while (statsIterator.hasNext()) { + ThreadPoolStats.Stats stats = statsIterator.next(); + String id = type + "." + stats.getName(); + + this.sendGauge(id, "threads", stats.getThreads()); + this.sendGauge(id, "queue", stats.getQueue()); + this.sendGauge(id, "active", stats.getActive()); + this.sendGauge(id, "rejected", stats.getRejected()); + this.sendGauge(id, "largest", stats.getLargest()); + this.sendGauge(id, "completed", stats.getCompleted()); + } + } + + private void sendNodeTransportStats(TransportStats transportStats) + { + String type = this.buildMetricName("node.transport"); + this.sendGauge(type, "serverOpen", transportStats.serverOpen()); + this.sendCount(type, "rxCount", transportStats.rxCount()); + this.sendCount(type, "rxSizeBytes", transportStats.rxSize().bytes()); + this.sendCount(type, "txCount", transportStats.txCount()); + this.sendCount(type, "txSizeBytes", transportStats.txSize().bytes()); + } + + private void sendNodeProcessStats(ProcessStats processStats) + { + String type = this.buildMetricName("node.process"); + + this.sendGauge(type, "openFileDescriptors", processStats.openFileDescriptors()); + if (processStats.cpu() != null) { + this.sendGauge(type + ".cpu", "percent", processStats.cpu().percent()); + this.sendGauge(type + ".cpu", "sysSeconds", processStats.cpu().sys().seconds()); + this.sendGauge(type + ".cpu", "totalSeconds", processStats.cpu().total().seconds()); + this.sendGauge(type + ".cpu", "userSeconds", processStats.cpu().user().seconds()); + } + + if (processStats.mem() != null) { + this.sendGauge(type + ".mem", "totalVirtual", processStats.mem().totalVirtual().bytes()); + this.sendGauge(type + ".mem", "resident", processStats.mem().resident().bytes()); + this.sendGauge(type + ".mem", "share", processStats.mem().share().bytes()); + } + } + + private void sendNodeOsStats(OsStats osStats) + { + String type = this.buildMetricName("node.os"); + + if (osStats.cpu() != null) { + this.sendGauge(type + ".cpu", "sys", osStats.cpu().sys()); + this.sendGauge(type + ".cpu", "idle", osStats.cpu().idle()); + this.sendGauge(type + ".cpu", "user", osStats.cpu().user()); + } + + if (osStats.mem() != null) { + this.sendGauge(type + ".mem", "freeBytes", osStats.mem().free().bytes()); + this.sendGauge(type + ".mem", "usedBytes", osStats.mem().used().bytes()); + this.sendGauge(type + ".mem", "freePercent", osStats.mem().freePercent()); + this.sendGauge(type + ".mem", "usedPercent", osStats.mem().usedPercent()); + this.sendGauge(type + ".mem", "actualFreeBytes", osStats.mem().actualFree().bytes()); + this.sendGauge(type + ".mem", "actualUsedBytes", osStats.mem().actualUsed().bytes()); + } + + if (osStats.swap() != null) { + this.sendGauge(type + ".swap", "freeBytes", osStats.swap().free().bytes()); + this.sendGauge(type + ".swap", "usedBytes", osStats.swap().used().bytes()); + } + } + + private void sendNodeNetworkStats(NetworkStats networkStats) + { + String type = this.buildMetricName("node.network.tcp"); + NetworkStats.Tcp tcp = networkStats.tcp(); + + // might be null, if sigar isnt loaded + if (tcp != null) { + this.sendGauge(type, "activeOpens", tcp.activeOpens()); + this.sendGauge(type, "passiveOpens", tcp.passiveOpens()); + this.sendGauge(type, "attemptFails", tcp.attemptFails()); + this.sendGauge(type, "estabResets", tcp.estabResets()); + this.sendGauge(type, "currEstab", tcp.currEstab()); + this.sendGauge(type, "inSegs", tcp.inSegs()); + this.sendGauge(type, "outSegs", tcp.outSegs()); + this.sendGauge(type, "retransSegs", tcp.retransSegs()); + this.sendGauge(type, "inErrs", tcp.inErrs()); + this.sendGauge(type, "outRsts", tcp.outRsts()); + } + } + + private void sendNodeJvmStats(JvmStats jvmStats) + { + String type = this.buildMetricName("node.jvm"); + this.sendGauge(type, "uptime", jvmStats.uptime().seconds()); + + // mem + this.sendGauge(type + ".mem", "heapCommitted", jvmStats.mem().heapCommitted().bytes()); + this.sendGauge(type + ".mem", "heapUsed", jvmStats.mem().heapUsed().bytes()); + this.sendGauge(type + ".mem", "nonHeapCommitted", jvmStats.mem().nonHeapCommitted().bytes()); + this.sendGauge(type + ".mem", "nonHeapUsed", jvmStats.mem().nonHeapUsed().bytes()); + + Iterator memoryPoolIterator = jvmStats.mem().iterator(); + while (memoryPoolIterator.hasNext()) { + JvmStats.MemoryPool memoryPool = memoryPoolIterator.next(); + String memoryPoolType = type + ".mem.pool." + memoryPool.name(); + + this.sendGauge(memoryPoolType, "max", memoryPool.max().bytes()); + this.sendGauge(memoryPoolType, "used", memoryPool.used().bytes()); + this.sendGauge(memoryPoolType, "peakUsed", memoryPool.peakUsed().bytes()); + this.sendGauge(memoryPoolType, "peakMax", memoryPool.peakMax().bytes()); + } + + // threads + this.sendGauge(type + ".threads", "count", jvmStats.threads().count()); + this.sendGauge(type + ".threads", "peakCount", jvmStats.threads().peakCount()); + + // garbage collectors + long gcCounter = 0; + long gcTimes = 0; + for (GarbageCollector gc : jvmStats.gc()) { + gcCounter += gc.collectionCount(); + gcTimes += gc.collectionTime().getSeconds(); + } + this.sendCount(type + ".gc", "collectionCount", gcCounter); + this.sendTime(type + ".gc", "collectionTimeSeconds", gcTimes); + for (JvmStats.GarbageCollector collector : jvmStats.gc().collectors()) { + String id = type + ".gc." + collector.name(); + this.sendCount(id, "collectionCount", collector.collectionCount()); + this.sendTime(id, "collectionTimeSeconds", collector.collectionTime().seconds()); + + JvmStats.GarbageCollector.LastGc lastGc = collector.lastGc(); + String lastGcType = type + ".lastGc"; + if (lastGc != null) { + this.sendTime(lastGcType, "time", lastGc.endTime() - lastGc.startTime()); + this.sendGauge(lastGcType, "max", lastGc.max().bytes()); + this.sendGauge(lastGcType, "beforeUsed", lastGc.beforeUsed().bytes()); + this.sendGauge(lastGcType, "afterUsed", lastGc.afterUsed().bytes()); + this.sendGauge(lastGcType, "durationSeconds", lastGc.duration().seconds()); + } + } + } + + private void sendNodeHttpStats(HttpStats httpStats) + { + String type = this.buildMetricName("node.http"); + this.sendGauge(type, "serverOpen", httpStats.getServerOpen()); + this.sendGauge(type, "totalOpen", httpStats.getTotalOpen()); + } + + private void sendNodeFsStats(FsStats fs) + { + Iterator infoIterator = fs.iterator(); + int i = 0; + while (infoIterator.hasNext()) { + String type = this.buildMetricName("node.fs") + i; + FsStats.Info info = infoIterator.next(); + this.sendGauge(type, "available", info.getAvailable().bytes()); + this.sendGauge(type, "total", info.getTotal().bytes()); + this.sendGauge(type, "free", info.getFree().bytes()); + this.sendCount(type, "diskReads", info.getDiskReads()); + this.sendCount(type, "diskReadsInBytes", info.getDiskReadSizeInBytes()); + this.sendCount(type, "diskWrites", info.getDiskWrites()); + this.sendCount(type, "diskWritesInBytes", info.getDiskWriteSizeInBytes()); + this.sendGauge(type, "diskQueue", (long) info.getDiskQueue()); + this.sendGauge(type, "diskService", (long) info.getDiskServiceTime()); + i++; + } + } +} diff --git a/src/main/java/org/elasticsearch/service/statsd/StatsdService.java b/src/main/java/org/elasticsearch/service/statsd/StatsdService.java index c01377a..bd97c7c 100644 --- a/src/main/java/org/elasticsearch/service/statsd/StatsdService.java +++ b/src/main/java/org/elasticsearch/service/statsd/StatsdService.java @@ -109,30 +109,42 @@ public void run() { if (node != null && isClusterStarted) { // Master Node sends cluster wide stats if (node.isMasterNode()) { - NodeIndicesStats nodeIndicesStats = StatsdService.this.indicesService.stats( - false // includePrevious + // Report node stats + StatsdReporter nodeStatsReporter = new StatsdReporterNodeStats( + StatsdService.this.nodeService.stats( + new CommonStatsFlags().clear(), // indices + true, // os + true, // process + true, // jvm + true, // threadPool + true, // network + true, // fs + true, // transport + true, // http + false // circuitBreaker + ) ); - NodeStats nodeStats = StatsdService.this.nodeService.stats( - new CommonStatsFlags().clear(), // indices - true, // os - true, // process - true, // jvm - true, // threadPool - true, // network - true, // fs - true, // transport - true, // http - false // circuitBreaker - ); - List indexShards = this.getIndexShards(StatsdService.this.indicesService); - - StatsdReporter statsdReporter = new StatsdReporter( - nodeIndicesStats, - indexShards, - nodeStats, - StatsdService.this.statsdClient - ); - statsdReporter.run(); + nodeStatsReporter + .setStatsDClient(StatsdService.this.statsdClient) + .run(); + + // Report node indice stats + StatsdReporter nodeIndicesStatsReporter = new StatsdReporterNodeIndicesStats( + StatsdService.this.indicesService.stats( + false // includePrevious + ) + ); + nodeIndicesStatsReporter + .setStatsDClient(StatsdService.this.statsdClient) + .run(); + + // Report indices stats + StatsdReporter indicesReporter = new StatsdReporterIndices( + this.getIndexShards(StatsdService.this.indicesService) + ); + indicesReporter + .setStatsDClient(StatsdService.this.statsdClient) + .run(); } else { StatsdService.this.logger.debug( "[{}]/[{}] is not master node, not triggering update", From 3e61e37e624b861e27fd09c3dc33c2d0a20beb67 Mon Sep 17 00:00:00 2001 From: Xiao Yu Date: Wed, 6 Aug 2014 11:51:59 -0400 Subject: [PATCH 08/48] Fix whitespace --- .../service/statsd/StatsdReporterIndices.java | 230 +++++------ .../StatsdReporterNodeIndicesStats.java | 214 +++++----- .../statsd/StatsdReporterNodeStats.java | 380 +++++++++--------- .../service/statsd/StatsdService.java | 66 +-- 4 files changed, 445 insertions(+), 445 deletions(-) diff --git a/src/main/java/org/elasticsearch/service/statsd/StatsdReporterIndices.java b/src/main/java/org/elasticsearch/service/statsd/StatsdReporterIndices.java index ebfc4ad..fa0264e 100644 --- a/src/main/java/org/elasticsearch/service/statsd/StatsdReporterIndices.java +++ b/src/main/java/org/elasticsearch/service/statsd/StatsdReporterIndices.java @@ -16,119 +16,119 @@ public class StatsdReporterIndices extends StatsdReporter { - private final List indexShards; - - public StatsdReporterIndices(List indexShards) { - this.indexShards = indexShards; - } - - public void run() - { - try { - for (IndexShard indexShard : this.indexShards) { - String type = this.buildMetricName("indexes.") + indexShard.shardId().index().name() + ".id." + indexShard.shardId().id(); - this.sendSearchStats(type + ".search", indexShard.searchStats()); - this.sendGetStats(type + ".get", indexShard.getStats()); - this.sendDocsStats(type + ".docs", indexShard.docStats()); - this.sendRefreshStats(type + ".refresh", indexShard.refreshStats()); - this.sendIndexingStats(type + ".indexing", indexShard.indexingStats("_all")); - this.sendMergeStats(type + ".merge", indexShard.mergeStats()); - this.sendWarmerStats(type + ".warmer", indexShard.warmerStats()); - this.sendStoreStats(type + ".store", indexShard.storeStats()); - } - } - catch (Exception e) { - this.logException(e); - } - } - - private void sendStoreStats(String type, StoreStats storeStats) - { - this.sendGauge(type, "sizeInBytes", storeStats.sizeInBytes()); - this.sendGauge(type, "throttleTimeInNanos", storeStats.throttleTime().getNanos()); - } - - private void sendWarmerStats(String type, WarmerStats warmerStats) - { - this.sendGauge(type, "current", warmerStats.current()); - this.sendGauge(type, "total", warmerStats.total()); - this.sendTime(type, "totalTimeInMillis", warmerStats.totalTimeInMillis()); - } - - private void sendMergeStats(String type, MergeStats mergeStats) - { - this.sendGauge(type, "total", mergeStats.getTotal()); - this.sendTime(type, "totalTimeInMillis", mergeStats.getTotalTimeInMillis()); - this.sendGauge(type, "totalNumDocs", mergeStats.getTotalNumDocs()); - this.sendGauge(type, "current", mergeStats.getCurrent()); - this.sendGauge(type, "currentNumDocs", mergeStats.getCurrentNumDocs()); - this.sendGauge(type, "currentSizeInBytes", mergeStats.getCurrentSizeInBytes()); - } - - private void sendSearchStats(String type, SearchStats searchStats) - { - SearchStats.Stats totalSearchStats = searchStats.getTotal(); - this.sendSearchStatsStats(type + "._all", totalSearchStats); - - if (searchStats.getGroupStats() != null) { - for (Map.Entry statsEntry : searchStats.getGroupStats().entrySet()) { - this.sendSearchStatsStats(type + "." + statsEntry.getKey(), statsEntry.getValue()); - } - } - } - - private void sendSearchStatsStats(String group, SearchStats.Stats searchStats) - { - String type = this.buildMetricName("search.stats.") + group; - this.sendCount(type, "queryCount", searchStats.getQueryCount()); - this.sendCount(type, "queryTimeInMillis", searchStats.getQueryTimeInMillis()); - this.sendGauge(type, "queryCurrent", searchStats.getQueryCurrent()); - this.sendCount(type, "fetchCount", searchStats.getFetchCount()); - this.sendCount(type, "fetchTimeInMillis", searchStats.getFetchTimeInMillis()); - this.sendGauge(type, "fetchCurrent", searchStats.getFetchCurrent()); - } - - private void sendRefreshStats(String type, RefreshStats refreshStats) - { - this.sendCount(type, "total", refreshStats.getTotal()); - this.sendCount(type, "totalTimeInMillis", refreshStats.getTotalTimeInMillis()); - } - - private void sendIndexingStats(String type, IndexingStats indexingStats) - { - IndexingStats.Stats totalStats = indexingStats.getTotal(); - this.sendStats(type + "._all", totalStats); - - Map typeStats = indexingStats.getTypeStats(); - if (typeStats != null) { - for (Map.Entry statsEntry : typeStats.entrySet()) { - this.sendStats(type + "." + statsEntry.getKey(), statsEntry.getValue()); - } - } - } - - private void sendStats(String type, IndexingStats.Stats stats) - { - this.sendCount(type, "indexCount", stats.getIndexCount()); - this.sendCount(type, "indexTimeInMillis", stats.getIndexTimeInMillis()); - this.sendGauge(type, "indexCurrent", stats.getIndexCount()); - this.sendCount(type, "deleteCount", stats.getDeleteCount()); - this.sendCount(type, "deleteTimeInMillis", stats.getDeleteTimeInMillis()); - this.sendGauge(type, "deleteCurrent", stats.getDeleteCurrent()); - } - - private void sendGetStats(String type, GetStats getStats) - { - this.sendCount(type, "existsCount", getStats.getExistsCount()); - this.sendCount(type, "existsTimeInMillis", getStats.getExistsTimeInMillis()); - this.sendCount(type, "missingCount", getStats.getMissingCount()); - this.sendCount(type, "missingTimeInMillis", getStats.getMissingTimeInMillis()); - this.sendGauge(type, "current", getStats.current()); - } - - private void sendDocsStats(String name, DocsStats docsStats) - { - this.sendCount(name, "count", docsStats.getCount()); - this.sendCount(name, "deleted", docsStats.getDeleted()); - } + private final List indexShards; + + public StatsdReporterIndices(List indexShards) { + this.indexShards = indexShards; + } + + public void run() + { + try { + for (IndexShard indexShard : this.indexShards) { + String type = this.buildMetricName("indexes.") + indexShard.shardId().index().name() + ".id." + indexShard.shardId().id(); + this.sendSearchStats(type + ".search", indexShard.searchStats()); + this.sendGetStats(type + ".get", indexShard.getStats()); + this.sendDocsStats(type + ".docs", indexShard.docStats()); + this.sendRefreshStats(type + ".refresh", indexShard.refreshStats()); + this.sendIndexingStats(type + ".indexing", indexShard.indexingStats("_all")); + this.sendMergeStats(type + ".merge", indexShard.mergeStats()); + this.sendWarmerStats(type + ".warmer", indexShard.warmerStats()); + this.sendStoreStats(type + ".store", indexShard.storeStats()); + } + } + catch (Exception e) { + this.logException(e); + } + } + + private void sendStoreStats(String type, StoreStats storeStats) + { + this.sendGauge(type, "sizeInBytes", storeStats.sizeInBytes()); + this.sendGauge(type, "throttleTimeInNanos", storeStats.throttleTime().getNanos()); + } + + private void sendWarmerStats(String type, WarmerStats warmerStats) + { + this.sendGauge(type, "current", warmerStats.current()); + this.sendGauge(type, "total", warmerStats.total()); + this.sendTime(type, "totalTimeInMillis", warmerStats.totalTimeInMillis()); + } + + private void sendMergeStats(String type, MergeStats mergeStats) + { + this.sendGauge(type, "total", mergeStats.getTotal()); + this.sendTime(type, "totalTimeInMillis", mergeStats.getTotalTimeInMillis()); + this.sendGauge(type, "totalNumDocs", mergeStats.getTotalNumDocs()); + this.sendGauge(type, "current", mergeStats.getCurrent()); + this.sendGauge(type, "currentNumDocs", mergeStats.getCurrentNumDocs()); + this.sendGauge(type, "currentSizeInBytes", mergeStats.getCurrentSizeInBytes()); + } + + private void sendSearchStats(String type, SearchStats searchStats) + { + SearchStats.Stats totalSearchStats = searchStats.getTotal(); + this.sendSearchStatsStats(type + "._all", totalSearchStats); + + if (searchStats.getGroupStats() != null) { + for (Map.Entry statsEntry : searchStats.getGroupStats().entrySet()) { + this.sendSearchStatsStats(type + "." + statsEntry.getKey(), statsEntry.getValue()); + } + } + } + + private void sendSearchStatsStats(String group, SearchStats.Stats searchStats) + { + String type = this.buildMetricName("search.stats.") + group; + this.sendCount(type, "queryCount", searchStats.getQueryCount()); + this.sendCount(type, "queryTimeInMillis", searchStats.getQueryTimeInMillis()); + this.sendGauge(type, "queryCurrent", searchStats.getQueryCurrent()); + this.sendCount(type, "fetchCount", searchStats.getFetchCount()); + this.sendCount(type, "fetchTimeInMillis", searchStats.getFetchTimeInMillis()); + this.sendGauge(type, "fetchCurrent", searchStats.getFetchCurrent()); + } + + private void sendRefreshStats(String type, RefreshStats refreshStats) + { + this.sendCount(type, "total", refreshStats.getTotal()); + this.sendCount(type, "totalTimeInMillis", refreshStats.getTotalTimeInMillis()); + } + + private void sendIndexingStats(String type, IndexingStats indexingStats) + { + IndexingStats.Stats totalStats = indexingStats.getTotal(); + this.sendStats(type + "._all", totalStats); + + Map typeStats = indexingStats.getTypeStats(); + if (typeStats != null) { + for (Map.Entry statsEntry : typeStats.entrySet()) { + this.sendStats(type + "." + statsEntry.getKey(), statsEntry.getValue()); + } + } + } + + private void sendStats(String type, IndexingStats.Stats stats) + { + this.sendCount(type, "indexCount", stats.getIndexCount()); + this.sendCount(type, "indexTimeInMillis", stats.getIndexTimeInMillis()); + this.sendGauge(type, "indexCurrent", stats.getIndexCount()); + this.sendCount(type, "deleteCount", stats.getDeleteCount()); + this.sendCount(type, "deleteTimeInMillis", stats.getDeleteTimeInMillis()); + this.sendGauge(type, "deleteCurrent", stats.getDeleteCurrent()); + } + + private void sendGetStats(String type, GetStats getStats) + { + this.sendCount(type, "existsCount", getStats.getExistsCount()); + this.sendCount(type, "existsTimeInMillis", getStats.getExistsTimeInMillis()); + this.sendCount(type, "missingCount", getStats.getMissingCount()); + this.sendCount(type, "missingTimeInMillis", getStats.getMissingTimeInMillis()); + this.sendGauge(type, "current", getStats.current()); + } + + private void sendDocsStats(String name, DocsStats docsStats) + { + this.sendCount(name, "count", docsStats.getCount()); + this.sendCount(name, "deleted", docsStats.getDeleted()); + } } diff --git a/src/main/java/org/elasticsearch/service/statsd/StatsdReporterNodeIndicesStats.java b/src/main/java/org/elasticsearch/service/statsd/StatsdReporterNodeIndicesStats.java index b4e4ac9..c3cbe44 100644 --- a/src/main/java/org/elasticsearch/service/statsd/StatsdReporterNodeIndicesStats.java +++ b/src/main/java/org/elasticsearch/service/statsd/StatsdReporterNodeIndicesStats.java @@ -16,111 +16,111 @@ public class StatsdReporterNodeIndicesStats extends StatsdReporter { - private final NodeIndicesStats nodeIndicesStats; - - public StatsdReporterNodeIndicesStats(NodeIndicesStats nodeIndicesStats) { - this.nodeIndicesStats = nodeIndicesStats; - } - - public void run() - { - try { - String type = this.buildMetricName("node"); - this.sendFilterCacheStats(type + ".filtercache", this.nodeIndicesStats.getFilterCache()); - this.sendIdCacheStats(type + ".idcache", this.nodeIndicesStats.getIdCache()); - this.sendDocsStats(type + ".docs", this.nodeIndicesStats.getDocs()); - this.sendFlushStats(type + ".flush", this.nodeIndicesStats.getFlush()); - this.sendGetStats(type + ".get", this.nodeIndicesStats.getGet()); - this.sendIndexingStats(type + ".indexing", this.nodeIndicesStats.getIndexing()); - this.sendRefreshStats(type + ".refresh", this.nodeIndicesStats.getRefresh()); - this.sendSearchStats(type + ".search", this.nodeIndicesStats.getSearch()); - } - catch (Exception e) { - this.logException(e); - } - } - - private void sendSearchStats(String type, SearchStats searchStats) - { - SearchStats.Stats totalSearchStats = searchStats.getTotal(); - this.sendSearchStatsStats(type + "._all", totalSearchStats); - - if (searchStats.getGroupStats() != null) { - for (Map.Entry statsEntry : searchStats.getGroupStats().entrySet()) { - this.sendSearchStatsStats(type + "." + statsEntry.getKey(), statsEntry.getValue()); - } - } - } - - private void sendSearchStatsStats(String group, SearchStats.Stats searchStats) - { - String type = this.buildMetricName("search.stats.") + group; - this.sendCount(type, "queryCount", searchStats.getQueryCount()); - this.sendCount(type, "queryTimeInMillis", searchStats.getQueryTimeInMillis()); - this.sendGauge(type, "queryCurrent", searchStats.getQueryCurrent()); - this.sendCount(type, "fetchCount", searchStats.getFetchCount()); - this.sendCount(type, "fetchTimeInMillis", searchStats.getFetchTimeInMillis()); - this.sendGauge(type, "fetchCurrent", searchStats.getFetchCurrent()); - } - - private void sendRefreshStats(String type, RefreshStats refreshStats) - { - this.sendCount(type, "total", refreshStats.getTotal()); - this.sendCount(type, "totalTimeInMillis", refreshStats.getTotalTimeInMillis()); - } - - private void sendIndexingStats(String type, IndexingStats indexingStats) - { - IndexingStats.Stats totalStats = indexingStats.getTotal(); - this.sendStats(type + "._all", totalStats); - - Map typeStats = indexingStats.getTypeStats(); - if (typeStats != null) { - for (Map.Entry statsEntry : typeStats.entrySet()) { - this.sendStats(type + "." + statsEntry.getKey(), statsEntry.getValue()); - } - } - } - - private void sendStats(String type, IndexingStats.Stats stats) - { - this.sendCount(type, "indexCount", stats.getIndexCount()); - this.sendCount(type, "indexTimeInMillis", stats.getIndexTimeInMillis()); - this.sendGauge(type, "indexCurrent", stats.getIndexCount()); - this.sendCount(type, "deleteCount", stats.getDeleteCount()); - this.sendCount(type, "deleteTimeInMillis", stats.getDeleteTimeInMillis()); - this.sendGauge(type, "deleteCurrent", stats.getDeleteCurrent()); - } - - private void sendGetStats(String type, GetStats getStats) - { - this.sendCount(type, "existsCount", getStats.getExistsCount()); - this.sendCount(type, "existsTimeInMillis", getStats.getExistsTimeInMillis()); - this.sendCount(type, "missingCount", getStats.getMissingCount()); - this.sendCount(type, "missingTimeInMillis", getStats.getMissingTimeInMillis()); - this.sendGauge(type, "current", getStats.current()); - } - - private void sendFlushStats(String type, FlushStats flush) - { - this.sendCount(type, "total", flush.getTotal()); - this.sendCount(type, "totalTimeInMillis", flush.getTotalTimeInMillis()); - } - - private void sendDocsStats(String name, DocsStats docsStats) - { - this.sendCount(name, "count", docsStats.getCount()); - this.sendCount(name, "deleted", docsStats.getDeleted()); - } - - private void sendIdCacheStats(String name, IdCacheStats idCache) - { - this.sendGauge(name, "memorySizeInBytes", idCache.getMemorySizeInBytes()); - } - - private void sendFilterCacheStats(String name, FilterCacheStats filterCache) - { - this.sendGauge(name, "memorySizeInBytes", filterCache.getMemorySizeInBytes()); - this.sendGauge(name, "evictions", filterCache.getEvictions()); - } + private final NodeIndicesStats nodeIndicesStats; + + public StatsdReporterNodeIndicesStats(NodeIndicesStats nodeIndicesStats) { + this.nodeIndicesStats = nodeIndicesStats; + } + + public void run() + { + try { + String type = this.buildMetricName("node"); + this.sendFilterCacheStats(type + ".filtercache", this.nodeIndicesStats.getFilterCache()); + this.sendIdCacheStats(type + ".idcache", this.nodeIndicesStats.getIdCache()); + this.sendDocsStats(type + ".docs", this.nodeIndicesStats.getDocs()); + this.sendFlushStats(type + ".flush", this.nodeIndicesStats.getFlush()); + this.sendGetStats(type + ".get", this.nodeIndicesStats.getGet()); + this.sendIndexingStats(type + ".indexing", this.nodeIndicesStats.getIndexing()); + this.sendRefreshStats(type + ".refresh", this.nodeIndicesStats.getRefresh()); + this.sendSearchStats(type + ".search", this.nodeIndicesStats.getSearch()); + } + catch (Exception e) { + this.logException(e); + } + } + + private void sendSearchStats(String type, SearchStats searchStats) + { + SearchStats.Stats totalSearchStats = searchStats.getTotal(); + this.sendSearchStatsStats(type + "._all", totalSearchStats); + + if (searchStats.getGroupStats() != null) { + for (Map.Entry statsEntry : searchStats.getGroupStats().entrySet()) { + this.sendSearchStatsStats(type + "." + statsEntry.getKey(), statsEntry.getValue()); + } + } + } + + private void sendSearchStatsStats(String group, SearchStats.Stats searchStats) + { + String type = this.buildMetricName("search.stats.") + group; + this.sendCount(type, "queryCount", searchStats.getQueryCount()); + this.sendCount(type, "queryTimeInMillis", searchStats.getQueryTimeInMillis()); + this.sendGauge(type, "queryCurrent", searchStats.getQueryCurrent()); + this.sendCount(type, "fetchCount", searchStats.getFetchCount()); + this.sendCount(type, "fetchTimeInMillis", searchStats.getFetchTimeInMillis()); + this.sendGauge(type, "fetchCurrent", searchStats.getFetchCurrent()); + } + + private void sendRefreshStats(String type, RefreshStats refreshStats) + { + this.sendCount(type, "total", refreshStats.getTotal()); + this.sendCount(type, "totalTimeInMillis", refreshStats.getTotalTimeInMillis()); + } + + private void sendIndexingStats(String type, IndexingStats indexingStats) + { + IndexingStats.Stats totalStats = indexingStats.getTotal(); + this.sendStats(type + "._all", totalStats); + + Map typeStats = indexingStats.getTypeStats(); + if (typeStats != null) { + for (Map.Entry statsEntry : typeStats.entrySet()) { + this.sendStats(type + "." + statsEntry.getKey(), statsEntry.getValue()); + } + } + } + + private void sendStats(String type, IndexingStats.Stats stats) + { + this.sendCount(type, "indexCount", stats.getIndexCount()); + this.sendCount(type, "indexTimeInMillis", stats.getIndexTimeInMillis()); + this.sendGauge(type, "indexCurrent", stats.getIndexCount()); + this.sendCount(type, "deleteCount", stats.getDeleteCount()); + this.sendCount(type, "deleteTimeInMillis", stats.getDeleteTimeInMillis()); + this.sendGauge(type, "deleteCurrent", stats.getDeleteCurrent()); + } + + private void sendGetStats(String type, GetStats getStats) + { + this.sendCount(type, "existsCount", getStats.getExistsCount()); + this.sendCount(type, "existsTimeInMillis", getStats.getExistsTimeInMillis()); + this.sendCount(type, "missingCount", getStats.getMissingCount()); + this.sendCount(type, "missingTimeInMillis", getStats.getMissingTimeInMillis()); + this.sendGauge(type, "current", getStats.current()); + } + + private void sendFlushStats(String type, FlushStats flush) + { + this.sendCount(type, "total", flush.getTotal()); + this.sendCount(type, "totalTimeInMillis", flush.getTotalTimeInMillis()); + } + + private void sendDocsStats(String name, DocsStats docsStats) + { + this.sendCount(name, "count", docsStats.getCount()); + this.sendCount(name, "deleted", docsStats.getDeleted()); + } + + private void sendIdCacheStats(String name, IdCacheStats idCache) + { + this.sendGauge(name, "memorySizeInBytes", idCache.getMemorySizeInBytes()); + } + + private void sendFilterCacheStats(String name, FilterCacheStats filterCache) + { + this.sendGauge(name, "memorySizeInBytes", filterCache.getMemorySizeInBytes()); + this.sendGauge(name, "evictions", filterCache.getEvictions()); + } } diff --git a/src/main/java/org/elasticsearch/service/statsd/StatsdReporterNodeStats.java b/src/main/java/org/elasticsearch/service/statsd/StatsdReporterNodeStats.java index b292333..c5e7ea9 100644 --- a/src/main/java/org/elasticsearch/service/statsd/StatsdReporterNodeStats.java +++ b/src/main/java/org/elasticsearch/service/statsd/StatsdReporterNodeStats.java @@ -17,194 +17,194 @@ public class StatsdReporterNodeStats extends StatsdReporter { - private final NodeStats nodeStats; - - public StatsdReporterNodeStats(NodeStats nodeStats) { - this.nodeStats = nodeStats; - } - - public void run() { - try { - this.sendNodeFsStats(this.nodeStats.getFs()); - this.sendNodeJvmStats(this.nodeStats.getJvm()); - this.sendNodeNetworkStats(this.nodeStats.getNetwork()); - this.sendNodeOsStats(this.nodeStats.getOs()); - this.sendNodeProcessStats(this.nodeStats.getProcess()); - this.sendNodeHttpStats(this.nodeStats.getHttp()); - this.sendNodeTransportStats(this.nodeStats.getTransport()); - this.sendNodeThreadPoolStats(this.nodeStats.getThreadPool()); - } catch (Exception e) { - this.logException(e); - } - } - - private void sendNodeThreadPoolStats(ThreadPoolStats threadPoolStats) - { - String type = this.buildMetricName("node.threadpool"); - Iterator statsIterator = threadPoolStats.iterator(); - while (statsIterator.hasNext()) { - ThreadPoolStats.Stats stats = statsIterator.next(); - String id = type + "." + stats.getName(); - - this.sendGauge(id, "threads", stats.getThreads()); - this.sendGauge(id, "queue", stats.getQueue()); - this.sendGauge(id, "active", stats.getActive()); - this.sendGauge(id, "rejected", stats.getRejected()); - this.sendGauge(id, "largest", stats.getLargest()); - this.sendGauge(id, "completed", stats.getCompleted()); - } - } - - private void sendNodeTransportStats(TransportStats transportStats) - { - String type = this.buildMetricName("node.transport"); - this.sendGauge(type, "serverOpen", transportStats.serverOpen()); - this.sendCount(type, "rxCount", transportStats.rxCount()); - this.sendCount(type, "rxSizeBytes", transportStats.rxSize().bytes()); - this.sendCount(type, "txCount", transportStats.txCount()); - this.sendCount(type, "txSizeBytes", transportStats.txSize().bytes()); - } - - private void sendNodeProcessStats(ProcessStats processStats) - { - String type = this.buildMetricName("node.process"); - - this.sendGauge(type, "openFileDescriptors", processStats.openFileDescriptors()); - if (processStats.cpu() != null) { - this.sendGauge(type + ".cpu", "percent", processStats.cpu().percent()); - this.sendGauge(type + ".cpu", "sysSeconds", processStats.cpu().sys().seconds()); - this.sendGauge(type + ".cpu", "totalSeconds", processStats.cpu().total().seconds()); - this.sendGauge(type + ".cpu", "userSeconds", processStats.cpu().user().seconds()); - } - - if (processStats.mem() != null) { - this.sendGauge(type + ".mem", "totalVirtual", processStats.mem().totalVirtual().bytes()); - this.sendGauge(type + ".mem", "resident", processStats.mem().resident().bytes()); - this.sendGauge(type + ".mem", "share", processStats.mem().share().bytes()); - } - } - - private void sendNodeOsStats(OsStats osStats) - { - String type = this.buildMetricName("node.os"); - - if (osStats.cpu() != null) { - this.sendGauge(type + ".cpu", "sys", osStats.cpu().sys()); - this.sendGauge(type + ".cpu", "idle", osStats.cpu().idle()); - this.sendGauge(type + ".cpu", "user", osStats.cpu().user()); - } - - if (osStats.mem() != null) { - this.sendGauge(type + ".mem", "freeBytes", osStats.mem().free().bytes()); - this.sendGauge(type + ".mem", "usedBytes", osStats.mem().used().bytes()); - this.sendGauge(type + ".mem", "freePercent", osStats.mem().freePercent()); - this.sendGauge(type + ".mem", "usedPercent", osStats.mem().usedPercent()); - this.sendGauge(type + ".mem", "actualFreeBytes", osStats.mem().actualFree().bytes()); - this.sendGauge(type + ".mem", "actualUsedBytes", osStats.mem().actualUsed().bytes()); - } - - if (osStats.swap() != null) { - this.sendGauge(type + ".swap", "freeBytes", osStats.swap().free().bytes()); - this.sendGauge(type + ".swap", "usedBytes", osStats.swap().used().bytes()); - } - } - - private void sendNodeNetworkStats(NetworkStats networkStats) - { - String type = this.buildMetricName("node.network.tcp"); - NetworkStats.Tcp tcp = networkStats.tcp(); - - // might be null, if sigar isnt loaded - if (tcp != null) { - this.sendGauge(type, "activeOpens", tcp.activeOpens()); - this.sendGauge(type, "passiveOpens", tcp.passiveOpens()); - this.sendGauge(type, "attemptFails", tcp.attemptFails()); - this.sendGauge(type, "estabResets", tcp.estabResets()); - this.sendGauge(type, "currEstab", tcp.currEstab()); - this.sendGauge(type, "inSegs", tcp.inSegs()); - this.sendGauge(type, "outSegs", tcp.outSegs()); - this.sendGauge(type, "retransSegs", tcp.retransSegs()); - this.sendGauge(type, "inErrs", tcp.inErrs()); - this.sendGauge(type, "outRsts", tcp.outRsts()); - } - } - - private void sendNodeJvmStats(JvmStats jvmStats) - { - String type = this.buildMetricName("node.jvm"); - this.sendGauge(type, "uptime", jvmStats.uptime().seconds()); - - // mem - this.sendGauge(type + ".mem", "heapCommitted", jvmStats.mem().heapCommitted().bytes()); - this.sendGauge(type + ".mem", "heapUsed", jvmStats.mem().heapUsed().bytes()); - this.sendGauge(type + ".mem", "nonHeapCommitted", jvmStats.mem().nonHeapCommitted().bytes()); - this.sendGauge(type + ".mem", "nonHeapUsed", jvmStats.mem().nonHeapUsed().bytes()); - - Iterator memoryPoolIterator = jvmStats.mem().iterator(); - while (memoryPoolIterator.hasNext()) { - JvmStats.MemoryPool memoryPool = memoryPoolIterator.next(); - String memoryPoolType = type + ".mem.pool." + memoryPool.name(); - - this.sendGauge(memoryPoolType, "max", memoryPool.max().bytes()); - this.sendGauge(memoryPoolType, "used", memoryPool.used().bytes()); - this.sendGauge(memoryPoolType, "peakUsed", memoryPool.peakUsed().bytes()); - this.sendGauge(memoryPoolType, "peakMax", memoryPool.peakMax().bytes()); - } - - // threads - this.sendGauge(type + ".threads", "count", jvmStats.threads().count()); - this.sendGauge(type + ".threads", "peakCount", jvmStats.threads().peakCount()); - - // garbage collectors - long gcCounter = 0; - long gcTimes = 0; - for (GarbageCollector gc : jvmStats.gc()) { - gcCounter += gc.collectionCount(); - gcTimes += gc.collectionTime().getSeconds(); - } - this.sendCount(type + ".gc", "collectionCount", gcCounter); - this.sendTime(type + ".gc", "collectionTimeSeconds", gcTimes); - for (JvmStats.GarbageCollector collector : jvmStats.gc().collectors()) { - String id = type + ".gc." + collector.name(); - this.sendCount(id, "collectionCount", collector.collectionCount()); - this.sendTime(id, "collectionTimeSeconds", collector.collectionTime().seconds()); - - JvmStats.GarbageCollector.LastGc lastGc = collector.lastGc(); - String lastGcType = type + ".lastGc"; - if (lastGc != null) { - this.sendTime(lastGcType, "time", lastGc.endTime() - lastGc.startTime()); - this.sendGauge(lastGcType, "max", lastGc.max().bytes()); - this.sendGauge(lastGcType, "beforeUsed", lastGc.beforeUsed().bytes()); - this.sendGauge(lastGcType, "afterUsed", lastGc.afterUsed().bytes()); - this.sendGauge(lastGcType, "durationSeconds", lastGc.duration().seconds()); - } - } - } - - private void sendNodeHttpStats(HttpStats httpStats) - { - String type = this.buildMetricName("node.http"); - this.sendGauge(type, "serverOpen", httpStats.getServerOpen()); - this.sendGauge(type, "totalOpen", httpStats.getTotalOpen()); - } - - private void sendNodeFsStats(FsStats fs) - { - Iterator infoIterator = fs.iterator(); - int i = 0; - while (infoIterator.hasNext()) { - String type = this.buildMetricName("node.fs") + i; - FsStats.Info info = infoIterator.next(); - this.sendGauge(type, "available", info.getAvailable().bytes()); - this.sendGauge(type, "total", info.getTotal().bytes()); - this.sendGauge(type, "free", info.getFree().bytes()); - this.sendCount(type, "diskReads", info.getDiskReads()); - this.sendCount(type, "diskReadsInBytes", info.getDiskReadSizeInBytes()); - this.sendCount(type, "diskWrites", info.getDiskWrites()); - this.sendCount(type, "diskWritesInBytes", info.getDiskWriteSizeInBytes()); - this.sendGauge(type, "diskQueue", (long) info.getDiskQueue()); - this.sendGauge(type, "diskService", (long) info.getDiskServiceTime()); - i++; - } - } + private final NodeStats nodeStats; + + public StatsdReporterNodeStats(NodeStats nodeStats) { + this.nodeStats = nodeStats; + } + + public void run() { + try { + this.sendNodeFsStats(this.nodeStats.getFs()); + this.sendNodeJvmStats(this.nodeStats.getJvm()); + this.sendNodeNetworkStats(this.nodeStats.getNetwork()); + this.sendNodeOsStats(this.nodeStats.getOs()); + this.sendNodeProcessStats(this.nodeStats.getProcess()); + this.sendNodeHttpStats(this.nodeStats.getHttp()); + this.sendNodeTransportStats(this.nodeStats.getTransport()); + this.sendNodeThreadPoolStats(this.nodeStats.getThreadPool()); + } catch (Exception e) { + this.logException(e); + } + } + + private void sendNodeThreadPoolStats(ThreadPoolStats threadPoolStats) + { + String type = this.buildMetricName("node.threadpool"); + Iterator statsIterator = threadPoolStats.iterator(); + while (statsIterator.hasNext()) { + ThreadPoolStats.Stats stats = statsIterator.next(); + String id = type + "." + stats.getName(); + + this.sendGauge(id, "threads", stats.getThreads()); + this.sendGauge(id, "queue", stats.getQueue()); + this.sendGauge(id, "active", stats.getActive()); + this.sendGauge(id, "rejected", stats.getRejected()); + this.sendGauge(id, "largest", stats.getLargest()); + this.sendGauge(id, "completed", stats.getCompleted()); + } + } + + private void sendNodeTransportStats(TransportStats transportStats) + { + String type = this.buildMetricName("node.transport"); + this.sendGauge(type, "serverOpen", transportStats.serverOpen()); + this.sendCount(type, "rxCount", transportStats.rxCount()); + this.sendCount(type, "rxSizeBytes", transportStats.rxSize().bytes()); + this.sendCount(type, "txCount", transportStats.txCount()); + this.sendCount(type, "txSizeBytes", transportStats.txSize().bytes()); + } + + private void sendNodeProcessStats(ProcessStats processStats) + { + String type = this.buildMetricName("node.process"); + + this.sendGauge(type, "openFileDescriptors", processStats.openFileDescriptors()); + if (processStats.cpu() != null) { + this.sendGauge(type + ".cpu", "percent", processStats.cpu().percent()); + this.sendGauge(type + ".cpu", "sysSeconds", processStats.cpu().sys().seconds()); + this.sendGauge(type + ".cpu", "totalSeconds", processStats.cpu().total().seconds()); + this.sendGauge(type + ".cpu", "userSeconds", processStats.cpu().user().seconds()); + } + + if (processStats.mem() != null) { + this.sendGauge(type + ".mem", "totalVirtual", processStats.mem().totalVirtual().bytes()); + this.sendGauge(type + ".mem", "resident", processStats.mem().resident().bytes()); + this.sendGauge(type + ".mem", "share", processStats.mem().share().bytes()); + } + } + + private void sendNodeOsStats(OsStats osStats) + { + String type = this.buildMetricName("node.os"); + + if (osStats.cpu() != null) { + this.sendGauge(type + ".cpu", "sys", osStats.cpu().sys()); + this.sendGauge(type + ".cpu", "idle", osStats.cpu().idle()); + this.sendGauge(type + ".cpu", "user", osStats.cpu().user()); + } + + if (osStats.mem() != null) { + this.sendGauge(type + ".mem", "freeBytes", osStats.mem().free().bytes()); + this.sendGauge(type + ".mem", "usedBytes", osStats.mem().used().bytes()); + this.sendGauge(type + ".mem", "freePercent", osStats.mem().freePercent()); + this.sendGauge(type + ".mem", "usedPercent", osStats.mem().usedPercent()); + this.sendGauge(type + ".mem", "actualFreeBytes", osStats.mem().actualFree().bytes()); + this.sendGauge(type + ".mem", "actualUsedBytes", osStats.mem().actualUsed().bytes()); + } + + if (osStats.swap() != null) { + this.sendGauge(type + ".swap", "freeBytes", osStats.swap().free().bytes()); + this.sendGauge(type + ".swap", "usedBytes", osStats.swap().used().bytes()); + } + } + + private void sendNodeNetworkStats(NetworkStats networkStats) + { + String type = this.buildMetricName("node.network.tcp"); + NetworkStats.Tcp tcp = networkStats.tcp(); + + // might be null, if sigar isnt loaded + if (tcp != null) { + this.sendGauge(type, "activeOpens", tcp.activeOpens()); + this.sendGauge(type, "passiveOpens", tcp.passiveOpens()); + this.sendGauge(type, "attemptFails", tcp.attemptFails()); + this.sendGauge(type, "estabResets", tcp.estabResets()); + this.sendGauge(type, "currEstab", tcp.currEstab()); + this.sendGauge(type, "inSegs", tcp.inSegs()); + this.sendGauge(type, "outSegs", tcp.outSegs()); + this.sendGauge(type, "retransSegs", tcp.retransSegs()); + this.sendGauge(type, "inErrs", tcp.inErrs()); + this.sendGauge(type, "outRsts", tcp.outRsts()); + } + } + + private void sendNodeJvmStats(JvmStats jvmStats) + { + String type = this.buildMetricName("node.jvm"); + this.sendGauge(type, "uptime", jvmStats.uptime().seconds()); + + // mem + this.sendGauge(type + ".mem", "heapCommitted", jvmStats.mem().heapCommitted().bytes()); + this.sendGauge(type + ".mem", "heapUsed", jvmStats.mem().heapUsed().bytes()); + this.sendGauge(type + ".mem", "nonHeapCommitted", jvmStats.mem().nonHeapCommitted().bytes()); + this.sendGauge(type + ".mem", "nonHeapUsed", jvmStats.mem().nonHeapUsed().bytes()); + + Iterator memoryPoolIterator = jvmStats.mem().iterator(); + while (memoryPoolIterator.hasNext()) { + JvmStats.MemoryPool memoryPool = memoryPoolIterator.next(); + String memoryPoolType = type + ".mem.pool." + memoryPool.name(); + + this.sendGauge(memoryPoolType, "max", memoryPool.max().bytes()); + this.sendGauge(memoryPoolType, "used", memoryPool.used().bytes()); + this.sendGauge(memoryPoolType, "peakUsed", memoryPool.peakUsed().bytes()); + this.sendGauge(memoryPoolType, "peakMax", memoryPool.peakMax().bytes()); + } + + // threads + this.sendGauge(type + ".threads", "count", jvmStats.threads().count()); + this.sendGauge(type + ".threads", "peakCount", jvmStats.threads().peakCount()); + + // garbage collectors + long gcCounter = 0; + long gcTimes = 0; + for (GarbageCollector gc : jvmStats.gc()) { + gcCounter += gc.collectionCount(); + gcTimes += gc.collectionTime().getSeconds(); + } + this.sendCount(type + ".gc", "collectionCount", gcCounter); + this.sendTime(type + ".gc", "collectionTimeSeconds", gcTimes); + for (JvmStats.GarbageCollector collector : jvmStats.gc().collectors()) { + String id = type + ".gc." + collector.name(); + this.sendCount(id, "collectionCount", collector.collectionCount()); + this.sendTime(id, "collectionTimeSeconds", collector.collectionTime().seconds()); + + JvmStats.GarbageCollector.LastGc lastGc = collector.lastGc(); + String lastGcType = type + ".lastGc"; + if (lastGc != null) { + this.sendTime(lastGcType, "time", lastGc.endTime() - lastGc.startTime()); + this.sendGauge(lastGcType, "max", lastGc.max().bytes()); + this.sendGauge(lastGcType, "beforeUsed", lastGc.beforeUsed().bytes()); + this.sendGauge(lastGcType, "afterUsed", lastGc.afterUsed().bytes()); + this.sendGauge(lastGcType, "durationSeconds", lastGc.duration().seconds()); + } + } + } + + private void sendNodeHttpStats(HttpStats httpStats) + { + String type = this.buildMetricName("node.http"); + this.sendGauge(type, "serverOpen", httpStats.getServerOpen()); + this.sendGauge(type, "totalOpen", httpStats.getTotalOpen()); + } + + private void sendNodeFsStats(FsStats fs) + { + Iterator infoIterator = fs.iterator(); + int i = 0; + while (infoIterator.hasNext()) { + String type = this.buildMetricName("node.fs") + i; + FsStats.Info info = infoIterator.next(); + this.sendGauge(type, "available", info.getAvailable().bytes()); + this.sendGauge(type, "total", info.getTotal().bytes()); + this.sendGauge(type, "free", info.getFree().bytes()); + this.sendCount(type, "diskReads", info.getDiskReads()); + this.sendCount(type, "diskReadsInBytes", info.getDiskReadSizeInBytes()); + this.sendCount(type, "diskWrites", info.getDiskWrites()); + this.sendCount(type, "diskWritesInBytes", info.getDiskWriteSizeInBytes()); + this.sendGauge(type, "diskQueue", (long) info.getDiskQueue()); + this.sendGauge(type, "diskService", (long) info.getDiskServiceTime()); + i++; + } + } } diff --git a/src/main/java/org/elasticsearch/service/statsd/StatsdService.java b/src/main/java/org/elasticsearch/service/statsd/StatsdService.java index bd97c7c..f2490ba 100644 --- a/src/main/java/org/elasticsearch/service/statsd/StatsdService.java +++ b/src/main/java/org/elasticsearch/service/statsd/StatsdService.java @@ -109,42 +109,42 @@ public void run() { if (node != null && isClusterStarted) { // Master Node sends cluster wide stats if (node.isMasterNode()) { - // Report node stats + // Report node stats StatsdReporter nodeStatsReporter = new StatsdReporterNodeStats( - StatsdService.this.nodeService.stats( - new CommonStatsFlags().clear(), // indices - true, // os - true, // process - true, // jvm - true, // threadPool - true, // network - true, // fs - true, // transport - true, // http - false // circuitBreaker - ) + StatsdService.this.nodeService.stats( + new CommonStatsFlags().clear(), // indices + true, // os + true, // process + true, // jvm + true, // threadPool + true, // network + true, // fs + true, // transport + true, // http + false // circuitBreaker + ) ); nodeStatsReporter - .setStatsDClient(StatsdService.this.statsdClient) - .run(); - - // Report node indice stats - StatsdReporter nodeIndicesStatsReporter = new StatsdReporterNodeIndicesStats( - StatsdService.this.indicesService.stats( - false // includePrevious - ) - ); - nodeIndicesStatsReporter - .setStatsDClient(StatsdService.this.statsdClient) - .run(); - - // Report indices stats - StatsdReporter indicesReporter = new StatsdReporterIndices( - this.getIndexShards(StatsdService.this.indicesService) - ); - indicesReporter - .setStatsDClient(StatsdService.this.statsdClient) - .run(); + .setStatsDClient(StatsdService.this.statsdClient) + .run(); + + // Report node indice stats + StatsdReporter nodeIndicesStatsReporter = new StatsdReporterNodeIndicesStats( + StatsdService.this.indicesService.stats( + false // includePrevious + ) + ); + nodeIndicesStatsReporter + .setStatsDClient(StatsdService.this.statsdClient) + .run(); + + // Report indices stats + StatsdReporter indicesReporter = new StatsdReporterIndices( + this.getIndexShards(StatsdService.this.indicesService) + ); + indicesReporter + .setStatsDClient(StatsdService.this.statsdClient) + .run(); } else { StatsdService.this.logger.debug( "[{}]/[{}] is not master node, not triggering update", From 10a749ba2b5b073dd2f071aa3e2f447b95aac430 Mon Sep 17 00:00:00 2001 From: Xiao Yu Date: Wed, 6 Aug 2014 15:57:53 -0400 Subject: [PATCH 09/48] Cleanup Node reporting and standardize naming to match those used by ES --- .../service/statsd/StatsdReporter.java | 10 +- .../statsd/StatsdReporterNodeStats.java | 225 ++++++++++-------- .../service/statsd/StatsdService.java | 11 +- .../test/StatsdPluginIntegrationTest.java | 2 +- 4 files changed, 138 insertions(+), 110 deletions(-) diff --git a/src/main/java/org/elasticsearch/service/statsd/StatsdReporter.java b/src/main/java/org/elasticsearch/service/statsd/StatsdReporter.java index bec514e..7adf76c 100644 --- a/src/main/java/org/elasticsearch/service/statsd/StatsdReporter.java +++ b/src/main/java/org/elasticsearch/service/statsd/StatsdReporter.java @@ -53,11 +53,15 @@ private String join(String... parts) { } protected void logException(Exception e) { - if (logger.isDebugEnabled()) { - logger.debug("Error writing to StatsD", e); + if (this.logger.isDebugEnabled()) { + this.logger.debug("Error writing to StatsD", e); } else { - logger.warn("Error writing to StatsD: {}", e.getMessage()); + this.logger.warn("Error writing to StatsD: {}", e.getMessage()); } } + + protected ESLogger getLogger() { + return this.logger; + } } diff --git a/src/main/java/org/elasticsearch/service/statsd/StatsdReporterNodeStats.java b/src/main/java/org/elasticsearch/service/statsd/StatsdReporterNodeStats.java index c5e7ea9..b362a1c 100644 --- a/src/main/java/org/elasticsearch/service/statsd/StatsdReporterNodeStats.java +++ b/src/main/java/org/elasticsearch/service/statsd/StatsdReporterNodeStats.java @@ -18,9 +18,11 @@ public class StatsdReporterNodeStats extends StatsdReporter { private final NodeStats nodeStats; + private final Boolean statsdReportFsDetails; - public StatsdReporterNodeStats(NodeStats nodeStats) { + public StatsdReporterNodeStats(NodeStats nodeStats, Boolean statsdReportFsDetails) { this.nodeStats = nodeStats; + this.statsdReportFsDetails = statsdReportFsDetails; } public void run() { @@ -38,9 +40,8 @@ public void run() { } } - private void sendNodeThreadPoolStats(ThreadPoolStats threadPoolStats) - { - String type = this.buildMetricName("node.threadpool"); + private void sendNodeThreadPoolStats(ThreadPoolStats threadPoolStats) { + String type = this.buildMetricName("node.thread_pool"); Iterator statsIterator = threadPoolStats.iterator(); while (statsIterator.hasNext()) { ThreadPoolStats.Stats stats = statsIterator.next(); @@ -55,156 +56,174 @@ private void sendNodeThreadPoolStats(ThreadPoolStats threadPoolStats) } } - private void sendNodeTransportStats(TransportStats transportStats) - { + private void sendNodeTransportStats(TransportStats transportStats) { String type = this.buildMetricName("node.transport"); - this.sendGauge(type, "serverOpen", transportStats.serverOpen()); - this.sendCount(type, "rxCount", transportStats.rxCount()); - this.sendCount(type, "rxSizeBytes", transportStats.rxSize().bytes()); - this.sendCount(type, "txCount", transportStats.txCount()); - this.sendCount(type, "txSizeBytes", transportStats.txSize().bytes()); + this.sendGauge(type, "server_open", transportStats.serverOpen()); + this.sendCount(type, "rx_count", transportStats.rxCount()); + this.sendCount(type, "rx_size_in_bytes", transportStats.rxSize().bytes()); + this.sendCount(type, "tx_count", transportStats.txCount()); + this.sendCount(type, "tx_size_in_bytes", transportStats.txSize().bytes()); } - private void sendNodeProcessStats(ProcessStats processStats) - { + private void sendNodeProcessStats(ProcessStats processStats) { String type = this.buildMetricName("node.process"); - this.sendGauge(type, "openFileDescriptors", processStats.openFileDescriptors()); + this.sendGauge(type, "open_file_descriptors", processStats.openFileDescriptors()); + if (processStats.cpu() != null) { this.sendGauge(type + ".cpu", "percent", processStats.cpu().percent()); - this.sendGauge(type + ".cpu", "sysSeconds", processStats.cpu().sys().seconds()); - this.sendGauge(type + ".cpu", "totalSeconds", processStats.cpu().total().seconds()); - this.sendGauge(type + ".cpu", "userSeconds", processStats.cpu().user().seconds()); + this.sendGauge(type + ".cpu", "sys_in_millis", processStats.cpu().sys().millis()); + this.sendGauge(type + ".cpu", "user_in_millis", processStats.cpu().user().millis()); + this.sendGauge(type + ".cpu", "total_in_millis", processStats.cpu().total().millis()); } if (processStats.mem() != null) { - this.sendGauge(type + ".mem", "totalVirtual", processStats.mem().totalVirtual().bytes()); - this.sendGauge(type + ".mem", "resident", processStats.mem().resident().bytes()); - this.sendGauge(type + ".mem", "share", processStats.mem().share().bytes()); + this.sendGauge(type + ".mem", "resident_in_bytes", processStats.mem().resident().bytes()); + this.sendGauge(type + ".mem", "share_in_bytes", processStats.mem().share().bytes()); + this.sendGauge(type + ".mem", "total_virtual_in_bytes", processStats.mem().totalVirtual().bytes()); } } - private void sendNodeOsStats(OsStats osStats) - { + private void sendNodeOsStats(OsStats osStats) { String type = this.buildMetricName("node.os"); + // Java client does not support doubles yet :( + // https://github.com/tim-group/java-statsd-client/issues/19 + double[] loadAverage = osStats.getLoadAverage(); + if (loadAverage.length > 0) { + this.sendGauge(type + ".load_average", "1m", (long) loadAverage[0]); + this.sendGauge(type + ".load_average", "5m", (long) loadAverage[1]); + this.sendGauge(type + ".load_average", "15m", (long) loadAverage[2]); + } + if (osStats.cpu() != null) { this.sendGauge(type + ".cpu", "sys", osStats.cpu().sys()); - this.sendGauge(type + ".cpu", "idle", osStats.cpu().idle()); this.sendGauge(type + ".cpu", "user", osStats.cpu().user()); + this.sendGauge(type + ".cpu", "idle", osStats.cpu().idle()); + this.sendGauge(type + ".cpu", "stolen", osStats.cpu().stolen()); } if (osStats.mem() != null) { - this.sendGauge(type + ".mem", "freeBytes", osStats.mem().free().bytes()); - this.sendGauge(type + ".mem", "usedBytes", osStats.mem().used().bytes()); - this.sendGauge(type + ".mem", "freePercent", osStats.mem().freePercent()); - this.sendGauge(type + ".mem", "usedPercent", osStats.mem().usedPercent()); - this.sendGauge(type + ".mem", "actualFreeBytes", osStats.mem().actualFree().bytes()); - this.sendGauge(type + ".mem", "actualUsedBytes", osStats.mem().actualUsed().bytes()); + this.sendGauge(type + ".mem", "free_in_bytes", osStats.mem().free().bytes()); + this.sendGauge(type + ".mem", "used_in_bytes", osStats.mem().used().bytes()); + this.sendGauge(type + ".mem", "free_percent", osStats.mem().freePercent()); + this.sendGauge(type + ".mem", "used_percent", osStats.mem().usedPercent()); + this.sendGauge(type + ".mem", "actual_free_in_bytes", osStats.mem().actualFree().bytes()); + this.sendGauge(type + ".mem", "actual_used_in_bytes", osStats.mem().actualUsed().bytes()); } if (osStats.swap() != null) { - this.sendGauge(type + ".swap", "freeBytes", osStats.swap().free().bytes()); - this.sendGauge(type + ".swap", "usedBytes", osStats.swap().used().bytes()); + this.sendGauge(type + ".swap", "free_in_bytes", osStats.swap().free().bytes()); + this.sendGauge(type + ".swap", "used_in_bytes", osStats.swap().used().bytes()); } } - private void sendNodeNetworkStats(NetworkStats networkStats) - { + private void sendNodeNetworkStats(NetworkStats networkStats) { String type = this.buildMetricName("node.network.tcp"); NetworkStats.Tcp tcp = networkStats.tcp(); // might be null, if sigar isnt loaded if (tcp != null) { - this.sendGauge(type, "activeOpens", tcp.activeOpens()); - this.sendGauge(type, "passiveOpens", tcp.passiveOpens()); - this.sendGauge(type, "attemptFails", tcp.attemptFails()); - this.sendGauge(type, "estabResets", tcp.estabResets()); - this.sendGauge(type, "currEstab", tcp.currEstab()); - this.sendGauge(type, "inSegs", tcp.inSegs()); - this.sendGauge(type, "outSegs", tcp.outSegs()); - this.sendGauge(type, "retransSegs", tcp.retransSegs()); - this.sendGauge(type, "inErrs", tcp.inErrs()); - this.sendGauge(type, "outRsts", tcp.outRsts()); + this.sendGauge(type, "active_opens", tcp.getActiveOpens()); + this.sendGauge(type, "passive_opens", tcp.getPassiveOpens()); + this.sendGauge(type, "curr_estab", tcp.getCurrEstab()); + this.sendGauge(type, "in_segs", tcp.inSegs()); + this.sendGauge(type, "out_segs", tcp.outSegs()); + this.sendGauge(type, "retrans_segs", tcp.retransSegs()); + this.sendGauge(type, "estab_resets", tcp.estabResets()); + this.sendGauge(type, "attempt_fails", tcp.attemptFails()); + this.sendGauge(type, "in_errs", tcp.inErrs()); + this.sendGauge(type, "out_rsts", tcp.outRsts()); } } - private void sendNodeJvmStats(JvmStats jvmStats) - { + private void sendNodeJvmStats(JvmStats jvmStats) { String type = this.buildMetricName("node.jvm"); this.sendGauge(type, "uptime", jvmStats.uptime().seconds()); // mem - this.sendGauge(type + ".mem", "heapCommitted", jvmStats.mem().heapCommitted().bytes()); - this.sendGauge(type + ".mem", "heapUsed", jvmStats.mem().heapUsed().bytes()); - this.sendGauge(type + ".mem", "nonHeapCommitted", jvmStats.mem().nonHeapCommitted().bytes()); - this.sendGauge(type + ".mem", "nonHeapUsed", jvmStats.mem().nonHeapUsed().bytes()); - - Iterator memoryPoolIterator = jvmStats.mem().iterator(); - while (memoryPoolIterator.hasNext()) { - JvmStats.MemoryPool memoryPool = memoryPoolIterator.next(); - String memoryPoolType = type + ".mem.pool." + memoryPool.name(); - - this.sendGauge(memoryPoolType, "max", memoryPool.max().bytes()); - this.sendGauge(memoryPoolType, "used", memoryPool.used().bytes()); - this.sendGauge(memoryPoolType, "peakUsed", memoryPool.peakUsed().bytes()); - this.sendGauge(memoryPoolType, "peakMax", memoryPool.peakMax().bytes()); + this.sendGauge(type + ".mem", "heap_used_percent", jvmStats.mem().heapUsedPercent()); + this.sendGauge(type + ".mem", "heap_used_in_bytes", jvmStats.mem().heapUsed().bytes()); + this.sendGauge(type + ".mem", "heap_committed_in_bytes", jvmStats.mem().heapCommitted().bytes()); + this.sendGauge(type + ".mem", "non_heap_used_in_bytes", jvmStats.mem().nonHeapUsed().bytes()); + this.sendGauge(type + ".mem", "non_heap_committed_in_bytes", jvmStats.mem().nonHeapCommitted().bytes()); + for (JvmStats.MemoryPool memoryPool : jvmStats.mem()) { + String memoryPoolType = type + ".mem.pools." + memoryPool.name(); + + this.sendGauge(memoryPoolType, "max_in_bytes", memoryPool.max().bytes()); + this.sendGauge(memoryPoolType, "used_in_bytes", memoryPool.used().bytes()); + this.sendGauge(memoryPoolType, "peak_used_in_bytes", memoryPool.peakUsed().bytes()); + this.sendGauge(memoryPoolType, "peak_max_in_bytes", memoryPool.peakMax().bytes()); } // threads this.sendGauge(type + ".threads", "count", jvmStats.threads().count()); - this.sendGauge(type + ".threads", "peakCount", jvmStats.threads().peakCount()); + this.sendGauge(type + ".threads", "peak_count", jvmStats.threads().peakCount()); // garbage collectors - long gcCounter = 0; - long gcTimes = 0; - for (GarbageCollector gc : jvmStats.gc()) { - gcCounter += gc.collectionCount(); - gcTimes += gc.collectionTime().getSeconds(); - } - this.sendCount(type + ".gc", "collectionCount", gcCounter); - this.sendTime(type + ".gc", "collectionTimeSeconds", gcTimes); - for (JvmStats.GarbageCollector collector : jvmStats.gc().collectors()) { - String id = type + ".gc." + collector.name(); - this.sendCount(id, "collectionCount", collector.collectionCount()); - this.sendTime(id, "collectionTimeSeconds", collector.collectionTime().seconds()); - - JvmStats.GarbageCollector.LastGc lastGc = collector.lastGc(); - String lastGcType = type + ".lastGc"; - if (lastGc != null) { - this.sendTime(lastGcType, "time", lastGc.endTime() - lastGc.startTime()); - this.sendGauge(lastGcType, "max", lastGc.max().bytes()); - this.sendGauge(lastGcType, "beforeUsed", lastGc.beforeUsed().bytes()); - this.sendGauge(lastGcType, "afterUsed", lastGc.afterUsed().bytes()); - this.sendGauge(lastGcType, "durationSeconds", lastGc.duration().seconds()); - } + for (JvmStats.GarbageCollector collector : jvmStats.gc()) { + String id = type + ".gc.collectors." + collector.name(); + + this.sendCount(id, "collection_count", collector.collectionCount()); + this.sendTime(id, "collection_time_in_millis", collector.collectionTime().millis()); } + + // TODO: buffer pools } - private void sendNodeHttpStats(HttpStats httpStats) - { + private void sendNodeHttpStats(HttpStats httpStats) { String type = this.buildMetricName("node.http"); - this.sendGauge(type, "serverOpen", httpStats.getServerOpen()); - this.sendGauge(type, "totalOpen", httpStats.getTotalOpen()); + this.sendGauge(type, "current_open", httpStats.getServerOpen()); + this.sendGauge(type, "total_opened", httpStats.getTotalOpen()); } - private void sendNodeFsStats(FsStats fs) - { - Iterator infoIterator = fs.iterator(); - int i = 0; - while (infoIterator.hasNext()) { - String type = this.buildMetricName("node.fs") + i; - FsStats.Info info = infoIterator.next(); - this.sendGauge(type, "available", info.getAvailable().bytes()); - this.sendGauge(type, "total", info.getTotal().bytes()); - this.sendGauge(type, "free", info.getFree().bytes()); - this.sendCount(type, "diskReads", info.getDiskReads()); - this.sendCount(type, "diskReadsInBytes", info.getDiskReadSizeInBytes()); - this.sendCount(type, "diskWrites", info.getDiskWrites()); - this.sendCount(type, "diskWritesInBytes", info.getDiskWriteSizeInBytes()); - this.sendGauge(type, "diskQueue", (long) info.getDiskQueue()); - this.sendGauge(type, "diskService", (long) info.getDiskServiceTime()); - i++; + private void sendNodeFsStats(FsStats fs) { + // Send total + String type = this.buildMetricName("node.fs"); + this.sendNodeFsStatsInfo(type + ".total", fs.total()); + + // Maybe send details + if (this.statsdReportFsDetails) { + Iterator infoIterator = fs.iterator(); + while (infoIterator.hasNext()) { + FsStats.Info info = infoIterator.next(); + this.sendNodeFsStatsInfo(type, info); + } } } + + private void sendNodeFsStatsInfo(String type, FsStats.Info info) { + // Construct detailed path + String typeAppend = ""; + if (info.getPath() != null) + typeAppend += "." + info.getPath(); + if (info.getMount() != null) + typeAppend += "." + info.getMount(); + if (info.getDev() != null) + typeAppend += "." + info.getDev(); + + if (info.getAvailable().bytes() != -1) + this.sendGauge(type + typeAppend, "available_in_bytes", info.getAvailable().bytes()); + if (info.getTotal().bytes() != -1) + this.sendGauge(type + typeAppend, "total_in_bytes", info.getTotal().bytes()); + if (info.getFree().bytes() != -1) + this.sendGauge(type + typeAppend, "free_in_bytes", info.getFree().bytes()); + + // disk_io_op is sum of reads and writes (use graphite functions) + if (info.getDiskReads() != -1) + this.sendCount(type + typeAppend, "disk_reads", info.getDiskReads()); + if (info.getDiskWrites() != -1) + this.sendCount(type + typeAppend, "disk_writes", info.getDiskWrites()); + + // disk_io_size_in_bytes is sum of reads and writes (use graphite functions) + if (info.getDiskReadSizeInBytes() != -1) + this.sendCount(type + typeAppend, "disk_read_size_in_bytes", info.getDiskReadSizeInBytes()); + if (info.getDiskWriteSizeInBytes() != -1) + this.sendCount(type + typeAppend, "disk_write_size_in_bytes", info.getDiskWriteSizeInBytes()); + + if (info.getDiskQueue() != -1) + this.sendGauge(type + typeAppend, "disk_queue", (long) info.getDiskQueue()); + if (info.getDiskServiceTime() != -1) + this.sendGauge(type + typeAppend, "disk_service_time", (long) info.getDiskServiceTime()); + } } diff --git a/src/main/java/org/elasticsearch/service/statsd/StatsdService.java b/src/main/java/org/elasticsearch/service/statsd/StatsdService.java index f2490ba..7123334 100644 --- a/src/main/java/org/elasticsearch/service/statsd/StatsdService.java +++ b/src/main/java/org/elasticsearch/service/statsd/StatsdService.java @@ -27,12 +27,13 @@ public class StatsdService extends AbstractLifecycleComponent { private final ClusterService clusterService; private final IndicesService indicesService; - private NodeService nodeService; + private final NodeService nodeService; private final String statsdHost; private final Integer statsdPort; private final TimeValue statsdRefreshInternal; private final String statsdPrefix; private final Boolean statsdReportShards; + private final Boolean statsdReportFsDetails; private final StatsDClient statsdClient; private volatile Thread statsdReporterThread; @@ -57,7 +58,10 @@ public StatsdService(Settings settings, ClusterService clusterService, IndicesSe "metrics.statsd.prefix", "elasticsearch" + "." + settings.get("cluster.name") ); this.statsdReportShards = settings.getAsBoolean( - "metrics.statsd.report_shards", true + "metrics.statsd.report.shards", true + ); + this.statsdReportFsDetails = settings.getAsBoolean( + "metrics.statsd.report.fs_details", false ); this.statsdClient = new NonBlockingStatsDClient(this.statsdPrefix, this.statsdHost, this.statsdPort); } @@ -122,7 +126,8 @@ public void run() { true, // transport true, // http false // circuitBreaker - ) + ), + StatsdService.this.statsdReportFsDetails ); nodeStatsReporter .setStatsDClient(StatsdService.this.statsdClient) diff --git a/src/test/java/org/elasticsearch/module/statsd/test/StatsdPluginIntegrationTest.java b/src/test/java/org/elasticsearch/module/statsd/test/StatsdPluginIntegrationTest.java index d33f00d..39f04e3 100644 --- a/src/test/java/org/elasticsearch/module/statsd/test/StatsdPluginIntegrationTest.java +++ b/src/test/java/org/elasticsearch/module/statsd/test/StatsdPluginIntegrationTest.java @@ -53,7 +53,7 @@ public void testThatIndexingResultsInMonitoring() throws Exception ensureValidKeyNames(); assertStatsdMetricIsContained("elasticsearch." + clusterName + ".indexes." + index + ".id.0.indexing._all.indexCount:1|c"); assertStatsdMetricIsContained("elasticsearch." + clusterName + ".indexes." + index + ".id.0.indexing." + type + ".indexCount:1|c"); - assertStatsdMetricIsContained("elasticsearch." + clusterName + ".node.jvm.threads.peakCount:"); + assertStatsdMetricIsContained("elasticsearch." + clusterName + ".node.jvm.threads.peak_count:"); } @Test From e13cc588b1c1434449cc286ec79336806695ab9b Mon Sep 17 00:00:00 2001 From: Xiao Yu Date: Wed, 6 Aug 2014 18:24:17 -0400 Subject: [PATCH 10/48] Fix node indice stat name to match ES native ones and add some new stats --- .../StatsdReporterNodeIndicesStats.java | 168 +++++++++++------- .../service/statsd/StatsdService.java | 2 +- 2 files changed, 101 insertions(+), 69 deletions(-) diff --git a/src/main/java/org/elasticsearch/service/statsd/StatsdReporterNodeIndicesStats.java b/src/main/java/org/elasticsearch/service/statsd/StatsdReporterNodeIndicesStats.java index c3cbe44..ae5d129 100644 --- a/src/main/java/org/elasticsearch/service/statsd/StatsdReporterNodeIndicesStats.java +++ b/src/main/java/org/elasticsearch/service/statsd/StatsdReporterNodeIndicesStats.java @@ -13,6 +13,12 @@ import org.elasticsearch.index.refresh.RefreshStats; import org.elasticsearch.index.search.stats.SearchStats; import org.elasticsearch.index.shard.DocsStats; +import org.elasticsearch.index.store.StoreStats; +import org.elasticsearch.index.fielddata.FieldDataStats; +import org.elasticsearch.index.merge.MergeStats; +import org.elasticsearch.index.percolator.stats.PercolateStats; +import org.elasticsearch.search.suggest.completion.CompletionStats; +import org.elasticsearch.index.engine.SegmentsStats; public class StatsdReporterNodeIndicesStats extends StatsdReporter { @@ -22,105 +28,131 @@ public StatsdReporterNodeIndicesStats(NodeIndicesStats nodeIndicesStats) { this.nodeIndicesStats = nodeIndicesStats; } - public void run() - { + public void run() { try { - String type = this.buildMetricName("node"); - this.sendFilterCacheStats(type + ".filtercache", this.nodeIndicesStats.getFilterCache()); - this.sendIdCacheStats(type + ".idcache", this.nodeIndicesStats.getIdCache()); + String type = this.buildMetricName("node.indices"); this.sendDocsStats(type + ".docs", this.nodeIndicesStats.getDocs()); - this.sendFlushStats(type + ".flush", this.nodeIndicesStats.getFlush()); - this.sendGetStats(type + ".get", this.nodeIndicesStats.getGet()); + this.sendStoreStats(type + ".store", this.nodeIndicesStats.getStore()); this.sendIndexingStats(type + ".indexing", this.nodeIndicesStats.getIndexing()); - this.sendRefreshStats(type + ".refresh", this.nodeIndicesStats.getRefresh()); + this.sendGetStats(type + ".get", this.nodeIndicesStats.getGet()); this.sendSearchStats(type + ".search", this.nodeIndicesStats.getSearch()); + this.sendMergeStats(type + ".merges", this.nodeIndicesStats.getMerge()); + this.sendRefreshStats(type + ".refresh", this.nodeIndicesStats.getRefresh()); + this.sendFlushStats(type + ".flush", this.nodeIndicesStats.getFlush()); + this.sendFilterCacheStats(type + ".filter_cache", this.nodeIndicesStats.getFilterCache()); + this.sendIdCacheStats(type + ".id_cache", this.nodeIndicesStats.getIdCache()); + this.sendFielddataCacheStats(type + ".fielddata", this.nodeIndicesStats.getFieldData()); + this.sendPercolateStats(type + ".percolate", this.nodeIndicesStats.getPercolate()); + this.sendCompletionStats(type + ".completion", this.nodeIndicesStats.getCompletion()); + this.sendSegmentsStats(type + ".segments", this.nodeIndicesStats.getSegments()); } catch (Exception e) { this.logException(e); } } - private void sendSearchStats(String type, SearchStats searchStats) - { + private void sendDocsStats(String name, DocsStats docsStats) { + this.sendCount(name, "count", docsStats.getCount()); + this.sendCount(name, "deleted", docsStats.getDeleted()); + } + + private void sendStoreStats(String name, StoreStats storeStats) { + this.sendCount(name, "size_in_bytes", storeStats.sizeInBytes()); + this.sendCount(name, "throttle_time_in_millis", storeStats.getThrottleTime().millis()); + } + + private void sendIndexingStats(String type, IndexingStats indexingStats) { + IndexingStats.Stats totalStats = indexingStats.getTotal(); + this.sendIndexingStatsStats(type, totalStats); + + // TODO: Maybe print out stats to shards level? + } + + private void sendGetStats(String type, GetStats getStats) { + this.sendCount(type, "total", getStats.getCount()); + this.sendCount(type, "time_in_millis", getStats.getTimeInMillis()); + this.sendCount(type, "exists_total", getStats.getExistsCount()); + this.sendCount(type, "exists_time_in_millis", getStats.getExistsTimeInMillis()); + this.sendCount(type, "missing_total", getStats.getMissingCount()); + this.sendCount(type, "missing_time_in_millis", getStats.getMissingTimeInMillis()); + this.sendGauge(type, "current", getStats.current()); + } + + private void sendSearchStats(String type, SearchStats searchStats) { SearchStats.Stats totalSearchStats = searchStats.getTotal(); - this.sendSearchStatsStats(type + "._all", totalSearchStats); + this.sendSearchStatsStats(type, totalSearchStats); - if (searchStats.getGroupStats() != null) { - for (Map.Entry statsEntry : searchStats.getGroupStats().entrySet()) { - this.sendSearchStatsStats(type + "." + statsEntry.getKey(), statsEntry.getValue()); - } - } + // TODO: Maybe print out stats to shards level? } - private void sendSearchStatsStats(String group, SearchStats.Stats searchStats) - { - String type = this.buildMetricName("search.stats.") + group; - this.sendCount(type, "queryCount", searchStats.getQueryCount()); - this.sendCount(type, "queryTimeInMillis", searchStats.getQueryTimeInMillis()); - this.sendGauge(type, "queryCurrent", searchStats.getQueryCurrent()); - this.sendCount(type, "fetchCount", searchStats.getFetchCount()); - this.sendCount(type, "fetchTimeInMillis", searchStats.getFetchTimeInMillis()); - this.sendGauge(type, "fetchCurrent", searchStats.getFetchCurrent()); + private void sendMergeStats(String type, MergeStats mergeStats) { + this.sendGauge(type, "current", mergeStats.getCurrent()); + this.sendGauge(type, "current_docs", mergeStats.getCurrentNumDocs()); + this.sendGauge(type, "current_size_in_bytes", mergeStats.getCurrentSizeInBytes()); + this.sendCount(type, "total", mergeStats.getTotal()); + this.sendCount(type, "total_time_in_millis", mergeStats.getTotalTimeInMillis()); + this.sendCount(type, "total_docs", mergeStats.getTotalNumDocs()); + this.sendCount(type, "total_size_in_bytes", mergeStats.getTotalSizeInBytes()); } - private void sendRefreshStats(String type, RefreshStats refreshStats) - { + private void sendRefreshStats(String type, RefreshStats refreshStats) { this.sendCount(type, "total", refreshStats.getTotal()); - this.sendCount(type, "totalTimeInMillis", refreshStats.getTotalTimeInMillis()); + this.sendCount(type, "total_time_in_millis", refreshStats.getTotalTimeInMillis()); } - private void sendIndexingStats(String type, IndexingStats indexingStats) - { - IndexingStats.Stats totalStats = indexingStats.getTotal(); - this.sendStats(type + "._all", totalStats); + private void sendFlushStats(String type, FlushStats flush) { + this.sendCount(type, "total", flush.getTotal()); + this.sendCount(type, "total_time_in_millis", flush.getTotalTimeInMillis()); + } - Map typeStats = indexingStats.getTypeStats(); - if (typeStats != null) { - for (Map.Entry statsEntry : typeStats.entrySet()) { - this.sendStats(type + "." + statsEntry.getKey(), statsEntry.getValue()); - } - } + private void sendFilterCacheStats(String name, FilterCacheStats filterCache) { + this.sendGauge(name, "memory_size_in_bytes", filterCache.getMemorySizeInBytes()); + this.sendGauge(name, "evictions", filterCache.getEvictions()); } - private void sendStats(String type, IndexingStats.Stats stats) - { - this.sendCount(type, "indexCount", stats.getIndexCount()); - this.sendCount(type, "indexTimeInMillis", stats.getIndexTimeInMillis()); - this.sendGauge(type, "indexCurrent", stats.getIndexCount()); - this.sendCount(type, "deleteCount", stats.getDeleteCount()); - this.sendCount(type, "deleteTimeInMillis", stats.getDeleteTimeInMillis()); - this.sendGauge(type, "deleteCurrent", stats.getDeleteCurrent()); + private void sendIdCacheStats(String name, IdCacheStats idCache) { + this.sendGauge(name, "memory_size_in_bytes", idCache.getMemorySizeInBytes()); } - private void sendGetStats(String type, GetStats getStats) - { - this.sendCount(type, "existsCount", getStats.getExistsCount()); - this.sendCount(type, "existsTimeInMillis", getStats.getExistsTimeInMillis()); - this.sendCount(type, "missingCount", getStats.getMissingCount()); - this.sendCount(type, "missingTimeInMillis", getStats.getMissingTimeInMillis()); - this.sendGauge(type, "current", getStats.current()); + private void sendFielddataCacheStats(String name, FieldDataStats fielddataCache) { + this.sendGauge(name, "memory_size_in_bytes", fielddataCache.getMemorySizeInBytes()); + this.sendGauge(name, "evictions", fielddataCache.getEvictions()); } - private void sendFlushStats(String type, FlushStats flush) - { - this.sendCount(type, "total", flush.getTotal()); - this.sendCount(type, "totalTimeInMillis", flush.getTotalTimeInMillis()); + private void sendPercolateStats(String name, PercolateStats stats) { + this.sendCount(name, "total", stats.getCount()); + this.sendCount(name, "time_in_millis", stats.getTimeInMillis()); + this.sendGauge(name, "current", stats.getCurrent()); + this.sendCount(name, "queries", stats.getNumQueries()); + + if (stats.getMemorySizeInBytes() != -1) + this.sendGauge(name, "memory_size_in_bytes", stats.getMemorySizeInBytes()); } - private void sendDocsStats(String name, DocsStats docsStats) - { - this.sendCount(name, "count", docsStats.getCount()); - this.sendCount(name, "deleted", docsStats.getDeleted()); + private void sendCompletionStats(String name, CompletionStats stats) { + this.sendGauge(name, "size_in_bytes", stats.getSizeInBytes()); } - private void sendIdCacheStats(String name, IdCacheStats idCache) - { - this.sendGauge(name, "memorySizeInBytes", idCache.getMemorySizeInBytes()); + private void sendSegmentsStats(String name, SegmentsStats stats) { + this.sendGauge(name, "count", stats.getCount()); + this.sendGauge(name, "memory_in_bytes", stats.getMemoryInBytes()); } - private void sendFilterCacheStats(String name, FilterCacheStats filterCache) - { - this.sendGauge(name, "memorySizeInBytes", filterCache.getMemorySizeInBytes()); - this.sendGauge(name, "evictions", filterCache.getEvictions()); + private void sendIndexingStatsStats(String type, IndexingStats.Stats stats) { + this.sendCount(type, "index_total", stats.getIndexCount()); + this.sendCount(type, "index_time_in_millis", stats.getIndexTimeInMillis()); + this.sendGauge(type, "index_current", stats.getIndexCount()); + this.sendCount(type, "delete_total", stats.getDeleteCount()); + this.sendCount(type, "delete_time_in_millis", stats.getDeleteTimeInMillis()); + this.sendGauge(type, "delete_current", stats.getDeleteCurrent()); + } + + private void sendSearchStatsStats(String type, SearchStats.Stats stats) { + this.sendCount(type, "query_total", stats.getQueryCount()); + this.sendCount(type, "query_time_in_millis", stats.getQueryTimeInMillis()); + this.sendGauge(type, "query_current", stats.getQueryCurrent()); + this.sendCount(type, "fetch_total", stats.getFetchCount()); + this.sendCount(type, "fetch_time_in_millis", stats.getFetchTimeInMillis()); + this.sendGauge(type, "fetch_current", stats.getFetchCurrent()); } } diff --git a/src/main/java/org/elasticsearch/service/statsd/StatsdService.java b/src/main/java/org/elasticsearch/service/statsd/StatsdService.java index 7123334..16d0da8 100644 --- a/src/main/java/org/elasticsearch/service/statsd/StatsdService.java +++ b/src/main/java/org/elasticsearch/service/statsd/StatsdService.java @@ -58,7 +58,7 @@ public StatsdService(Settings settings, ClusterService clusterService, IndicesSe "metrics.statsd.prefix", "elasticsearch" + "." + settings.get("cluster.name") ); this.statsdReportShards = settings.getAsBoolean( - "metrics.statsd.report.shards", true + "metrics.statsd.report.shards", false ); this.statsdReportFsDetails = settings.getAsBoolean( "metrics.statsd.report.fs_details", false From d3df5a393e2cb26cf29845099c529afe4555b668 Mon Sep 17 00:00:00 2001 From: Xiao Yu Date: Wed, 6 Aug 2014 18:44:11 -0400 Subject: [PATCH 11/48] Extract out common index stats reporting functions --- .../statsd/StatsdReporterIndexStats.java | 129 ++++++++++++++++++ .../StatsdReporterNodeIndicesStats.java | 121 +--------------- 2 files changed, 130 insertions(+), 120 deletions(-) create mode 100644 src/main/java/org/elasticsearch/service/statsd/StatsdReporterIndexStats.java diff --git a/src/main/java/org/elasticsearch/service/statsd/StatsdReporterIndexStats.java b/src/main/java/org/elasticsearch/service/statsd/StatsdReporterIndexStats.java new file mode 100644 index 0000000..6f92413 --- /dev/null +++ b/src/main/java/org/elasticsearch/service/statsd/StatsdReporterIndexStats.java @@ -0,0 +1,129 @@ +package org.elasticsearch.service.statsd; + +import java.util.Iterator; +import java.util.List; +import java.util.Map; + +import org.elasticsearch.indices.NodeIndicesStats; +import org.elasticsearch.index.cache.filter.FilterCacheStats; +import org.elasticsearch.index.cache.id.IdCacheStats; +import org.elasticsearch.index.flush.FlushStats; +import org.elasticsearch.index.get.GetStats; +import org.elasticsearch.index.indexing.IndexingStats; +import org.elasticsearch.index.refresh.RefreshStats; +import org.elasticsearch.index.search.stats.SearchStats; +import org.elasticsearch.index.shard.DocsStats; +import org.elasticsearch.index.store.StoreStats; +import org.elasticsearch.index.fielddata.FieldDataStats; +import org.elasticsearch.index.merge.MergeStats; +import org.elasticsearch.index.percolator.stats.PercolateStats; +import org.elasticsearch.search.suggest.completion.CompletionStats; +import org.elasticsearch.index.engine.SegmentsStats; + +public abstract class StatsdReporterIndexStats extends StatsdReporter { + + protected void sendDocsStats(String name, DocsStats docsStats) { + this.sendCount(name, "count", docsStats.getCount()); + this.sendCount(name, "deleted", docsStats.getDeleted()); + } + + protected void sendStoreStats(String name, StoreStats storeStats) { + this.sendCount(name, "size_in_bytes", storeStats.sizeInBytes()); + this.sendCount(name, "throttle_time_in_millis", storeStats.getThrottleTime().millis()); + } + + protected void sendIndexingStats(String type, IndexingStats indexingStats) { + IndexingStats.Stats totalStats = indexingStats.getTotal(); + this.sendIndexingStatsStats(type, totalStats); + + // TODO: Maybe print out stats to shards level? + } + + protected void sendGetStats(String type, GetStats getStats) { + this.sendCount(type, "total", getStats.getCount()); + this.sendCount(type, "time_in_millis", getStats.getTimeInMillis()); + this.sendCount(type, "exists_total", getStats.getExistsCount()); + this.sendCount(type, "exists_time_in_millis", getStats.getExistsTimeInMillis()); + this.sendCount(type, "missing_total", getStats.getMissingCount()); + this.sendCount(type, "missing_time_in_millis", getStats.getMissingTimeInMillis()); + this.sendGauge(type, "current", getStats.current()); + } + + protected void sendSearchStats(String type, SearchStats searchStats) { + SearchStats.Stats totalSearchStats = searchStats.getTotal(); + this.sendSearchStatsStats(type, totalSearchStats); + + // TODO: Maybe print out stats to shards level? + } + + protected void sendMergeStats(String type, MergeStats mergeStats) { + this.sendGauge(type, "current", mergeStats.getCurrent()); + this.sendGauge(type, "current_docs", mergeStats.getCurrentNumDocs()); + this.sendGauge(type, "current_size_in_bytes", mergeStats.getCurrentSizeInBytes()); + this.sendCount(type, "total", mergeStats.getTotal()); + this.sendCount(type, "total_time_in_millis", mergeStats.getTotalTimeInMillis()); + this.sendCount(type, "total_docs", mergeStats.getTotalNumDocs()); + this.sendCount(type, "total_size_in_bytes", mergeStats.getTotalSizeInBytes()); + } + + protected void sendRefreshStats(String type, RefreshStats refreshStats) { + this.sendCount(type, "total", refreshStats.getTotal()); + this.sendCount(type, "total_time_in_millis", refreshStats.getTotalTimeInMillis()); + } + + protected void sendFlushStats(String type, FlushStats flush) { + this.sendCount(type, "total", flush.getTotal()); + this.sendCount(type, "total_time_in_millis", flush.getTotalTimeInMillis()); + } + + protected void sendFilterCacheStats(String name, FilterCacheStats filterCache) { + this.sendGauge(name, "memory_size_in_bytes", filterCache.getMemorySizeInBytes()); + this.sendGauge(name, "evictions", filterCache.getEvictions()); + } + + protected void sendIdCacheStats(String name, IdCacheStats idCache) { + this.sendGauge(name, "memory_size_in_bytes", idCache.getMemorySizeInBytes()); + } + + protected void sendFielddataCacheStats(String name, FieldDataStats fielddataCache) { + this.sendGauge(name, "memory_size_in_bytes", fielddataCache.getMemorySizeInBytes()); + this.sendGauge(name, "evictions", fielddataCache.getEvictions()); + } + + protected void sendPercolateStats(String name, PercolateStats stats) { + this.sendCount(name, "total", stats.getCount()); + this.sendCount(name, "time_in_millis", stats.getTimeInMillis()); + this.sendGauge(name, "current", stats.getCurrent()); + this.sendCount(name, "queries", stats.getNumQueries()); + + if (stats.getMemorySizeInBytes() != -1) + this.sendGauge(name, "memory_size_in_bytes", stats.getMemorySizeInBytes()); + } + + protected void sendCompletionStats(String name, CompletionStats stats) { + this.sendGauge(name, "size_in_bytes", stats.getSizeInBytes()); + } + + protected void sendSegmentsStats(String name, SegmentsStats stats) { + this.sendGauge(name, "count", stats.getCount()); + this.sendGauge(name, "memory_in_bytes", stats.getMemoryInBytes()); + } + + protected void sendIndexingStatsStats(String type, IndexingStats.Stats stats) { + this.sendCount(type, "index_total", stats.getIndexCount()); + this.sendCount(type, "index_time_in_millis", stats.getIndexTimeInMillis()); + this.sendGauge(type, "index_current", stats.getIndexCount()); + this.sendCount(type, "delete_total", stats.getDeleteCount()); + this.sendCount(type, "delete_time_in_millis", stats.getDeleteTimeInMillis()); + this.sendGauge(type, "delete_current", stats.getDeleteCurrent()); + } + + protected void sendSearchStatsStats(String type, SearchStats.Stats stats) { + this.sendCount(type, "query_total", stats.getQueryCount()); + this.sendCount(type, "query_time_in_millis", stats.getQueryTimeInMillis()); + this.sendGauge(type, "query_current", stats.getQueryCurrent()); + this.sendCount(type, "fetch_total", stats.getFetchCount()); + this.sendCount(type, "fetch_time_in_millis", stats.getFetchTimeInMillis()); + this.sendGauge(type, "fetch_current", stats.getFetchCurrent()); + } +} diff --git a/src/main/java/org/elasticsearch/service/statsd/StatsdReporterNodeIndicesStats.java b/src/main/java/org/elasticsearch/service/statsd/StatsdReporterNodeIndicesStats.java index ae5d129..39cf8d5 100644 --- a/src/main/java/org/elasticsearch/service/statsd/StatsdReporterNodeIndicesStats.java +++ b/src/main/java/org/elasticsearch/service/statsd/StatsdReporterNodeIndicesStats.java @@ -5,22 +5,8 @@ import java.util.Map; import org.elasticsearch.indices.NodeIndicesStats; -import org.elasticsearch.index.cache.filter.FilterCacheStats; -import org.elasticsearch.index.cache.id.IdCacheStats; -import org.elasticsearch.index.flush.FlushStats; -import org.elasticsearch.index.get.GetStats; -import org.elasticsearch.index.indexing.IndexingStats; -import org.elasticsearch.index.refresh.RefreshStats; -import org.elasticsearch.index.search.stats.SearchStats; -import org.elasticsearch.index.shard.DocsStats; -import org.elasticsearch.index.store.StoreStats; -import org.elasticsearch.index.fielddata.FieldDataStats; -import org.elasticsearch.index.merge.MergeStats; -import org.elasticsearch.index.percolator.stats.PercolateStats; -import org.elasticsearch.search.suggest.completion.CompletionStats; -import org.elasticsearch.index.engine.SegmentsStats; -public class StatsdReporterNodeIndicesStats extends StatsdReporter { +public class StatsdReporterNodeIndicesStats extends StatsdReporterIndexStats { private final NodeIndicesStats nodeIndicesStats; @@ -50,109 +36,4 @@ public void run() { this.logException(e); } } - - private void sendDocsStats(String name, DocsStats docsStats) { - this.sendCount(name, "count", docsStats.getCount()); - this.sendCount(name, "deleted", docsStats.getDeleted()); - } - - private void sendStoreStats(String name, StoreStats storeStats) { - this.sendCount(name, "size_in_bytes", storeStats.sizeInBytes()); - this.sendCount(name, "throttle_time_in_millis", storeStats.getThrottleTime().millis()); - } - - private void sendIndexingStats(String type, IndexingStats indexingStats) { - IndexingStats.Stats totalStats = indexingStats.getTotal(); - this.sendIndexingStatsStats(type, totalStats); - - // TODO: Maybe print out stats to shards level? - } - - private void sendGetStats(String type, GetStats getStats) { - this.sendCount(type, "total", getStats.getCount()); - this.sendCount(type, "time_in_millis", getStats.getTimeInMillis()); - this.sendCount(type, "exists_total", getStats.getExistsCount()); - this.sendCount(type, "exists_time_in_millis", getStats.getExistsTimeInMillis()); - this.sendCount(type, "missing_total", getStats.getMissingCount()); - this.sendCount(type, "missing_time_in_millis", getStats.getMissingTimeInMillis()); - this.sendGauge(type, "current", getStats.current()); - } - - private void sendSearchStats(String type, SearchStats searchStats) { - SearchStats.Stats totalSearchStats = searchStats.getTotal(); - this.sendSearchStatsStats(type, totalSearchStats); - - // TODO: Maybe print out stats to shards level? - } - - private void sendMergeStats(String type, MergeStats mergeStats) { - this.sendGauge(type, "current", mergeStats.getCurrent()); - this.sendGauge(type, "current_docs", mergeStats.getCurrentNumDocs()); - this.sendGauge(type, "current_size_in_bytes", mergeStats.getCurrentSizeInBytes()); - this.sendCount(type, "total", mergeStats.getTotal()); - this.sendCount(type, "total_time_in_millis", mergeStats.getTotalTimeInMillis()); - this.sendCount(type, "total_docs", mergeStats.getTotalNumDocs()); - this.sendCount(type, "total_size_in_bytes", mergeStats.getTotalSizeInBytes()); - } - - private void sendRefreshStats(String type, RefreshStats refreshStats) { - this.sendCount(type, "total", refreshStats.getTotal()); - this.sendCount(type, "total_time_in_millis", refreshStats.getTotalTimeInMillis()); - } - - private void sendFlushStats(String type, FlushStats flush) { - this.sendCount(type, "total", flush.getTotal()); - this.sendCount(type, "total_time_in_millis", flush.getTotalTimeInMillis()); - } - - private void sendFilterCacheStats(String name, FilterCacheStats filterCache) { - this.sendGauge(name, "memory_size_in_bytes", filterCache.getMemorySizeInBytes()); - this.sendGauge(name, "evictions", filterCache.getEvictions()); - } - - private void sendIdCacheStats(String name, IdCacheStats idCache) { - this.sendGauge(name, "memory_size_in_bytes", idCache.getMemorySizeInBytes()); - } - - private void sendFielddataCacheStats(String name, FieldDataStats fielddataCache) { - this.sendGauge(name, "memory_size_in_bytes", fielddataCache.getMemorySizeInBytes()); - this.sendGauge(name, "evictions", fielddataCache.getEvictions()); - } - - private void sendPercolateStats(String name, PercolateStats stats) { - this.sendCount(name, "total", stats.getCount()); - this.sendCount(name, "time_in_millis", stats.getTimeInMillis()); - this.sendGauge(name, "current", stats.getCurrent()); - this.sendCount(name, "queries", stats.getNumQueries()); - - if (stats.getMemorySizeInBytes() != -1) - this.sendGauge(name, "memory_size_in_bytes", stats.getMemorySizeInBytes()); - } - - private void sendCompletionStats(String name, CompletionStats stats) { - this.sendGauge(name, "size_in_bytes", stats.getSizeInBytes()); - } - - private void sendSegmentsStats(String name, SegmentsStats stats) { - this.sendGauge(name, "count", stats.getCount()); - this.sendGauge(name, "memory_in_bytes", stats.getMemoryInBytes()); - } - - private void sendIndexingStatsStats(String type, IndexingStats.Stats stats) { - this.sendCount(type, "index_total", stats.getIndexCount()); - this.sendCount(type, "index_time_in_millis", stats.getIndexTimeInMillis()); - this.sendGauge(type, "index_current", stats.getIndexCount()); - this.sendCount(type, "delete_total", stats.getDeleteCount()); - this.sendCount(type, "delete_time_in_millis", stats.getDeleteTimeInMillis()); - this.sendGauge(type, "delete_current", stats.getDeleteCurrent()); - } - - private void sendSearchStatsStats(String type, SearchStats.Stats stats) { - this.sendCount(type, "query_total", stats.getQueryCount()); - this.sendCount(type, "query_time_in_millis", stats.getQueryTimeInMillis()); - this.sendGauge(type, "query_current", stats.getQueryCurrent()); - this.sendCount(type, "fetch_total", stats.getFetchCount()); - this.sendCount(type, "fetch_time_in_millis", stats.getFetchTimeInMillis()); - this.sendGauge(type, "fetch_current", stats.getFetchCurrent()); - } } From 0351865b796d1bc2031d1682f147bcc0048c0e5d Mon Sep 17 00:00:00 2001 From: Xiao Yu Date: Thu, 7 Aug 2014 09:57:41 -0400 Subject: [PATCH 12/48] Use common class to report shard stats --- .../service/statsd/StatsdReporterIndices.java | 125 +++--------------- 1 file changed, 15 insertions(+), 110 deletions(-) diff --git a/src/main/java/org/elasticsearch/service/statsd/StatsdReporterIndices.java b/src/main/java/org/elasticsearch/service/statsd/StatsdReporterIndices.java index fa0264e..f453c2d 100644 --- a/src/main/java/org/elasticsearch/service/statsd/StatsdReporterIndices.java +++ b/src/main/java/org/elasticsearch/service/statsd/StatsdReporterIndices.java @@ -5,16 +5,8 @@ import java.util.Map; import org.elasticsearch.index.shard.service.IndexShard; -import org.elasticsearch.index.get.GetStats; -import org.elasticsearch.index.indexing.IndexingStats; -import org.elasticsearch.index.merge.MergeStats; -import org.elasticsearch.index.refresh.RefreshStats; -import org.elasticsearch.index.search.stats.SearchStats; -import org.elasticsearch.index.shard.DocsStats; -import org.elasticsearch.index.store.StoreStats; -import org.elasticsearch.index.warmer.WarmerStats; -public class StatsdReporterIndices extends StatsdReporter { +public class StatsdReporterIndices extends StatsdReporterIndexStats { private final List indexShards; @@ -22,113 +14,26 @@ public StatsdReporterIndices(List indexShards) { this.indexShards = indexShards; } - public void run() - { + public void run() { try { for (IndexShard indexShard : this.indexShards) { - String type = this.buildMetricName("indexes.") + indexShard.shardId().index().name() + ".id." + indexShard.shardId().id(); - this.sendSearchStats(type + ".search", indexShard.searchStats()); - this.sendGetStats(type + ".get", indexShard.getStats()); + String type = this.buildMetricName("index.") + indexShard.shardId().index().name() + ".shard." + indexShard.shardId().id(); this.sendDocsStats(type + ".docs", indexShard.docStats()); - this.sendRefreshStats(type + ".refresh", indexShard.refreshStats()); - this.sendIndexingStats(type + ".indexing", indexShard.indexingStats("_all")); - this.sendMergeStats(type + ".merge", indexShard.mergeStats()); - this.sendWarmerStats(type + ".warmer", indexShard.warmerStats()); this.sendStoreStats(type + ".store", indexShard.storeStats()); + this.sendIndexingStats(type + ".indexing", indexShard.indexingStats("_all")); + this.sendGetStats(type + ".get", indexShard.getStats()); + this.sendSearchStats(type + ".search", indexShard.searchStats("_all")); + this.sendMergeStats(type + ".merges", indexShard.mergeStats()); + this.sendRefreshStats(type + ".refresh", indexShard.refreshStats()); + this.sendFlushStats(type + ".flush", indexShard.flushStats()); + this.sendFilterCacheStats(type + ".filter_cache", indexShard.filterCacheStats()); + this.sendIdCacheStats(type + ".id_cache", indexShard.idCacheStats()); + this.sendFielddataCacheStats(type + ".fielddata", indexShard.fieldDataStats("_all")); + this.sendCompletionStats(type + ".completion", indexShard.completionStats("_all")); + this.sendSegmentsStats(type + ".segments", indexShard.segmentStats()); } - } - catch (Exception e) { + } catch (Exception e) { this.logException(e); } } - - private void sendStoreStats(String type, StoreStats storeStats) - { - this.sendGauge(type, "sizeInBytes", storeStats.sizeInBytes()); - this.sendGauge(type, "throttleTimeInNanos", storeStats.throttleTime().getNanos()); - } - - private void sendWarmerStats(String type, WarmerStats warmerStats) - { - this.sendGauge(type, "current", warmerStats.current()); - this.sendGauge(type, "total", warmerStats.total()); - this.sendTime(type, "totalTimeInMillis", warmerStats.totalTimeInMillis()); - } - - private void sendMergeStats(String type, MergeStats mergeStats) - { - this.sendGauge(type, "total", mergeStats.getTotal()); - this.sendTime(type, "totalTimeInMillis", mergeStats.getTotalTimeInMillis()); - this.sendGauge(type, "totalNumDocs", mergeStats.getTotalNumDocs()); - this.sendGauge(type, "current", mergeStats.getCurrent()); - this.sendGauge(type, "currentNumDocs", mergeStats.getCurrentNumDocs()); - this.sendGauge(type, "currentSizeInBytes", mergeStats.getCurrentSizeInBytes()); - } - - private void sendSearchStats(String type, SearchStats searchStats) - { - SearchStats.Stats totalSearchStats = searchStats.getTotal(); - this.sendSearchStatsStats(type + "._all", totalSearchStats); - - if (searchStats.getGroupStats() != null) { - for (Map.Entry statsEntry : searchStats.getGroupStats().entrySet()) { - this.sendSearchStatsStats(type + "." + statsEntry.getKey(), statsEntry.getValue()); - } - } - } - - private void sendSearchStatsStats(String group, SearchStats.Stats searchStats) - { - String type = this.buildMetricName("search.stats.") + group; - this.sendCount(type, "queryCount", searchStats.getQueryCount()); - this.sendCount(type, "queryTimeInMillis", searchStats.getQueryTimeInMillis()); - this.sendGauge(type, "queryCurrent", searchStats.getQueryCurrent()); - this.sendCount(type, "fetchCount", searchStats.getFetchCount()); - this.sendCount(type, "fetchTimeInMillis", searchStats.getFetchTimeInMillis()); - this.sendGauge(type, "fetchCurrent", searchStats.getFetchCurrent()); - } - - private void sendRefreshStats(String type, RefreshStats refreshStats) - { - this.sendCount(type, "total", refreshStats.getTotal()); - this.sendCount(type, "totalTimeInMillis", refreshStats.getTotalTimeInMillis()); - } - - private void sendIndexingStats(String type, IndexingStats indexingStats) - { - IndexingStats.Stats totalStats = indexingStats.getTotal(); - this.sendStats(type + "._all", totalStats); - - Map typeStats = indexingStats.getTypeStats(); - if (typeStats != null) { - for (Map.Entry statsEntry : typeStats.entrySet()) { - this.sendStats(type + "." + statsEntry.getKey(), statsEntry.getValue()); - } - } - } - - private void sendStats(String type, IndexingStats.Stats stats) - { - this.sendCount(type, "indexCount", stats.getIndexCount()); - this.sendCount(type, "indexTimeInMillis", stats.getIndexTimeInMillis()); - this.sendGauge(type, "indexCurrent", stats.getIndexCount()); - this.sendCount(type, "deleteCount", stats.getDeleteCount()); - this.sendCount(type, "deleteTimeInMillis", stats.getDeleteTimeInMillis()); - this.sendGauge(type, "deleteCurrent", stats.getDeleteCurrent()); - } - - private void sendGetStats(String type, GetStats getStats) - { - this.sendCount(type, "existsCount", getStats.getExistsCount()); - this.sendCount(type, "existsTimeInMillis", getStats.getExistsTimeInMillis()); - this.sendCount(type, "missingCount", getStats.getMissingCount()); - this.sendCount(type, "missingTimeInMillis", getStats.getMissingTimeInMillis()); - this.sendGauge(type, "current", getStats.current()); - } - - private void sendDocsStats(String name, DocsStats docsStats) - { - this.sendCount(name, "count", docsStats.getCount()); - this.sendCount(name, "deleted", docsStats.getDeleted()); - } } From 1363d45f0781770dc64f47eab5c2ea79b91e1391 Mon Sep 17 00:00:00 2001 From: Xiao Yu Date: Thu, 7 Aug 2014 10:04:15 -0400 Subject: [PATCH 13/48] Report node stats on a per node basis with node name in the stats key --- .../statsd/StatsdReporterNodeStats.java | 24 +++++++---- .../service/statsd/StatsdService.java | 41 ++++++++++--------- 2 files changed, 36 insertions(+), 29 deletions(-) diff --git a/src/main/java/org/elasticsearch/service/statsd/StatsdReporterNodeStats.java b/src/main/java/org/elasticsearch/service/statsd/StatsdReporterNodeStats.java index b362a1c..424e89f 100644 --- a/src/main/java/org/elasticsearch/service/statsd/StatsdReporterNodeStats.java +++ b/src/main/java/org/elasticsearch/service/statsd/StatsdReporterNodeStats.java @@ -18,10 +18,12 @@ public class StatsdReporterNodeStats extends StatsdReporter { private final NodeStats nodeStats; + private final String nodeName; private final Boolean statsdReportFsDetails; - public StatsdReporterNodeStats(NodeStats nodeStats, Boolean statsdReportFsDetails) { + public StatsdReporterNodeStats(NodeStats nodeStats, String nodeName, Boolean statsdReportFsDetails) { this.nodeStats = nodeStats; + this.nodeName = nodeName; this.statsdReportFsDetails = statsdReportFsDetails; } @@ -41,7 +43,7 @@ public void run() { } private void sendNodeThreadPoolStats(ThreadPoolStats threadPoolStats) { - String type = this.buildMetricName("node.thread_pool"); + String type = this.getPrefix("thread_pool"); Iterator statsIterator = threadPoolStats.iterator(); while (statsIterator.hasNext()) { ThreadPoolStats.Stats stats = statsIterator.next(); @@ -57,7 +59,7 @@ private void sendNodeThreadPoolStats(ThreadPoolStats threadPoolStats) { } private void sendNodeTransportStats(TransportStats transportStats) { - String type = this.buildMetricName("node.transport"); + String type = this.getPrefix("transport"); this.sendGauge(type, "server_open", transportStats.serverOpen()); this.sendCount(type, "rx_count", transportStats.rxCount()); this.sendCount(type, "rx_size_in_bytes", transportStats.rxSize().bytes()); @@ -66,7 +68,7 @@ private void sendNodeTransportStats(TransportStats transportStats) { } private void sendNodeProcessStats(ProcessStats processStats) { - String type = this.buildMetricName("node.process"); + String type = this.getPrefix("process"); this.sendGauge(type, "open_file_descriptors", processStats.openFileDescriptors()); @@ -85,7 +87,7 @@ private void sendNodeProcessStats(ProcessStats processStats) { } private void sendNodeOsStats(OsStats osStats) { - String type = this.buildMetricName("node.os"); + String type = this.getPrefix("os"); // Java client does not support doubles yet :( // https://github.com/tim-group/java-statsd-client/issues/19 @@ -119,7 +121,7 @@ private void sendNodeOsStats(OsStats osStats) { } private void sendNodeNetworkStats(NetworkStats networkStats) { - String type = this.buildMetricName("node.network.tcp"); + String type = this.getPrefix("network.tcp"); NetworkStats.Tcp tcp = networkStats.tcp(); // might be null, if sigar isnt loaded @@ -138,7 +140,7 @@ private void sendNodeNetworkStats(NetworkStats networkStats) { } private void sendNodeJvmStats(JvmStats jvmStats) { - String type = this.buildMetricName("node.jvm"); + String type = this.getPrefix("jvm"); this.sendGauge(type, "uptime", jvmStats.uptime().seconds()); // mem @@ -172,14 +174,14 @@ private void sendNodeJvmStats(JvmStats jvmStats) { } private void sendNodeHttpStats(HttpStats httpStats) { - String type = this.buildMetricName("node.http"); + String type = this.getPrefix("http"); this.sendGauge(type, "current_open", httpStats.getServerOpen()); this.sendGauge(type, "total_opened", httpStats.getTotalOpen()); } private void sendNodeFsStats(FsStats fs) { // Send total - String type = this.buildMetricName("node.fs"); + String type = this.getPrefix("fs"); this.sendNodeFsStatsInfo(type + ".total", fs.total()); // Maybe send details @@ -226,4 +228,8 @@ private void sendNodeFsStatsInfo(String type, FsStats.Info info) { if (info.getDiskServiceTime() != -1) this.sendGauge(type + typeAppend, "disk_service_time", (long) info.getDiskServiceTime()); } + + private String getPrefix(String prefix) { + return this.buildMetricName( "node." + this.nodeName + "." + prefix ); + } } diff --git a/src/main/java/org/elasticsearch/service/statsd/StatsdService.java b/src/main/java/org/elasticsearch/service/statsd/StatsdService.java index 16d0da8..8b60010 100644 --- a/src/main/java/org/elasticsearch/service/statsd/StatsdService.java +++ b/src/main/java/org/elasticsearch/service/statsd/StatsdService.java @@ -111,28 +111,29 @@ public void run() { .equals(Lifecycle.State.STARTED); if (node != null && isClusterStarted) { + // Report node stats + StatsdReporter nodeStatsReporter = new StatsdReporterNodeStats( + StatsdService.this.nodeService.stats( + new CommonStatsFlags().clear(), // indices + true, // os + true, // process + true, // jvm + true, // threadPool + true, // network + true, // fs + true, // transport + true, // http + false // circuitBreaker + ), + node.getName(), + StatsdService.this.statsdReportFsDetails + ); + nodeStatsReporter + .setStatsDClient(StatsdService.this.statsdClient) + .run(); + // Master Node sends cluster wide stats if (node.isMasterNode()) { - // Report node stats - StatsdReporter nodeStatsReporter = new StatsdReporterNodeStats( - StatsdService.this.nodeService.stats( - new CommonStatsFlags().clear(), // indices - true, // os - true, // process - true, // jvm - true, // threadPool - true, // network - true, // fs - true, // transport - true, // http - false // circuitBreaker - ), - StatsdService.this.statsdReportFsDetails - ); - nodeStatsReporter - .setStatsDClient(StatsdService.this.statsdClient) - .run(); - // Report node indice stats StatsdReporter nodeIndicesStatsReporter = new StatsdReporterNodeIndicesStats( StatsdService.this.indicesService.stats( From 58bb0e57a39364208d7fcee34ab045dcf1aae28a Mon Sep 17 00:00:00 2001 From: Xiao Yu Date: Thu, 7 Aug 2014 10:23:35 -0400 Subject: [PATCH 14/48] Audit stat types (counter vs guage) to match up ES and StatsD types for node stats --- .../statsd/StatsdReporterNodeStats.java | 35 ++++++++++--------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/src/main/java/org/elasticsearch/service/statsd/StatsdReporterNodeStats.java b/src/main/java/org/elasticsearch/service/statsd/StatsdReporterNodeStats.java index 424e89f..47b5c7e 100644 --- a/src/main/java/org/elasticsearch/service/statsd/StatsdReporterNodeStats.java +++ b/src/main/java/org/elasticsearch/service/statsd/StatsdReporterNodeStats.java @@ -54,7 +54,7 @@ private void sendNodeThreadPoolStats(ThreadPoolStats threadPoolStats) { this.sendGauge(id, "active", stats.getActive()); this.sendGauge(id, "rejected", stats.getRejected()); this.sendGauge(id, "largest", stats.getLargest()); - this.sendGauge(id, "completed", stats.getCompleted()); + this.sendCount(id, "completed", stats.getCompleted()); } } @@ -74,9 +74,9 @@ private void sendNodeProcessStats(ProcessStats processStats) { if (processStats.cpu() != null) { this.sendGauge(type + ".cpu", "percent", processStats.cpu().percent()); - this.sendGauge(type + ".cpu", "sys_in_millis", processStats.cpu().sys().millis()); - this.sendGauge(type + ".cpu", "user_in_millis", processStats.cpu().user().millis()); - this.sendGauge(type + ".cpu", "total_in_millis", processStats.cpu().total().millis()); + this.sendCount(type + ".cpu", "sys_in_millis", processStats.cpu().sys().millis()); + this.sendCount(type + ".cpu", "user_in_millis", processStats.cpu().user().millis()); + this.sendCount(type + ".cpu", "total_in_millis", processStats.cpu().total().millis()); } if (processStats.mem() != null) { @@ -126,22 +126,21 @@ private void sendNodeNetworkStats(NetworkStats networkStats) { // might be null, if sigar isnt loaded if (tcp != null) { - this.sendGauge(type, "active_opens", tcp.getActiveOpens()); - this.sendGauge(type, "passive_opens", tcp.getPassiveOpens()); + this.sendCount(type, "active_opens", tcp.getActiveOpens()); + this.sendCount(type, "passive_opens", tcp.getPassiveOpens()); this.sendGauge(type, "curr_estab", tcp.getCurrEstab()); - this.sendGauge(type, "in_segs", tcp.inSegs()); - this.sendGauge(type, "out_segs", tcp.outSegs()); - this.sendGauge(type, "retrans_segs", tcp.retransSegs()); - this.sendGauge(type, "estab_resets", tcp.estabResets()); - this.sendGauge(type, "attempt_fails", tcp.attemptFails()); - this.sendGauge(type, "in_errs", tcp.inErrs()); - this.sendGauge(type, "out_rsts", tcp.outRsts()); + this.sendCount(type, "in_segs", tcp.inSegs()); + this.sendCount(type, "out_segs", tcp.outSegs()); + this.sendCount(type, "retrans_segs", tcp.retransSegs()); + this.sendCount(type, "estab_resets", tcp.estabResets()); + this.sendCount(type, "attempt_fails", tcp.attemptFails()); + this.sendCount(type, "in_errs", tcp.inErrs()); + this.sendCount(type, "out_rsts", tcp.outRsts()); } } private void sendNodeJvmStats(JvmStats jvmStats) { String type = this.getPrefix("jvm"); - this.sendGauge(type, "uptime", jvmStats.uptime().seconds()); // mem this.sendGauge(type + ".mem", "heap_used_percent", jvmStats.mem().heapUsedPercent()); @@ -167,7 +166,7 @@ private void sendNodeJvmStats(JvmStats jvmStats) { String id = type + ".gc.collectors." + collector.name(); this.sendCount(id, "collection_count", collector.collectionCount()); - this.sendTime(id, "collection_time_in_millis", collector.collectionTime().millis()); + this.sendCount(id, "collection_time_in_millis", collector.collectionTime().millis()); } // TODO: buffer pools @@ -176,7 +175,7 @@ private void sendNodeJvmStats(JvmStats jvmStats) { private void sendNodeHttpStats(HttpStats httpStats) { String type = this.getPrefix("http"); this.sendGauge(type, "current_open", httpStats.getServerOpen()); - this.sendGauge(type, "total_opened", httpStats.getTotalOpen()); + this.sendCount(type, "total_opened", httpStats.getTotalOpen()); } private void sendNodeFsStats(FsStats fs) { @@ -189,7 +188,7 @@ private void sendNodeFsStats(FsStats fs) { Iterator infoIterator = fs.iterator(); while (infoIterator.hasNext()) { FsStats.Info info = infoIterator.next(); - this.sendNodeFsStatsInfo(type, info); + this.sendNodeFsStatsInfo(type + ".data", info); } } } @@ -223,10 +222,12 @@ private void sendNodeFsStatsInfo(String type, FsStats.Info info) { if (info.getDiskWriteSizeInBytes() != -1) this.sendCount(type + typeAppend, "disk_write_size_in_bytes", info.getDiskWriteSizeInBytes()); + /** TODO: Find out if these stats are useful. if (info.getDiskQueue() != -1) this.sendGauge(type + typeAppend, "disk_queue", (long) info.getDiskQueue()); if (info.getDiskServiceTime() != -1) this.sendGauge(type + typeAppend, "disk_service_time", (long) info.getDiskServiceTime()); + */ } private String getPrefix(String prefix) { From 980af4958ef1a6f526e185cf7fd04e415ac128f0 Mon Sep 17 00:00:00 2001 From: Xiao Yu Date: Thu, 7 Aug 2014 10:32:30 -0400 Subject: [PATCH 15/48] Allow customization of node name used in StatsD metric key for node level stats --- .../org/elasticsearch/service/statsd/StatsdService.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/elasticsearch/service/statsd/StatsdService.java b/src/main/java/org/elasticsearch/service/statsd/StatsdService.java index 8b60010..2462052 100644 --- a/src/main/java/org/elasticsearch/service/statsd/StatsdService.java +++ b/src/main/java/org/elasticsearch/service/statsd/StatsdService.java @@ -32,6 +32,7 @@ public class StatsdService extends AbstractLifecycleComponent { private final Integer statsdPort; private final TimeValue statsdRefreshInternal; private final String statsdPrefix; + private final String statsdNodeName; private final Boolean statsdReportShards; private final Boolean statsdReportFsDetails; private final StatsDClient statsdClient; @@ -57,6 +58,9 @@ public StatsdService(Settings settings, ClusterService clusterService, IndicesSe this.statsdPrefix = settings.get( "metrics.statsd.prefix", "elasticsearch" + "." + settings.get("cluster.name") ); + this.statsdNodeName = settings.get( + "metrics.statsd.node_name" + ); this.statsdReportShards = settings.getAsBoolean( "metrics.statsd.report.shards", false ); @@ -125,7 +129,7 @@ public void run() { true, // http false // circuitBreaker ), - node.getName(), + StatsdService.this.statsdNodeName == null ? node.getName() : StatsdService.this.statsdNodeName, StatsdService.this.statsdReportFsDetails ); nodeStatsReporter From 9fc5d9e12efb47ba2ceeef0a7b135486e92e8c35 Mon Sep 17 00:00:00 2001 From: Xiao Yu Date: Thu, 7 Aug 2014 10:33:43 -0400 Subject: [PATCH 16/48] Spin up 2 nodes during tests --- .../test/StatsdPluginIntegrationTest.java | 27 +++++++++++-------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/src/test/java/org/elasticsearch/module/statsd/test/StatsdPluginIntegrationTest.java b/src/test/java/org/elasticsearch/module/statsd/test/StatsdPluginIntegrationTest.java index 39f04e3..f7a2177 100644 --- a/src/test/java/org/elasticsearch/module/statsd/test/StatsdPluginIntegrationTest.java +++ b/src/test/java/org/elasticsearch/module/statsd/test/StatsdPluginIntegrationTest.java @@ -23,32 +23,37 @@ public class StatsdPluginIntegrationTest private String clusterName = RandomStringGenerator.randomAlphabetic(10); private String index = RandomStringGenerator.randomAlphabetic(6).toLowerCase(); private String type = RandomStringGenerator.randomAlphabetic(6).toLowerCase(); - private Node node; + private Node node_1; + private Node node_2; @Before public void startStatsdMockServerAndNode() throws Exception { statsdMockServer = new StatsdMockServer(STATSD_SERVER_PORT); statsdMockServer.start(); - node = createNode(clusterName, 1, STATSD_SERVER_PORT, "1s"); + node_1 = createNode(clusterName, 2, STATSD_SERVER_PORT, "1s"); + node_2 = createNode(clusterName, 2, STATSD_SERVER_PORT, "1s"); } @After public void stopStatsdServer() throws Exception { statsdMockServer.close(); - if (!node.isClosed()) { - node.close(); + if (!node_1.isClosed()) { + node_1.close(); + } + if (!node_2.isClosed()) { + node_2.close(); } } @Test public void testThatIndexingResultsInMonitoring() throws Exception { - IndexResponse indexResponse = indexElement(node, index, type, "value"); + IndexResponse indexResponse = indexElement(node_1, index, type, "value"); assertThat(indexResponse.getId(), is(notNullValue())); - Thread.sleep(2000); + Thread.sleep(4000); ensureValidKeyNames(); assertStatsdMetricIsContained("elasticsearch." + clusterName + ".indexes." + index + ".id.0.indexing._all.indexCount:1|c"); @@ -60,18 +65,18 @@ public void testThatIndexingResultsInMonitoring() throws Exception public void masterFailOverShouldWork() throws Exception { String clusterName = RandomStringGenerator.randomAlphabetic(10); - IndexResponse indexResponse = indexElement(node, index, type, "value"); + IndexResponse indexResponse = indexElement(node_1, index, type, "value"); assertThat(indexResponse.getId(), is(notNullValue())); - Node origNode = node; - node = createNode(clusterName, 1, STATSD_SERVER_PORT, "1s"); + Node origNode = node_1; + node_1 = createNode(clusterName, 1, STATSD_SERVER_PORT, "1s"); statsdMockServer.content.clear(); origNode.stop(); - indexResponse = indexElement(node, index, type, "value"); + indexResponse = indexElement(node_1, index, type, "value"); assertThat(indexResponse.getId(), is(notNullValue())); // wait for master fail over and writing to graph reporter - Thread.sleep(2000); + Thread.sleep(4000); assertStatsdMetricIsContained("elasticsearch." + clusterName + ".indexes." + index + ".id.0.indexing._all.indexCount:1|c"); } From 84fffc49a2c269ee45026603ffb6fb60b0536c2f Mon Sep 17 00:00:00 2001 From: Xiao Yu Date: Thu, 7 Aug 2014 10:48:48 -0400 Subject: [PATCH 17/48] Index more docs during tests to get more data --- .../statsd/test/StatsdPluginIntegrationTest.java | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/test/java/org/elasticsearch/module/statsd/test/StatsdPluginIntegrationTest.java b/src/test/java/org/elasticsearch/module/statsd/test/StatsdPluginIntegrationTest.java index f7a2177..1f7a85d 100644 --- a/src/test/java/org/elasticsearch/module/statsd/test/StatsdPluginIntegrationTest.java +++ b/src/test/java/org/elasticsearch/module/statsd/test/StatsdPluginIntegrationTest.java @@ -53,6 +53,9 @@ public void testThatIndexingResultsInMonitoring() throws Exception IndexResponse indexResponse = indexElement(node_1, index, type, "value"); assertThat(indexResponse.getId(), is(notNullValue())); + //Index some more docs + this.indexSomeDocs(100); + Thread.sleep(4000); ensureValidKeyNames(); @@ -101,4 +104,12 @@ private IndexResponse indexElement(Node node, String index, String type, String { return node.client().prepareIndex(index, type).setSource("field", fieldValue).execute().actionGet(); } + + private void indexSomeDocs(int docs) + { + while( docs > 0 ) { + indexElement(node_1, index, type, "value " + docs); + docs--; + } + } } From 747397100e7f3d7c92a04796b00d62b714b14a43 Mon Sep 17 00:00:00 2001 From: Xiao Yu Date: Thu, 7 Aug 2014 11:04:33 -0400 Subject: [PATCH 18/48] Fix test assertions --- .../statsd/test/StatsdPluginIntegrationTest.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/test/java/org/elasticsearch/module/statsd/test/StatsdPluginIntegrationTest.java b/src/test/java/org/elasticsearch/module/statsd/test/StatsdPluginIntegrationTest.java index 1f7a85d..b0b4fda 100644 --- a/src/test/java/org/elasticsearch/module/statsd/test/StatsdPluginIntegrationTest.java +++ b/src/test/java/org/elasticsearch/module/statsd/test/StatsdPluginIntegrationTest.java @@ -54,14 +54,14 @@ public void testThatIndexingResultsInMonitoring() throws Exception assertThat(indexResponse.getId(), is(notNullValue())); //Index some more docs - this.indexSomeDocs(100); + this.indexSomeDocs(101); Thread.sleep(4000); ensureValidKeyNames(); - assertStatsdMetricIsContained("elasticsearch." + clusterName + ".indexes." + index + ".id.0.indexing._all.indexCount:1|c"); - assertStatsdMetricIsContained("elasticsearch." + clusterName + ".indexes." + index + ".id.0.indexing." + type + ".indexCount:1|c"); - assertStatsdMetricIsContained("elasticsearch." + clusterName + ".node.jvm.threads.peak_count:"); + assertStatsdMetricIsContained("elasticsearch." + clusterName + ".index." + index + ".shard.0.indexing.index_total:51|c"); + assertStatsdMetricIsContained("elasticsearch." + clusterName + ".index." + index + ".shard.1.indexing.index_total:51|c"); + assertStatsdMetricIsContained(".jvm.threads.peak_count:"); } @Test @@ -80,7 +80,7 @@ public void masterFailOverShouldWork() throws Exception // wait for master fail over and writing to graph reporter Thread.sleep(4000); - assertStatsdMetricIsContained("elasticsearch." + clusterName + ".indexes." + index + ".id.0.indexing._all.indexCount:1|c"); + assertStatsdMetricIsContained("elasticsearch." + clusterName + ".index." + index + ".shard.0.indexing.index_total:1|c"); } // the stupid hamcrest matchers have compile erros depending whether they run on java6 or java7, so I rolled my own version From 61feeaf034d71d8f061fcf5f8d954bf903f8b862 Mon Sep 17 00:00:00 2001 From: Xiao Yu Date: Thu, 7 Aug 2014 13:49:21 -0400 Subject: [PATCH 19/48] Add some error checking and fix names --- .../statsd/StatsdReporterIndexStats.java | 90 +++++++++++-------- 1 file changed, 53 insertions(+), 37 deletions(-) diff --git a/src/main/java/org/elasticsearch/service/statsd/StatsdReporterIndexStats.java b/src/main/java/org/elasticsearch/service/statsd/StatsdReporterIndexStats.java index 6f92413..01bd169 100644 --- a/src/main/java/org/elasticsearch/service/statsd/StatsdReporterIndexStats.java +++ b/src/main/java/org/elasticsearch/service/statsd/StatsdReporterIndexStats.java @@ -23,16 +23,19 @@ public abstract class StatsdReporterIndexStats extends StatsdReporter { protected void sendDocsStats(String name, DocsStats docsStats) { + if (null == docsStats) return; this.sendCount(name, "count", docsStats.getCount()); this.sendCount(name, "deleted", docsStats.getDeleted()); } protected void sendStoreStats(String name, StoreStats storeStats) { + if (null == storeStats) return; this.sendCount(name, "size_in_bytes", storeStats.sizeInBytes()); this.sendCount(name, "throttle_time_in_millis", storeStats.getThrottleTime().millis()); } protected void sendIndexingStats(String type, IndexingStats indexingStats) { + if (null == indexingStats) return; IndexingStats.Stats totalStats = indexingStats.getTotal(); this.sendIndexingStatsStats(type, totalStats); @@ -40,6 +43,7 @@ protected void sendIndexingStats(String type, IndexingStats indexingStats) { } protected void sendGetStats(String type, GetStats getStats) { + if (null == getStats) return; this.sendCount(type, "total", getStats.getCount()); this.sendCount(type, "time_in_millis", getStats.getTimeInMillis()); this.sendCount(type, "exists_total", getStats.getExistsCount()); @@ -50,6 +54,7 @@ protected void sendGetStats(String type, GetStats getStats) { } protected void sendSearchStats(String type, SearchStats searchStats) { + if (null == searchStats) return; SearchStats.Stats totalSearchStats = searchStats.getTotal(); this.sendSearchStatsStats(type, totalSearchStats); @@ -57,6 +62,7 @@ protected void sendSearchStats(String type, SearchStats searchStats) { } protected void sendMergeStats(String type, MergeStats mergeStats) { + if (null == mergeStats) return; this.sendGauge(type, "current", mergeStats.getCurrent()); this.sendGauge(type, "current_docs", mergeStats.getCurrentNumDocs()); this.sendGauge(type, "current_size_in_bytes", mergeStats.getCurrentSizeInBytes()); @@ -67,63 +73,73 @@ protected void sendMergeStats(String type, MergeStats mergeStats) { } protected void sendRefreshStats(String type, RefreshStats refreshStats) { + if (null == refreshStats) return; this.sendCount(type, "total", refreshStats.getTotal()); this.sendCount(type, "total_time_in_millis", refreshStats.getTotalTimeInMillis()); } - protected void sendFlushStats(String type, FlushStats flush) { - this.sendCount(type, "total", flush.getTotal()); - this.sendCount(type, "total_time_in_millis", flush.getTotalTimeInMillis()); + protected void sendFlushStats(String type, FlushStats flushStats) { + if (null == flushStats) return; + this.sendCount(type, "total", flushStats.getTotal()); + this.sendCount(type, "total_time_in_millis", flushStats.getTotalTimeInMillis()); } - protected void sendFilterCacheStats(String name, FilterCacheStats filterCache) { - this.sendGauge(name, "memory_size_in_bytes", filterCache.getMemorySizeInBytes()); - this.sendGauge(name, "evictions", filterCache.getEvictions()); + protected void sendFilterCacheStats(String name, FilterCacheStats filterCacheStats) { + if (null == filterCacheStats) return; + this.sendGauge(name, "memory_size_in_bytes", filterCacheStats.getMemorySizeInBytes()); + this.sendGauge(name, "evictions", filterCacheStats.getEvictions()); } - protected void sendIdCacheStats(String name, IdCacheStats idCache) { - this.sendGauge(name, "memory_size_in_bytes", idCache.getMemorySizeInBytes()); + protected void sendIdCacheStats(String name, IdCacheStats idCacheStats) { + if (null == idCacheStats) return; + this.sendGauge(name, "memory_size_in_bytes", idCacheStats.getMemorySizeInBytes()); } - protected void sendFielddataCacheStats(String name, FieldDataStats fielddataCache) { - this.sendGauge(name, "memory_size_in_bytes", fielddataCache.getMemorySizeInBytes()); - this.sendGauge(name, "evictions", fielddataCache.getEvictions()); + protected void sendFielddataCacheStats(String name, FieldDataStats fielddataStats) { + if (null == fielddataStats) return; + this.sendGauge(name, "memory_size_in_bytes", fielddataStats.getMemorySizeInBytes()); + this.sendGauge(name, "evictions", fielddataStats.getEvictions()); } - protected void sendPercolateStats(String name, PercolateStats stats) { - this.sendCount(name, "total", stats.getCount()); - this.sendCount(name, "time_in_millis", stats.getTimeInMillis()); - this.sendGauge(name, "current", stats.getCurrent()); - this.sendCount(name, "queries", stats.getNumQueries()); + protected void sendPercolateStats(String name, PercolateStats percolateStats) { + if (null == percolateStats) return; + this.sendCount(name, "total", percolateStats.getCount()); + this.sendCount(name, "time_in_millis", percolateStats.getTimeInMillis()); + this.sendGauge(name, "current", percolateStats.getCurrent()); + this.sendCount(name, "queries", percolateStats.getNumQueries()); - if (stats.getMemorySizeInBytes() != -1) - this.sendGauge(name, "memory_size_in_bytes", stats.getMemorySizeInBytes()); + if (percolateStats.getMemorySizeInBytes() != -1) + this.sendGauge(name, "memory_size_in_bytes", percolateStats.getMemorySizeInBytes()); } - protected void sendCompletionStats(String name, CompletionStats stats) { - this.sendGauge(name, "size_in_bytes", stats.getSizeInBytes()); + protected void sendCompletionStats(String name, CompletionStats completionStats) { + if (null == completionStats) return; + this.sendGauge(name, "size_in_bytes", completionStats.getSizeInBytes()); } - protected void sendSegmentsStats(String name, SegmentsStats stats) { - this.sendGauge(name, "count", stats.getCount()); - this.sendGauge(name, "memory_in_bytes", stats.getMemoryInBytes()); + protected void sendSegmentsStats(String name, SegmentsStats segmentsStats) { + if (null == segmentsStats) return; + this.sendGauge(name, "count", segmentsStats.getCount()); + this.sendGauge(name, "memory_in_bytes", segmentsStats.getMemoryInBytes()); } - protected void sendIndexingStatsStats(String type, IndexingStats.Stats stats) { - this.sendCount(type, "index_total", stats.getIndexCount()); - this.sendCount(type, "index_time_in_millis", stats.getIndexTimeInMillis()); - this.sendGauge(type, "index_current", stats.getIndexCount()); - this.sendCount(type, "delete_total", stats.getDeleteCount()); - this.sendCount(type, "delete_time_in_millis", stats.getDeleteTimeInMillis()); - this.sendGauge(type, "delete_current", stats.getDeleteCurrent()); + protected void sendIndexingStatsStats(String type, IndexingStats.Stats indexingStatsStats) { + if (null == indexingStatsStats) return; + this.sendCount(type, "index_total", indexingStatsStats.getIndexCount()); + this.sendCount(type, "index_time_in_millis", indexingStatsStats.getIndexTimeInMillis()); + this.sendGauge(type, "index_current", indexingStatsStats.getIndexCount()); + this.sendCount(type, "delete_total", indexingStatsStats.getDeleteCount()); + this.sendCount(type, "delete_time_in_millis", indexingStatsStats.getDeleteTimeInMillis()); + this.sendGauge(type, "delete_current", indexingStatsStats.getDeleteCurrent()); } - protected void sendSearchStatsStats(String type, SearchStats.Stats stats) { - this.sendCount(type, "query_total", stats.getQueryCount()); - this.sendCount(type, "query_time_in_millis", stats.getQueryTimeInMillis()); - this.sendGauge(type, "query_current", stats.getQueryCurrent()); - this.sendCount(type, "fetch_total", stats.getFetchCount()); - this.sendCount(type, "fetch_time_in_millis", stats.getFetchTimeInMillis()); - this.sendGauge(type, "fetch_current", stats.getFetchCurrent()); + protected void sendSearchStatsStats(String type, SearchStats.Stats searchStatsStats) { + if (null == searchStatsStats) return; + this.sendCount(type, "query_total", searchStatsStats.getQueryCount()); + this.sendCount(type, "query_time_in_millis", searchStatsStats.getQueryTimeInMillis()); + this.sendGauge(type, "query_current", searchStatsStats.getQueryCurrent()); + this.sendCount(type, "fetch_total", searchStatsStats.getFetchCount()); + this.sendCount(type, "fetch_time_in_millis", searchStatsStats.getFetchTimeInMillis()); + this.sendGauge(type, "fetch_current", searchStatsStats.getFetchCurrent()); } } From 22097a41d66574dc34f91adf7e4a6bbd8fde907f Mon Sep 17 00:00:00 2001 From: Xiao Yu Date: Thu, 7 Aug 2014 14:23:43 -0400 Subject: [PATCH 20/48] Use for key prefix var is confusing --- .../statsd/StatsdReporterIndexStats.java | 80 +++++++++---------- 1 file changed, 40 insertions(+), 40 deletions(-) diff --git a/src/main/java/org/elasticsearch/service/statsd/StatsdReporterIndexStats.java b/src/main/java/org/elasticsearch/service/statsd/StatsdReporterIndexStats.java index 01bd169..bdf049c 100644 --- a/src/main/java/org/elasticsearch/service/statsd/StatsdReporterIndexStats.java +++ b/src/main/java/org/elasticsearch/service/statsd/StatsdReporterIndexStats.java @@ -34,54 +34,54 @@ protected void sendStoreStats(String name, StoreStats storeStats) { this.sendCount(name, "throttle_time_in_millis", storeStats.getThrottleTime().millis()); } - protected void sendIndexingStats(String type, IndexingStats indexingStats) { + protected void sendIndexingStats(String name, IndexingStats indexingStats) { if (null == indexingStats) return; IndexingStats.Stats totalStats = indexingStats.getTotal(); - this.sendIndexingStatsStats(type, totalStats); + this.sendIndexingStatsStats(name, totalStats); // TODO: Maybe print out stats to shards level? } - protected void sendGetStats(String type, GetStats getStats) { + protected void sendGetStats(String name, GetStats getStats) { if (null == getStats) return; - this.sendCount(type, "total", getStats.getCount()); - this.sendCount(type, "time_in_millis", getStats.getTimeInMillis()); - this.sendCount(type, "exists_total", getStats.getExistsCount()); - this.sendCount(type, "exists_time_in_millis", getStats.getExistsTimeInMillis()); - this.sendCount(type, "missing_total", getStats.getMissingCount()); - this.sendCount(type, "missing_time_in_millis", getStats.getMissingTimeInMillis()); - this.sendGauge(type, "current", getStats.current()); + this.sendCount(name, "total", getStats.getCount()); + this.sendCount(name, "time_in_millis", getStats.getTimeInMillis()); + this.sendCount(name, "exists_total", getStats.getExistsCount()); + this.sendCount(name, "exists_time_in_millis", getStats.getExistsTimeInMillis()); + this.sendCount(name, "missing_total", getStats.getMissingCount()); + this.sendCount(name, "missing_time_in_millis", getStats.getMissingTimeInMillis()); + this.sendGauge(name, "current", getStats.current()); } - protected void sendSearchStats(String type, SearchStats searchStats) { + protected void sendSearchStats(String name, SearchStats searchStats) { if (null == searchStats) return; SearchStats.Stats totalSearchStats = searchStats.getTotal(); - this.sendSearchStatsStats(type, totalSearchStats); + this.sendSearchStatsStats(name, totalSearchStats); // TODO: Maybe print out stats to shards level? } - protected void sendMergeStats(String type, MergeStats mergeStats) { + protected void sendMergeStats(String name, MergeStats mergeStats) { if (null == mergeStats) return; - this.sendGauge(type, "current", mergeStats.getCurrent()); - this.sendGauge(type, "current_docs", mergeStats.getCurrentNumDocs()); - this.sendGauge(type, "current_size_in_bytes", mergeStats.getCurrentSizeInBytes()); - this.sendCount(type, "total", mergeStats.getTotal()); - this.sendCount(type, "total_time_in_millis", mergeStats.getTotalTimeInMillis()); - this.sendCount(type, "total_docs", mergeStats.getTotalNumDocs()); - this.sendCount(type, "total_size_in_bytes", mergeStats.getTotalSizeInBytes()); + this.sendGauge(name, "current", mergeStats.getCurrent()); + this.sendGauge(name, "current_docs", mergeStats.getCurrentNumDocs()); + this.sendGauge(name, "current_size_in_bytes", mergeStats.getCurrentSizeInBytes()); + this.sendCount(name, "total", mergeStats.getTotal()); + this.sendCount(name, "total_time_in_millis", mergeStats.getTotalTimeInMillis()); + this.sendCount(name, "total_docs", mergeStats.getTotalNumDocs()); + this.sendCount(name, "total_size_in_bytes", mergeStats.getTotalSizeInBytes()); } - protected void sendRefreshStats(String type, RefreshStats refreshStats) { + protected void sendRefreshStats(String name, RefreshStats refreshStats) { if (null == refreshStats) return; - this.sendCount(type, "total", refreshStats.getTotal()); - this.sendCount(type, "total_time_in_millis", refreshStats.getTotalTimeInMillis()); + this.sendCount(name, "total", refreshStats.getTotal()); + this.sendCount(name, "total_time_in_millis", refreshStats.getTotalTimeInMillis()); } - protected void sendFlushStats(String type, FlushStats flushStats) { + protected void sendFlushStats(String name, FlushStats flushStats) { if (null == flushStats) return; - this.sendCount(type, "total", flushStats.getTotal()); - this.sendCount(type, "total_time_in_millis", flushStats.getTotalTimeInMillis()); + this.sendCount(name, "total", flushStats.getTotal()); + this.sendCount(name, "total_time_in_millis", flushStats.getTotalTimeInMillis()); } protected void sendFilterCacheStats(String name, FilterCacheStats filterCacheStats) { @@ -123,23 +123,23 @@ protected void sendSegmentsStats(String name, SegmentsStats segmentsStats) { this.sendGauge(name, "memory_in_bytes", segmentsStats.getMemoryInBytes()); } - protected void sendIndexingStatsStats(String type, IndexingStats.Stats indexingStatsStats) { + protected void sendIndexingStatsStats(String name, IndexingStats.Stats indexingStatsStats) { if (null == indexingStatsStats) return; - this.sendCount(type, "index_total", indexingStatsStats.getIndexCount()); - this.sendCount(type, "index_time_in_millis", indexingStatsStats.getIndexTimeInMillis()); - this.sendGauge(type, "index_current", indexingStatsStats.getIndexCount()); - this.sendCount(type, "delete_total", indexingStatsStats.getDeleteCount()); - this.sendCount(type, "delete_time_in_millis", indexingStatsStats.getDeleteTimeInMillis()); - this.sendGauge(type, "delete_current", indexingStatsStats.getDeleteCurrent()); + this.sendCount(name, "index_total", indexingStatsStats.getIndexCount()); + this.sendCount(name, "index_time_in_millis", indexingStatsStats.getIndexTimeInMillis()); + this.sendGauge(name, "index_current", indexingStatsStats.getIndexCount()); + this.sendCount(name, "delete_total", indexingStatsStats.getDeleteCount()); + this.sendCount(name, "delete_time_in_millis", indexingStatsStats.getDeleteTimeInMillis()); + this.sendGauge(name, "delete_current", indexingStatsStats.getDeleteCurrent()); } - protected void sendSearchStatsStats(String type, SearchStats.Stats searchStatsStats) { + protected void sendSearchStatsStats(String name, SearchStats.Stats searchStatsStats) { if (null == searchStatsStats) return; - this.sendCount(type, "query_total", searchStatsStats.getQueryCount()); - this.sendCount(type, "query_time_in_millis", searchStatsStats.getQueryTimeInMillis()); - this.sendGauge(type, "query_current", searchStatsStats.getQueryCurrent()); - this.sendCount(type, "fetch_total", searchStatsStats.getFetchCount()); - this.sendCount(type, "fetch_time_in_millis", searchStatsStats.getFetchTimeInMillis()); - this.sendGauge(type, "fetch_current", searchStatsStats.getFetchCurrent()); + this.sendCount(name, "query_total", searchStatsStats.getQueryCount()); + this.sendCount(name, "query_time_in_millis", searchStatsStats.getQueryTimeInMillis()); + this.sendGauge(name, "query_current", searchStatsStats.getQueryCurrent()); + this.sendCount(name, "fetch_total", searchStatsStats.getFetchCount()); + this.sendCount(name, "fetch_time_in_millis", searchStatsStats.getFetchTimeInMillis()); + this.sendGauge(name, "fetch_current", searchStatsStats.getFetchCurrent()); } } From 185650d49cc28bce8ea59498157408c60f0a0119 Mon Sep 17 00:00:00 2001 From: Xiao Yu Date: Thu, 7 Aug 2014 14:25:36 -0400 Subject: [PATCH 21/48] Simplify error logging --- .../org/elasticsearch/service/statsd/StatsdReporter.java | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/src/main/java/org/elasticsearch/service/statsd/StatsdReporter.java b/src/main/java/org/elasticsearch/service/statsd/StatsdReporter.java index 7adf76c..6649454 100644 --- a/src/main/java/org/elasticsearch/service/statsd/StatsdReporter.java +++ b/src/main/java/org/elasticsearch/service/statsd/StatsdReporter.java @@ -53,12 +53,7 @@ private String join(String... parts) { } protected void logException(Exception e) { - if (this.logger.isDebugEnabled()) { - this.logger.debug("Error writing to StatsD", e); - } - else { - this.logger.warn("Error writing to StatsD: {}", e.getMessage()); - } + this.logger.warn("Error writing to StatsD", e); } protected ESLogger getLogger() { From 7b9b5829f4bb96c9ba13d8acc4a89270c8f11a92 Mon Sep 17 00:00:00 2001 From: Xiao Yu Date: Thu, 7 Aug 2014 14:27:23 -0400 Subject: [PATCH 22/48] Simplify stat key for cluster index totals --- .../service/statsd/StatsdReporterNodeIndicesStats.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/elasticsearch/service/statsd/StatsdReporterNodeIndicesStats.java b/src/main/java/org/elasticsearch/service/statsd/StatsdReporterNodeIndicesStats.java index 39cf8d5..5975fcd 100644 --- a/src/main/java/org/elasticsearch/service/statsd/StatsdReporterNodeIndicesStats.java +++ b/src/main/java/org/elasticsearch/service/statsd/StatsdReporterNodeIndicesStats.java @@ -16,7 +16,7 @@ public StatsdReporterNodeIndicesStats(NodeIndicesStats nodeIndicesStats) { public void run() { try { - String type = this.buildMetricName("node.indices"); + String type = this.buildMetricName("indices"); this.sendDocsStats(type + ".docs", this.nodeIndicesStats.getDocs()); this.sendStoreStats(type + ".store", this.nodeIndicesStats.getStore()); this.sendIndexingStats(type + ".indexing", this.nodeIndicesStats.getIndexing()); @@ -31,8 +31,7 @@ public void run() { this.sendPercolateStats(type + ".percolate", this.nodeIndicesStats.getPercolate()); this.sendCompletionStats(type + ".completion", this.nodeIndicesStats.getCompletion()); this.sendSegmentsStats(type + ".segments", this.nodeIndicesStats.getSegments()); - } - catch (Exception e) { + } catch (Exception e) { this.logException(e); } } From 21fba7ffedb52aee83a45a9c4f8e227c37e52cd8 Mon Sep 17 00:00:00 2001 From: Xiao Yu Date: Thu, 7 Aug 2014 14:30:29 -0400 Subject: [PATCH 23/48] Sum and report index stats on a per index and per shard basis; make it configureable --- .../service/statsd/StatsdReporterIndices.java | 74 +++++++++++++++---- .../service/statsd/StatsdService.java | 44 +++++------ 2 files changed, 81 insertions(+), 37 deletions(-) diff --git a/src/main/java/org/elasticsearch/service/statsd/StatsdReporterIndices.java b/src/main/java/org/elasticsearch/service/statsd/StatsdReporterIndices.java index f453c2d..ac187bd 100644 --- a/src/main/java/org/elasticsearch/service/statsd/StatsdReporterIndices.java +++ b/src/main/java/org/elasticsearch/service/statsd/StatsdReporterIndices.java @@ -5,35 +5,79 @@ import java.util.Map; import org.elasticsearch.index.shard.service.IndexShard; +import org.elasticsearch.action.admin.indices.stats.CommonStats; +import org.elasticsearch.action.admin.indices.stats.CommonStatsFlags; public class StatsdReporterIndices extends StatsdReporterIndexStats { private final List indexShards; + private final Boolean reportShards; + private String indexName; + private CommonStats indexStats; - public StatsdReporterIndices(List indexShards) { + public StatsdReporterIndices(List indexShards, Boolean reportShards) { this.indexShards = indexShards; + this.reportShards = reportShards; } public void run() { try { for (IndexShard indexShard : this.indexShards) { - String type = this.buildMetricName("index.") + indexShard.shardId().index().name() + ".shard." + indexShard.shardId().id(); - this.sendDocsStats(type + ".docs", indexShard.docStats()); - this.sendStoreStats(type + ".store", indexShard.storeStats()); - this.sendIndexingStats(type + ".indexing", indexShard.indexingStats("_all")); - this.sendGetStats(type + ".get", indexShard.getStats()); - this.sendSearchStats(type + ".search", indexShard.searchStats("_all")); - this.sendMergeStats(type + ".merges", indexShard.mergeStats()); - this.sendRefreshStats(type + ".refresh", indexShard.refreshStats()); - this.sendFlushStats(type + ".flush", indexShard.flushStats()); - this.sendFilterCacheStats(type + ".filter_cache", indexShard.filterCacheStats()); - this.sendIdCacheStats(type + ".id_cache", indexShard.idCacheStats()); - this.sendFielddataCacheStats(type + ".fielddata", indexShard.fieldDataStats("_all")); - this.sendCompletionStats(type + ".completion", indexShard.completionStats("_all")); - this.sendSegmentsStats(type + ".segments", indexShard.segmentStats()); + this.maybeResetSums(indexShard.shardId().index().name()); + + // Create common stats for shard + CommonStats shardStats = new CommonStats(indexShard, CommonStatsFlags.ALL); + + if (this.reportShards) { + this.sendCommonStats( + this.buildMetricName("index." + this.indexName + ".shard." + indexShard.shardId().id()), + shardStats + ); + } + + // Add to current index totals + this.indexStats.add(shardStats); } + // Send last index group + this.maybeResetSums(""); } catch (Exception e) { this.logException(e); } } + + private void maybeResetSums(String newIndexName) { + if (this.indexName == newIndexName) { + return; // Same index do nothing + } + + if (this.indexName != null) { + this.sendCommonStats( + this.buildMetricName("index." + this.indexName + ".total"), + this.indexStats + ); + } + + this.indexName = newIndexName; + this.indexStats = new CommonStats(CommonStatsFlags.ALL); + } + + private void sendCommonStats(String prefix, CommonStats stats) { + this.sendDocsStats(prefix + ".docs", stats.getDocs()); + this.sendStoreStats(prefix + ".store", stats.getStore()); + this.sendIndexingStats(prefix + ".indexing", stats.getIndexing()); + this.sendGetStats(prefix + ".get", stats.getGet()); + this.sendSearchStats(prefix + ".search", stats.getSearch()); + this.sendMergeStats(prefix + ".merges", stats.getMerge()); + this.sendRefreshStats(prefix + ".refresh", stats.getRefresh()); + this.sendFlushStats(prefix + ".flush", stats.getFlush()); + //TODO: getWarmer + this.sendFilterCacheStats(prefix + ".filter_cache", stats.getFilterCache()); + this.sendIdCacheStats(prefix + ".id_cache", stats.getIdCache()); + this.sendFielddataCacheStats(prefix + ".fielddata", stats.getFieldData()); + //TODO: getPercolate + this.sendCompletionStats(prefix + ".completion", stats.getCompletion()); + this.sendSegmentsStats(prefix + ".segments", stats.getSegments()); + //TODO: getTranslog + //TODO: getSuggest + } } diff --git a/src/main/java/org/elasticsearch/service/statsd/StatsdService.java b/src/main/java/org/elasticsearch/service/statsd/StatsdService.java index 2462052..d2857fa 100644 --- a/src/main/java/org/elasticsearch/service/statsd/StatsdService.java +++ b/src/main/java/org/elasticsearch/service/statsd/StatsdService.java @@ -33,6 +33,7 @@ public class StatsdService extends AbstractLifecycleComponent { private final TimeValue statsdRefreshInternal; private final String statsdPrefix; private final String statsdNodeName; + private final Boolean statsdReportIndices; private final Boolean statsdReportShards; private final Boolean statsdReportFsDetails; private final StatsDClient statsdClient; @@ -61,6 +62,9 @@ public StatsdService(Settings settings, ClusterService clusterService, IndicesSe this.statsdNodeName = settings.get( "metrics.statsd.node_name" ); + this.statsdReportIndices = settings.getAsBoolean( + "metrics.statsd.report.indices", true + ); this.statsdReportShards = settings.getAsBoolean( "metrics.statsd.report.shards", false ); @@ -115,7 +119,7 @@ public void run() { .equals(Lifecycle.State.STARTED); if (node != null && isClusterStarted) { - // Report node stats + // Report node stats -- runs for all nodes StatsdReporter nodeStatsReporter = new StatsdReporterNodeStats( StatsdService.this.nodeService.stats( new CommonStatsFlags().clear(), // indices @@ -136,9 +140,9 @@ public void run() { .setStatsDClient(StatsdService.this.statsdClient) .run(); - // Master Node sends cluster wide stats + // Master node is the only one allowed to send cluster wide sums / stats if (node.isMasterNode()) { - // Report node indice stats + // Report cluster wide index totals StatsdReporter nodeIndicesStatsReporter = new StatsdReporterNodeIndicesStats( StatsdService.this.indicesService.stats( false // includePrevious @@ -148,18 +152,16 @@ public void run() { .setStatsDClient(StatsdService.this.statsdClient) .run(); - // Report indices stats - StatsdReporter indicesReporter = new StatsdReporterIndices( - this.getIndexShards(StatsdService.this.indicesService) - ); - indicesReporter - .setStatsDClient(StatsdService.this.statsdClient) - .run(); - } else { - StatsdService.this.logger.debug( - "[{}]/[{}] is not master node, not triggering update", - node.getId(), node.getName() - ); + // Maybe breakdown numbers by index or shard + if ( StatsdService.this.statsdReportIndices || StatsdService.this.statsdReportShards ) { + StatsdReporter indicesReporter = new StatsdReporterIndices( + this.getIndexShards(StatsdService.this.indicesService), + StatsdService.this.statsdReportShards + ); + indicesReporter + .setStatsDClient(StatsdService.this.statsdClient) + .run(); + } } } @@ -174,13 +176,11 @@ public void run() { private List getIndexShards(IndicesService indicesService) { List indexShards = Lists.newArrayList(); - if ( StatsdService.this.statsdReportShards ) { - String[] indices = indicesService.indices().toArray(new String[] {}); - for (String indexName : indices) { - IndexService indexService = indicesService.indexServiceSafe(indexName); - for (int shardId : indexService.shardIds()) { - indexShards.add(indexService.shard(shardId)); - } + String[] indices = indicesService.indices().toArray(new String[] {}); + for (String indexName : indices) { + IndexService indexService = indicesService.indexServiceSafe(indexName); + for (int shardId : indexService.shardIds()) { + indexShards.add(indexService.shard(shardId)); } } From 2f156aef5af2951627c5e6a5fda8a5cef4c7758d Mon Sep 17 00:00:00 2001 From: Xiao Yu Date: Thu, 7 Aug 2014 14:31:21 -0400 Subject: [PATCH 24/48] Document StatsD key format --- README.md | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 37a04c7..bf2289e 100644 --- a/README.md +++ b/README.md @@ -33,6 +33,16 @@ Check your elasticsearch log file for a line like this after adding the configur ``` +## Stats Key Formats + +This plugin reports both node level and cluster level stats, the StatsD keys will be in the formats: + +* `{PREFIX}.node.{NODE_NAME}.{STAT_KEY}` -- Node level stats (CPU / JVM / etc.) +* `{PREFIX}.indices.{STAT_KEY}` -- The stats on indices across the entire cluster +* `{PREFIX}.index.{INDEX_NAME}.total.{STAT_KEY}` -- The stats per index summed across all shards, enabled with `metrics.statsd.report.indices` (defaults to `true`) +* `{PREFIX}.index.{INDEX_NAME}.{SHARD_ID}.{STAT_KEY}` -- The stats per shard, enabled with `metrics.statsd.report.shards` (defaults to `false`) + + ## Bugs/TODO * No really nice cluster support yet (needed it for a single instance system) @@ -48,4 +58,3 @@ Heavily inspired by the excellent [metrics library](http://metrics.codehale.com) ## License See LICENSE - From b144e880c6b73fc1161cbcd793342b61e45ede10 Mon Sep 17 00:00:00 2001 From: Xiao Yu Date: Thu, 7 Aug 2014 14:36:03 -0400 Subject: [PATCH 25/48] Cleanup stat key name for shard level stats --- .../org/elasticsearch/service/statsd/StatsdReporterIndices.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/elasticsearch/service/statsd/StatsdReporterIndices.java b/src/main/java/org/elasticsearch/service/statsd/StatsdReporterIndices.java index ac187bd..cb9d5b9 100644 --- a/src/main/java/org/elasticsearch/service/statsd/StatsdReporterIndices.java +++ b/src/main/java/org/elasticsearch/service/statsd/StatsdReporterIndices.java @@ -30,7 +30,7 @@ public void run() { if (this.reportShards) { this.sendCommonStats( - this.buildMetricName("index." + this.indexName + ".shard." + indexShard.shardId().id()), + this.buildMetricName("index." + this.indexName + "." + indexShard.shardId().id()), shardStats ); } From e4c43c31a7da19636206cbfd2069aa4fa2499161 Mon Sep 17 00:00:00 2001 From: Xiao Yu Date: Thu, 7 Aug 2014 14:59:47 -0400 Subject: [PATCH 26/48] Everything is a Guage Because we are sending stats to StatsD on an interval everything is a gauge even "counts" which are ever increasing. There's a current issue open for StatsD (etsy/statsd#324) which requests "absolute" counters which would solve this problem however until that's solved we have to use gauges and just apply derivative functions after the fact. Le Sigh. --- .../statsd/StatsdReporterIndexStats.java | 58 +++++++++---------- .../statsd/StatsdReporterNodeStats.java | 48 +++++++-------- 2 files changed, 53 insertions(+), 53 deletions(-) diff --git a/src/main/java/org/elasticsearch/service/statsd/StatsdReporterIndexStats.java b/src/main/java/org/elasticsearch/service/statsd/StatsdReporterIndexStats.java index bdf049c..5796335 100644 --- a/src/main/java/org/elasticsearch/service/statsd/StatsdReporterIndexStats.java +++ b/src/main/java/org/elasticsearch/service/statsd/StatsdReporterIndexStats.java @@ -24,14 +24,14 @@ public abstract class StatsdReporterIndexStats extends StatsdReporter { protected void sendDocsStats(String name, DocsStats docsStats) { if (null == docsStats) return; - this.sendCount(name, "count", docsStats.getCount()); - this.sendCount(name, "deleted", docsStats.getDeleted()); + this.sendGauge(name, "count", docsStats.getCount()); + this.sendGauge(name, "deleted", docsStats.getDeleted()); } protected void sendStoreStats(String name, StoreStats storeStats) { if (null == storeStats) return; - this.sendCount(name, "size_in_bytes", storeStats.sizeInBytes()); - this.sendCount(name, "throttle_time_in_millis", storeStats.getThrottleTime().millis()); + this.sendGauge(name, "size_in_bytes", storeStats.sizeInBytes()); + this.sendGauge(name, "throttle_time_in_millis", storeStats.getThrottleTime().millis()); } protected void sendIndexingStats(String name, IndexingStats indexingStats) { @@ -44,12 +44,12 @@ protected void sendIndexingStats(String name, IndexingStats indexingStats) { protected void sendGetStats(String name, GetStats getStats) { if (null == getStats) return; - this.sendCount(name, "total", getStats.getCount()); - this.sendCount(name, "time_in_millis", getStats.getTimeInMillis()); - this.sendCount(name, "exists_total", getStats.getExistsCount()); - this.sendCount(name, "exists_time_in_millis", getStats.getExistsTimeInMillis()); - this.sendCount(name, "missing_total", getStats.getMissingCount()); - this.sendCount(name, "missing_time_in_millis", getStats.getMissingTimeInMillis()); + this.sendGauge(name, "total", getStats.getCount()); + this.sendGauge(name, "time_in_millis", getStats.getTimeInMillis()); + this.sendGauge(name, "exists_total", getStats.getExistsCount()); + this.sendGauge(name, "exists_time_in_millis", getStats.getExistsTimeInMillis()); + this.sendGauge(name, "missing_total", getStats.getMissingCount()); + this.sendGauge(name, "missing_time_in_millis", getStats.getMissingTimeInMillis()); this.sendGauge(name, "current", getStats.current()); } @@ -66,22 +66,22 @@ protected void sendMergeStats(String name, MergeStats mergeStats) { this.sendGauge(name, "current", mergeStats.getCurrent()); this.sendGauge(name, "current_docs", mergeStats.getCurrentNumDocs()); this.sendGauge(name, "current_size_in_bytes", mergeStats.getCurrentSizeInBytes()); - this.sendCount(name, "total", mergeStats.getTotal()); - this.sendCount(name, "total_time_in_millis", mergeStats.getTotalTimeInMillis()); - this.sendCount(name, "total_docs", mergeStats.getTotalNumDocs()); - this.sendCount(name, "total_size_in_bytes", mergeStats.getTotalSizeInBytes()); + this.sendGauge(name, "total", mergeStats.getTotal()); + this.sendGauge(name, "total_time_in_millis", mergeStats.getTotalTimeInMillis()); + this.sendGauge(name, "total_docs", mergeStats.getTotalNumDocs()); + this.sendGauge(name, "total_size_in_bytes", mergeStats.getTotalSizeInBytes()); } protected void sendRefreshStats(String name, RefreshStats refreshStats) { if (null == refreshStats) return; - this.sendCount(name, "total", refreshStats.getTotal()); - this.sendCount(name, "total_time_in_millis", refreshStats.getTotalTimeInMillis()); + this.sendGauge(name, "total", refreshStats.getTotal()); + this.sendGauge(name, "total_time_in_millis", refreshStats.getTotalTimeInMillis()); } protected void sendFlushStats(String name, FlushStats flushStats) { if (null == flushStats) return; - this.sendCount(name, "total", flushStats.getTotal()); - this.sendCount(name, "total_time_in_millis", flushStats.getTotalTimeInMillis()); + this.sendGauge(name, "total", flushStats.getTotal()); + this.sendGauge(name, "total_time_in_millis", flushStats.getTotalTimeInMillis()); } protected void sendFilterCacheStats(String name, FilterCacheStats filterCacheStats) { @@ -103,10 +103,10 @@ protected void sendFielddataCacheStats(String name, FieldDataStats fielddataStat protected void sendPercolateStats(String name, PercolateStats percolateStats) { if (null == percolateStats) return; - this.sendCount(name, "total", percolateStats.getCount()); - this.sendCount(name, "time_in_millis", percolateStats.getTimeInMillis()); + this.sendGauge(name, "total", percolateStats.getCount()); + this.sendGauge(name, "time_in_millis", percolateStats.getTimeInMillis()); this.sendGauge(name, "current", percolateStats.getCurrent()); - this.sendCount(name, "queries", percolateStats.getNumQueries()); + this.sendGauge(name, "queries", percolateStats.getNumQueries()); if (percolateStats.getMemorySizeInBytes() != -1) this.sendGauge(name, "memory_size_in_bytes", percolateStats.getMemorySizeInBytes()); @@ -125,21 +125,21 @@ protected void sendSegmentsStats(String name, SegmentsStats segmentsStats) { protected void sendIndexingStatsStats(String name, IndexingStats.Stats indexingStatsStats) { if (null == indexingStatsStats) return; - this.sendCount(name, "index_total", indexingStatsStats.getIndexCount()); - this.sendCount(name, "index_time_in_millis", indexingStatsStats.getIndexTimeInMillis()); + this.sendGauge(name, "index_total", indexingStatsStats.getIndexCount()); + this.sendGauge(name, "index_time_in_millis", indexingStatsStats.getIndexTimeInMillis()); this.sendGauge(name, "index_current", indexingStatsStats.getIndexCount()); - this.sendCount(name, "delete_total", indexingStatsStats.getDeleteCount()); - this.sendCount(name, "delete_time_in_millis", indexingStatsStats.getDeleteTimeInMillis()); + this.sendGauge(name, "delete_total", indexingStatsStats.getDeleteCount()); + this.sendGauge(name, "delete_time_in_millis", indexingStatsStats.getDeleteTimeInMillis()); this.sendGauge(name, "delete_current", indexingStatsStats.getDeleteCurrent()); } protected void sendSearchStatsStats(String name, SearchStats.Stats searchStatsStats) { if (null == searchStatsStats) return; - this.sendCount(name, "query_total", searchStatsStats.getQueryCount()); - this.sendCount(name, "query_time_in_millis", searchStatsStats.getQueryTimeInMillis()); + this.sendGauge(name, "query_total", searchStatsStats.getQueryCount()); + this.sendGauge(name, "query_time_in_millis", searchStatsStats.getQueryTimeInMillis()); this.sendGauge(name, "query_current", searchStatsStats.getQueryCurrent()); - this.sendCount(name, "fetch_total", searchStatsStats.getFetchCount()); - this.sendCount(name, "fetch_time_in_millis", searchStatsStats.getFetchTimeInMillis()); + this.sendGauge(name, "fetch_total", searchStatsStats.getFetchCount()); + this.sendGauge(name, "fetch_time_in_millis", searchStatsStats.getFetchTimeInMillis()); this.sendGauge(name, "fetch_current", searchStatsStats.getFetchCurrent()); } } diff --git a/src/main/java/org/elasticsearch/service/statsd/StatsdReporterNodeStats.java b/src/main/java/org/elasticsearch/service/statsd/StatsdReporterNodeStats.java index 47b5c7e..f21e3b8 100644 --- a/src/main/java/org/elasticsearch/service/statsd/StatsdReporterNodeStats.java +++ b/src/main/java/org/elasticsearch/service/statsd/StatsdReporterNodeStats.java @@ -54,17 +54,17 @@ private void sendNodeThreadPoolStats(ThreadPoolStats threadPoolStats) { this.sendGauge(id, "active", stats.getActive()); this.sendGauge(id, "rejected", stats.getRejected()); this.sendGauge(id, "largest", stats.getLargest()); - this.sendCount(id, "completed", stats.getCompleted()); + this.sendGauge(id, "completed", stats.getCompleted()); } } private void sendNodeTransportStats(TransportStats transportStats) { String type = this.getPrefix("transport"); this.sendGauge(type, "server_open", transportStats.serverOpen()); - this.sendCount(type, "rx_count", transportStats.rxCount()); - this.sendCount(type, "rx_size_in_bytes", transportStats.rxSize().bytes()); - this.sendCount(type, "tx_count", transportStats.txCount()); - this.sendCount(type, "tx_size_in_bytes", transportStats.txSize().bytes()); + this.sendGauge(type, "rx_count", transportStats.rxCount()); + this.sendGauge(type, "rx_size_in_bytes", transportStats.rxSize().bytes()); + this.sendGauge(type, "tx_count", transportStats.txCount()); + this.sendGauge(type, "tx_size_in_bytes", transportStats.txSize().bytes()); } private void sendNodeProcessStats(ProcessStats processStats) { @@ -74,9 +74,9 @@ private void sendNodeProcessStats(ProcessStats processStats) { if (processStats.cpu() != null) { this.sendGauge(type + ".cpu", "percent", processStats.cpu().percent()); - this.sendCount(type + ".cpu", "sys_in_millis", processStats.cpu().sys().millis()); - this.sendCount(type + ".cpu", "user_in_millis", processStats.cpu().user().millis()); - this.sendCount(type + ".cpu", "total_in_millis", processStats.cpu().total().millis()); + this.sendGauge(type + ".cpu", "sys_in_millis", processStats.cpu().sys().millis()); + this.sendGauge(type + ".cpu", "user_in_millis", processStats.cpu().user().millis()); + this.sendGauge(type + ".cpu", "total_in_millis", processStats.cpu().total().millis()); } if (processStats.mem() != null) { @@ -126,16 +126,16 @@ private void sendNodeNetworkStats(NetworkStats networkStats) { // might be null, if sigar isnt loaded if (tcp != null) { - this.sendCount(type, "active_opens", tcp.getActiveOpens()); - this.sendCount(type, "passive_opens", tcp.getPassiveOpens()); + this.sendGauge(type, "active_opens", tcp.getActiveOpens()); + this.sendGauge(type, "passive_opens", tcp.getPassiveOpens()); this.sendGauge(type, "curr_estab", tcp.getCurrEstab()); - this.sendCount(type, "in_segs", tcp.inSegs()); - this.sendCount(type, "out_segs", tcp.outSegs()); - this.sendCount(type, "retrans_segs", tcp.retransSegs()); - this.sendCount(type, "estab_resets", tcp.estabResets()); - this.sendCount(type, "attempt_fails", tcp.attemptFails()); - this.sendCount(type, "in_errs", tcp.inErrs()); - this.sendCount(type, "out_rsts", tcp.outRsts()); + this.sendGauge(type, "in_segs", tcp.inSegs()); + this.sendGauge(type, "out_segs", tcp.outSegs()); + this.sendGauge(type, "retrans_segs", tcp.retransSegs()); + this.sendGauge(type, "estab_resets", tcp.estabResets()); + this.sendGauge(type, "attempt_fails", tcp.attemptFails()); + this.sendGauge(type, "in_errs", tcp.inErrs()); + this.sendGauge(type, "out_rsts", tcp.outRsts()); } } @@ -165,8 +165,8 @@ private void sendNodeJvmStats(JvmStats jvmStats) { for (JvmStats.GarbageCollector collector : jvmStats.gc()) { String id = type + ".gc.collectors." + collector.name(); - this.sendCount(id, "collection_count", collector.collectionCount()); - this.sendCount(id, "collection_time_in_millis", collector.collectionTime().millis()); + this.sendGauge(id, "collection_count", collector.collectionCount()); + this.sendGauge(id, "collection_time_in_millis", collector.collectionTime().millis()); } // TODO: buffer pools @@ -175,7 +175,7 @@ private void sendNodeJvmStats(JvmStats jvmStats) { private void sendNodeHttpStats(HttpStats httpStats) { String type = this.getPrefix("http"); this.sendGauge(type, "current_open", httpStats.getServerOpen()); - this.sendCount(type, "total_opened", httpStats.getTotalOpen()); + this.sendGauge(type, "total_opened", httpStats.getTotalOpen()); } private void sendNodeFsStats(FsStats fs) { @@ -212,15 +212,15 @@ private void sendNodeFsStatsInfo(String type, FsStats.Info info) { // disk_io_op is sum of reads and writes (use graphite functions) if (info.getDiskReads() != -1) - this.sendCount(type + typeAppend, "disk_reads", info.getDiskReads()); + this.sendGauge(type + typeAppend, "disk_reads", info.getDiskReads()); if (info.getDiskWrites() != -1) - this.sendCount(type + typeAppend, "disk_writes", info.getDiskWrites()); + this.sendGauge(type + typeAppend, "disk_writes", info.getDiskWrites()); // disk_io_size_in_bytes is sum of reads and writes (use graphite functions) if (info.getDiskReadSizeInBytes() != -1) - this.sendCount(type + typeAppend, "disk_read_size_in_bytes", info.getDiskReadSizeInBytes()); + this.sendGauge(type + typeAppend, "disk_read_size_in_bytes", info.getDiskReadSizeInBytes()); if (info.getDiskWriteSizeInBytes() != -1) - this.sendCount(type + typeAppend, "disk_write_size_in_bytes", info.getDiskWriteSizeInBytes()); + this.sendGauge(type + typeAppend, "disk_write_size_in_bytes", info.getDiskWriteSizeInBytes()); /** TODO: Find out if these stats are useful. if (info.getDiskQueue() != -1) From ff6ab800a42c9fac3603ed3240db390cb7f1491c Mon Sep 17 00:00:00 2001 From: Xiao Yu Date: Thu, 7 Aug 2014 15:30:45 -0400 Subject: [PATCH 27/48] Update README --- README.md | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index bf2289e..62d27df 100644 --- a/README.md +++ b/README.md @@ -1,30 +1,41 @@ # Elasticsearch statsd plugin -This plugin creates a little push service, which regularly updates a statsd host with indices stats and nodes stats. In case you are running a cluster, these datas are always only pushed from the master node. +This plugin creates a little push service, which regularly updates a StatsD host with indices stats and nodes stats. +Index stats that apply across the entire cluster is only pushed from the elected master which node level stats are pushed from every node. -The data sent to the statsd server tries to be roughly equivalent to [Indices Stats API](http://www.elasticsearch.org/guide/reference/api/admin-indices-stats.html) and [Nodes Stats Api](http://www.elasticsearch.org/guide/reference/api/admin-cluster-nodes-stats.html) +The data sent to the StatsD server tries to be roughly equivalent to the [Indices Stats API](http://www.elasticsearch.org/guide/reference/api/admin-indices-stats.html) and [Nodes Stats Api](http://www.elasticsearch.org/guide/reference/api/admin-cluster-nodes-stats.html). ## Installation -As plugins (except site plugins) cannot be automatically installed from github currently you need to build the plugin yourself (takes half a minute including an integrations test). +To install a prepackaged plugin use the following command: ``` -git clone http://github.com/swoop-inc/elasticsearch-statsd-plugin.git +bin/plugin -install statsd -url https://github.com/Automattic/elasticsearch-statsd-plugin/releases/download/v0.3/elasticsearch-statsd-0.3.zip +``` + +You can also build your own by doing the following: + +``` +git clone http://github.com/Automattic/elasticsearch-statsd-plugin.git cd elasticsearch-statsd-plugin mvn package -/path/to/elasticsearch/bin/plugin -install statsd -url file:///absolute/path/to/current/dir/target/releases/elasticsearch-statsd-0.2-SNAPSHOT.zip +bin/plugin -install statsd -url file:///absolute/path/to/current/dir/target/releases/elasticsearch-statsd-0.3-SNAPSHOT.zip ``` ## Configuration -Configuration is possible via three parameters: +Configuration is possible via these parameters: * `metrics.statsd.host`: The statsd host to connect to (default: none) * `metrics.statsd.port`: The port to connect to (default: 8125) * `metrics.statsd.every`: The interval to push data (default: 1m) * `metrics.statsd.prefix`: The metric prefix that's sent with metric names (default: elasticsearch.your_cluster_name) +* `metrics.statsd.node_name`: Override the name for node used in the stat keys (default: the ES node name) +* `metrics.statsd.report.indices`: If index level sums should be reported (default: true) +* `metrics.statsd.report.shards`: If shard level stats should be reported (default: false) +* `metrics.statsd.report.fs_details`: If nodes should break down the FS by device instead of total disk (default: false) Check your elasticsearch log file for a line like this after adding the configuration parameters below to the configuration file @@ -45,7 +56,6 @@ This plugin reports both node level and cluster level stats, the StatsD keys wil ## Bugs/TODO -* No really nice cluster support yet (needed it for a single instance system) * Not extensively tested * In case of a master node failover, counts are starting from 0 again (in case you are wondering about spikes) From a14f50512dfc4e6b98a3c78e57ba5ed9e9d46f1f Mon Sep 17 00:00:00 2001 From: Xiao Yu Date: Fri, 8 Aug 2014 10:47:43 -0400 Subject: [PATCH 28/48] Remove debug use only code --- .../java/org/elasticsearch/service/statsd/StatsdReporter.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/main/java/org/elasticsearch/service/statsd/StatsdReporter.java b/src/main/java/org/elasticsearch/service/statsd/StatsdReporter.java index 6649454..07ac3ec 100644 --- a/src/main/java/org/elasticsearch/service/statsd/StatsdReporter.java +++ b/src/main/java/org/elasticsearch/service/statsd/StatsdReporter.java @@ -55,8 +55,4 @@ private String join(String... parts) { protected void logException(Exception e) { this.logger.warn("Error writing to StatsD", e); } - - protected ESLogger getLogger() { - return this.logger; - } } From e8158f34f2a75afd6f335501f4818ea1eaf60599 Mon Sep 17 00:00:00 2001 From: Xiao Yu Date: Fri, 8 Aug 2014 10:54:40 -0400 Subject: [PATCH 29/48] Check cluster state for elected master --- .../org/elasticsearch/service/statsd/StatsdService.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/elasticsearch/service/statsd/StatsdService.java b/src/main/java/org/elasticsearch/service/statsd/StatsdService.java index d2857fa..5409790 100644 --- a/src/main/java/org/elasticsearch/service/statsd/StatsdService.java +++ b/src/main/java/org/elasticsearch/service/statsd/StatsdService.java @@ -19,6 +19,7 @@ import org.elasticsearch.indices.IndicesService; import org.elasticsearch.indices.NodeIndicesStats; import org.elasticsearch.node.service.NodeService; +import org.elasticsearch.cluster.ClusterState; import com.timgroup.statsd.NonBlockingStatsDClient; import com.timgroup.statsd.StatsDClient; @@ -114,11 +115,12 @@ public class StatsdReporterThread implements Runnable { public void run() { while (!StatsdService.this.closed) { DiscoveryNode node = StatsdService.this.clusterService.localNode(); + ClusterState state = StatsdService.this.clusterService.state(); boolean isClusterStarted = StatsdService.this.clusterService .lifecycleState() .equals(Lifecycle.State.STARTED); - if (node != null && isClusterStarted) { + if (node != null && state != null && isClusterStarted) { // Report node stats -- runs for all nodes StatsdReporter nodeStatsReporter = new StatsdReporterNodeStats( StatsdService.this.nodeService.stats( @@ -141,7 +143,7 @@ public void run() { .run(); // Master node is the only one allowed to send cluster wide sums / stats - if (node.isMasterNode()) { + if (state.nodes().localNodeMaster()) { // Report cluster wide index totals StatsdReporter nodeIndicesStatsReporter = new StatsdReporterNodeIndicesStats( StatsdService.this.indicesService.stats( From a5e1df0d075423ec684240ebc77acee43bbeabe1 Mon Sep 17 00:00:00 2001 From: Xiao Yu Date: Fri, 8 Aug 2014 11:29:15 -0400 Subject: [PATCH 30/48] Make node level index stats optional Add a config to turn on reporting of node level index stats (e.g. docs per node) which defaults to false. --- .../StatsdReporterNodeIndicesStats.java | 34 ++++++++++--------- .../service/statsd/StatsdService.java | 20 ++++++++--- 2 files changed, 33 insertions(+), 21 deletions(-) diff --git a/src/main/java/org/elasticsearch/service/statsd/StatsdReporterNodeIndicesStats.java b/src/main/java/org/elasticsearch/service/statsd/StatsdReporterNodeIndicesStats.java index 5975fcd..3f16c72 100644 --- a/src/main/java/org/elasticsearch/service/statsd/StatsdReporterNodeIndicesStats.java +++ b/src/main/java/org/elasticsearch/service/statsd/StatsdReporterNodeIndicesStats.java @@ -9,28 +9,30 @@ public class StatsdReporterNodeIndicesStats extends StatsdReporterIndexStats { private final NodeIndicesStats nodeIndicesStats; + private final String nodeName; - public StatsdReporterNodeIndicesStats(NodeIndicesStats nodeIndicesStats) { + public StatsdReporterNodeIndicesStats(NodeIndicesStats nodeIndicesStats, String nodeName) { this.nodeIndicesStats = nodeIndicesStats; + this.nodeName = nodeName; } public void run() { try { - String type = this.buildMetricName("indices"); - this.sendDocsStats(type + ".docs", this.nodeIndicesStats.getDocs()); - this.sendStoreStats(type + ".store", this.nodeIndicesStats.getStore()); - this.sendIndexingStats(type + ".indexing", this.nodeIndicesStats.getIndexing()); - this.sendGetStats(type + ".get", this.nodeIndicesStats.getGet()); - this.sendSearchStats(type + ".search", this.nodeIndicesStats.getSearch()); - this.sendMergeStats(type + ".merges", this.nodeIndicesStats.getMerge()); - this.sendRefreshStats(type + ".refresh", this.nodeIndicesStats.getRefresh()); - this.sendFlushStats(type + ".flush", this.nodeIndicesStats.getFlush()); - this.sendFilterCacheStats(type + ".filter_cache", this.nodeIndicesStats.getFilterCache()); - this.sendIdCacheStats(type + ".id_cache", this.nodeIndicesStats.getIdCache()); - this.sendFielddataCacheStats(type + ".fielddata", this.nodeIndicesStats.getFieldData()); - this.sendPercolateStats(type + ".percolate", this.nodeIndicesStats.getPercolate()); - this.sendCompletionStats(type + ".completion", this.nodeIndicesStats.getCompletion()); - this.sendSegmentsStats(type + ".segments", this.nodeIndicesStats.getSegments()); + String prefix = this.buildMetricName( "node." + this.nodeName + ".indices" ); + this.sendDocsStats(prefix + ".docs", this.nodeIndicesStats.getDocs()); + this.sendStoreStats(prefix + ".store", this.nodeIndicesStats.getStore()); + this.sendIndexingStats(prefix + ".indexing", this.nodeIndicesStats.getIndexing()); + this.sendGetStats(prefix + ".get", this.nodeIndicesStats.getGet()); + this.sendSearchStats(prefix + ".search", this.nodeIndicesStats.getSearch()); + this.sendMergeStats(prefix + ".merges", this.nodeIndicesStats.getMerge()); + this.sendRefreshStats(prefix + ".refresh", this.nodeIndicesStats.getRefresh()); + this.sendFlushStats(prefix + ".flush", this.nodeIndicesStats.getFlush()); + this.sendFilterCacheStats(prefix + ".filter_cache", this.nodeIndicesStats.getFilterCache()); + this.sendIdCacheStats(prefix + ".id_cache", this.nodeIndicesStats.getIdCache()); + this.sendFielddataCacheStats(prefix + ".fielddata", this.nodeIndicesStats.getFieldData()); + this.sendPercolateStats(prefix + ".percolate", this.nodeIndicesStats.getPercolate()); + this.sendCompletionStats(prefix + ".completion", this.nodeIndicesStats.getCompletion()); + this.sendSegmentsStats(prefix + ".segments", this.nodeIndicesStats.getSegments()); } catch (Exception e) { this.logException(e); } diff --git a/src/main/java/org/elasticsearch/service/statsd/StatsdService.java b/src/main/java/org/elasticsearch/service/statsd/StatsdService.java index 5409790..fbd319a 100644 --- a/src/main/java/org/elasticsearch/service/statsd/StatsdService.java +++ b/src/main/java/org/elasticsearch/service/statsd/StatsdService.java @@ -34,6 +34,7 @@ public class StatsdService extends AbstractLifecycleComponent { private final TimeValue statsdRefreshInternal; private final String statsdPrefix; private final String statsdNodeName; + private final Boolean statsdReportNodeIndices; private final Boolean statsdReportIndices; private final Boolean statsdReportShards; private final Boolean statsdReportFsDetails; @@ -63,6 +64,9 @@ public StatsdService(Settings settings, ClusterService clusterService, IndicesSe this.statsdNodeName = settings.get( "metrics.statsd.node_name" ); + this.statsdReportNodeIndices = settings.getAsBoolean( + "metrics.statsd.report.node_indices", false + ); this.statsdReportIndices = settings.getAsBoolean( "metrics.statsd.report.indices", true ); @@ -121,6 +125,9 @@ public void run() { .equals(Lifecycle.State.STARTED); if (node != null && state != null && isClusterStarted) { + String statsdNodeName = StatsdService.this.statsdNodeName; + if (null == statsdNodeName) statsdNodeName = node.getName(); + // Report node stats -- runs for all nodes StatsdReporter nodeStatsReporter = new StatsdReporterNodeStats( StatsdService.this.nodeService.stats( @@ -135,25 +142,28 @@ public void run() { true, // http false // circuitBreaker ), - StatsdService.this.statsdNodeName == null ? node.getName() : StatsdService.this.statsdNodeName, + statsdNodeName, StatsdService.this.statsdReportFsDetails ); nodeStatsReporter .setStatsDClient(StatsdService.this.statsdClient) .run(); - // Master node is the only one allowed to send cluster wide sums / stats - if (state.nodes().localNodeMaster()) { - // Report cluster wide index totals + // Maybe report index stats per node + if (StatsdService.this.statsdReportNodeIndices && node.isDataNode()) { StatsdReporter nodeIndicesStatsReporter = new StatsdReporterNodeIndicesStats( StatsdService.this.indicesService.stats( false // includePrevious - ) + ), + statsdNodeName ); nodeIndicesStatsReporter .setStatsDClient(StatsdService.this.statsdClient) .run(); + } + // Master node is the only one allowed to send cluster wide sums / stats + if (state.nodes().localNodeMaster()) { // Maybe breakdown numbers by index or shard if ( StatsdService.this.statsdReportIndices || StatsdService.this.statsdReportShards ) { StatsdReporter indicesReporter = new StatsdReporterIndices( From b365bac021634490ce21ae8a87f16d5acdf03fc2 Mon Sep 17 00:00:00 2001 From: Xiao Yu Date: Fri, 8 Aug 2014 11:36:27 -0400 Subject: [PATCH 31/48] Update README with new settings --- README.md | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 62d27df..63c3431 100644 --- a/README.md +++ b/README.md @@ -33,6 +33,7 @@ Configuration is possible via these parameters: * `metrics.statsd.every`: The interval to push data (default: 1m) * `metrics.statsd.prefix`: The metric prefix that's sent with metric names (default: elasticsearch.your_cluster_name) * `metrics.statsd.node_name`: Override the name for node used in the stat keys (default: the ES node name) +* `metrics.statsd.report.node_indices`: If per node index sums should be reported (default: false) * `metrics.statsd.report.indices`: If index level sums should be reported (default: true) * `metrics.statsd.report.shards`: If shard level stats should be reported (default: false) * `metrics.statsd.report.fs_details`: If nodes should break down the FS by device instead of total disk (default: false) @@ -48,10 +49,11 @@ Check your elasticsearch log file for a line like this after adding the configur This plugin reports both node level and cluster level stats, the StatsD keys will be in the formats: -* `{PREFIX}.node.{NODE_NAME}.{STAT_KEY}` -- Node level stats (CPU / JVM / etc.) -* `{PREFIX}.indices.{STAT_KEY}` -- The stats on indices across the entire cluster -* `{PREFIX}.index.{INDEX_NAME}.total.{STAT_KEY}` -- The stats per index summed across all shards, enabled with `metrics.statsd.report.indices` (defaults to `true`) -* `{PREFIX}.index.{INDEX_NAME}.{SHARD_ID}.{STAT_KEY}` -- The stats per shard, enabled with `metrics.statsd.report.shards` (defaults to `false`) +* `{PREFIX}.node.{NODE_NAME}.{STAT_KEY}`: Node level stats (CPU / JVM / etc.) +* `{PREFIX}.node.{NODE_NAME}.indices.{STAT_KEY}`: Index stats summed across the node (off by default) +* `{PREFIX}.indices.{STAT_KEY}`: Index stats summed across the entire cluster +* `{PREFIX}.index.{INDEX_NAME}.total.{STAT_KEY}`: Index stats summed per index across all shards +* `{PREFIX}.index.{INDEX_NAME}.{SHARD_ID}.{STAT_KEY}` -- Index stats per shard (off by default) ## Bugs/TODO From 43aeb904ba32574ec979012369c64bd927faf355 Mon Sep 17 00:00:00 2001 From: Xiao Yu Date: Fri, 8 Aug 2014 12:06:25 -0400 Subject: [PATCH 32/48] Indices Reporter Should Report Cluster Totals * Make indices reporter report cluster totals always. * Let indices reporter sum per index and report if requested. * Let indices reporter report per shard if requested. --- .../service/statsd/StatsdReporterIndices.java | 32 +++++++++++++++---- .../service/statsd/StatsdService.java | 20 ++++++------ 2 files changed, 34 insertions(+), 18 deletions(-) diff --git a/src/main/java/org/elasticsearch/service/statsd/StatsdReporterIndices.java b/src/main/java/org/elasticsearch/service/statsd/StatsdReporterIndices.java index cb9d5b9..8a2f4c1 100644 --- a/src/main/java/org/elasticsearch/service/statsd/StatsdReporterIndices.java +++ b/src/main/java/org/elasticsearch/service/statsd/StatsdReporterIndices.java @@ -11,20 +11,23 @@ public class StatsdReporterIndices extends StatsdReporterIndexStats { private final List indexShards; + private final Boolean reportIndices; private final Boolean reportShards; private String indexName; private CommonStats indexStats; + private CommonStats indexTotalStats; - public StatsdReporterIndices(List indexShards, Boolean reportShards) { + public StatsdReporterIndices(List indexShards, Boolean reportIndices, Boolean reportShards) { this.indexShards = indexShards; + this.reportIndices = reportIndices; this.reportShards = reportShards; } public void run() { try { - for (IndexShard indexShard : this.indexShards) { - this.maybeResetSums(indexShard.shardId().index().name()); + this.indexTotalStats = new CommonStats(CommonStatsFlags.ALL); + for (IndexShard indexShard : this.indexShards) { // Create common stats for shard CommonStats shardStats = new CommonStats(indexShard, CommonStatsFlags.ALL); @@ -35,11 +38,24 @@ public void run() { ); } - // Add to current index totals - this.indexStats.add(shardStats); + if (this.reportIndices) { + this.maybeResetSums(indexShard.shardId().index().name()); + this.indexStats.add(shardStats); + } + + this.indexTotalStats.add(shardStats); } - // Send last index group - this.maybeResetSums(""); + + // Send last index group... maybe + if (this.reportIndices) { + this.maybeResetSums(""); + } + + // Send index totals + this.sendCommonStats( + this.buildMetricName("indices"), + this.indexTotalStats + ); } catch (Exception e) { this.logException(e); } @@ -50,6 +66,7 @@ private void maybeResetSums(String newIndexName) { return; // Same index do nothing } + // Index name changed to some other value, send the last sum if (this.indexName != null) { this.sendCommonStats( this.buildMetricName("index." + this.indexName + ".total"), @@ -57,6 +74,7 @@ private void maybeResetSums(String newIndexName) { ); } + // Set new index name and reset to empty common stats this.indexName = newIndexName; this.indexStats = new CommonStats(CommonStatsFlags.ALL); } diff --git a/src/main/java/org/elasticsearch/service/statsd/StatsdService.java b/src/main/java/org/elasticsearch/service/statsd/StatsdService.java index fbd319a..38c94b2 100644 --- a/src/main/java/org/elasticsearch/service/statsd/StatsdService.java +++ b/src/main/java/org/elasticsearch/service/statsd/StatsdService.java @@ -92,7 +92,7 @@ protected void doStart() throws ElasticsearchException { ); } else { this.logger.error( - "StatsD reporting disabled, no statsd host configured" + "StatsD reporting disabled, no StatsD host configured" ); } } @@ -164,16 +164,14 @@ public void run() { // Master node is the only one allowed to send cluster wide sums / stats if (state.nodes().localNodeMaster()) { - // Maybe breakdown numbers by index or shard - if ( StatsdService.this.statsdReportIndices || StatsdService.this.statsdReportShards ) { - StatsdReporter indicesReporter = new StatsdReporterIndices( - this.getIndexShards(StatsdService.this.indicesService), - StatsdService.this.statsdReportShards - ); - indicesReporter - .setStatsDClient(StatsdService.this.statsdClient) - .run(); - } + StatsdReporter indicesReporter = new StatsdReporterIndices( + this.getIndexShards(StatsdService.this.indicesService), + StatsdService.this.statsdReportIndices, + StatsdService.this.statsdReportShards + ); + indicesReporter + .setStatsDClient(StatsdService.this.statsdClient) + .run(); } } From 523190101db28c7b4d4eec385ed7a57d589e44ea Mon Sep 17 00:00:00 2001 From: Xiao Yu Date: Fri, 8 Aug 2014 12:18:20 -0400 Subject: [PATCH 33/48] Fix var names, 'type' is a loaded word in ES so don't use it if it does not mean doc type --- .../statsd/StatsdReporterNodeStats.java | 166 +++++++++--------- 1 file changed, 83 insertions(+), 83 deletions(-) diff --git a/src/main/java/org/elasticsearch/service/statsd/StatsdReporterNodeStats.java b/src/main/java/org/elasticsearch/service/statsd/StatsdReporterNodeStats.java index f21e3b8..141240e 100644 --- a/src/main/java/org/elasticsearch/service/statsd/StatsdReporterNodeStats.java +++ b/src/main/java/org/elasticsearch/service/statsd/StatsdReporterNodeStats.java @@ -43,113 +43,113 @@ public void run() { } private void sendNodeThreadPoolStats(ThreadPoolStats threadPoolStats) { - String type = this.getPrefix("thread_pool"); + String prefix = this.getPrefix("thread_pool"); Iterator statsIterator = threadPoolStats.iterator(); while (statsIterator.hasNext()) { ThreadPoolStats.Stats stats = statsIterator.next(); - String id = type + "." + stats.getName(); - - this.sendGauge(id, "threads", stats.getThreads()); - this.sendGauge(id, "queue", stats.getQueue()); - this.sendGauge(id, "active", stats.getActive()); - this.sendGauge(id, "rejected", stats.getRejected()); - this.sendGauge(id, "largest", stats.getLargest()); - this.sendGauge(id, "completed", stats.getCompleted()); + String threadPoolType = prefix + "." + stats.getName(); + + this.sendGauge(threadPoolType, "threads", stats.getThreads()); + this.sendGauge(threadPoolType, "queue", stats.getQueue()); + this.sendGauge(threadPoolType, "active", stats.getActive()); + this.sendGauge(threadPoolType, "rejected", stats.getRejected()); + this.sendGauge(threadPoolType, "largest", stats.getLargest()); + this.sendGauge(threadPoolType, "completed", stats.getCompleted()); } } private void sendNodeTransportStats(TransportStats transportStats) { - String type = this.getPrefix("transport"); - this.sendGauge(type, "server_open", transportStats.serverOpen()); - this.sendGauge(type, "rx_count", transportStats.rxCount()); - this.sendGauge(type, "rx_size_in_bytes", transportStats.rxSize().bytes()); - this.sendGauge(type, "tx_count", transportStats.txCount()); - this.sendGauge(type, "tx_size_in_bytes", transportStats.txSize().bytes()); + String prefix = this.getPrefix("transport"); + this.sendGauge(prefix, "server_open", transportStats.serverOpen()); + this.sendGauge(prefix, "rx_count", transportStats.rxCount()); + this.sendGauge(prefix, "rx_size_in_bytes", transportStats.rxSize().bytes()); + this.sendGauge(prefix, "tx_count", transportStats.txCount()); + this.sendGauge(prefix, "tx_size_in_bytes", transportStats.txSize().bytes()); } private void sendNodeProcessStats(ProcessStats processStats) { - String type = this.getPrefix("process"); + String prefix = this.getPrefix("process"); - this.sendGauge(type, "open_file_descriptors", processStats.openFileDescriptors()); + this.sendGauge(prefix, "open_file_descriptors", processStats.openFileDescriptors()); if (processStats.cpu() != null) { - this.sendGauge(type + ".cpu", "percent", processStats.cpu().percent()); - this.sendGauge(type + ".cpu", "sys_in_millis", processStats.cpu().sys().millis()); - this.sendGauge(type + ".cpu", "user_in_millis", processStats.cpu().user().millis()); - this.sendGauge(type + ".cpu", "total_in_millis", processStats.cpu().total().millis()); + this.sendGauge(prefix + ".cpu", "percent", processStats.cpu().percent()); + this.sendGauge(prefix + ".cpu", "sys_in_millis", processStats.cpu().sys().millis()); + this.sendGauge(prefix + ".cpu", "user_in_millis", processStats.cpu().user().millis()); + this.sendGauge(prefix + ".cpu", "total_in_millis", processStats.cpu().total().millis()); } if (processStats.mem() != null) { - this.sendGauge(type + ".mem", "resident_in_bytes", processStats.mem().resident().bytes()); - this.sendGauge(type + ".mem", "share_in_bytes", processStats.mem().share().bytes()); - this.sendGauge(type + ".mem", "total_virtual_in_bytes", processStats.mem().totalVirtual().bytes()); + this.sendGauge(prefix + ".mem", "resident_in_bytes", processStats.mem().resident().bytes()); + this.sendGauge(prefix + ".mem", "share_in_bytes", processStats.mem().share().bytes()); + this.sendGauge(prefix + ".mem", "total_virtual_in_bytes", processStats.mem().totalVirtual().bytes()); } } private void sendNodeOsStats(OsStats osStats) { - String type = this.getPrefix("os"); + String prefix = this.getPrefix("os"); // Java client does not support doubles yet :( // https://github.com/tim-group/java-statsd-client/issues/19 double[] loadAverage = osStats.getLoadAverage(); if (loadAverage.length > 0) { - this.sendGauge(type + ".load_average", "1m", (long) loadAverage[0]); - this.sendGauge(type + ".load_average", "5m", (long) loadAverage[1]); - this.sendGauge(type + ".load_average", "15m", (long) loadAverage[2]); + this.sendGauge(prefix + ".load_average", "1m", (long) loadAverage[0]); + this.sendGauge(prefix + ".load_average", "5m", (long) loadAverage[1]); + this.sendGauge(prefix + ".load_average", "15m", (long) loadAverage[2]); } if (osStats.cpu() != null) { - this.sendGauge(type + ".cpu", "sys", osStats.cpu().sys()); - this.sendGauge(type + ".cpu", "user", osStats.cpu().user()); - this.sendGauge(type + ".cpu", "idle", osStats.cpu().idle()); - this.sendGauge(type + ".cpu", "stolen", osStats.cpu().stolen()); + this.sendGauge(prefix + ".cpu", "sys", osStats.cpu().sys()); + this.sendGauge(prefix + ".cpu", "user", osStats.cpu().user()); + this.sendGauge(prefix + ".cpu", "idle", osStats.cpu().idle()); + this.sendGauge(prefix + ".cpu", "stolen", osStats.cpu().stolen()); } if (osStats.mem() != null) { - this.sendGauge(type + ".mem", "free_in_bytes", osStats.mem().free().bytes()); - this.sendGauge(type + ".mem", "used_in_bytes", osStats.mem().used().bytes()); - this.sendGauge(type + ".mem", "free_percent", osStats.mem().freePercent()); - this.sendGauge(type + ".mem", "used_percent", osStats.mem().usedPercent()); - this.sendGauge(type + ".mem", "actual_free_in_bytes", osStats.mem().actualFree().bytes()); - this.sendGauge(type + ".mem", "actual_used_in_bytes", osStats.mem().actualUsed().bytes()); + this.sendGauge(prefix + ".mem", "free_in_bytes", osStats.mem().free().bytes()); + this.sendGauge(prefix + ".mem", "used_in_bytes", osStats.mem().used().bytes()); + this.sendGauge(prefix + ".mem", "free_percent", osStats.mem().freePercent()); + this.sendGauge(prefix + ".mem", "used_percent", osStats.mem().usedPercent()); + this.sendGauge(prefix + ".mem", "actual_free_in_bytes", osStats.mem().actualFree().bytes()); + this.sendGauge(prefix + ".mem", "actual_used_in_bytes", osStats.mem().actualUsed().bytes()); } if (osStats.swap() != null) { - this.sendGauge(type + ".swap", "free_in_bytes", osStats.swap().free().bytes()); - this.sendGauge(type + ".swap", "used_in_bytes", osStats.swap().used().bytes()); + this.sendGauge(prefix + ".swap", "free_in_bytes", osStats.swap().free().bytes()); + this.sendGauge(prefix + ".swap", "used_in_bytes", osStats.swap().used().bytes()); } } private void sendNodeNetworkStats(NetworkStats networkStats) { - String type = this.getPrefix("network.tcp"); + String prefix = this.getPrefix("network.tcp"); NetworkStats.Tcp tcp = networkStats.tcp(); // might be null, if sigar isnt loaded if (tcp != null) { - this.sendGauge(type, "active_opens", tcp.getActiveOpens()); - this.sendGauge(type, "passive_opens", tcp.getPassiveOpens()); - this.sendGauge(type, "curr_estab", tcp.getCurrEstab()); - this.sendGauge(type, "in_segs", tcp.inSegs()); - this.sendGauge(type, "out_segs", tcp.outSegs()); - this.sendGauge(type, "retrans_segs", tcp.retransSegs()); - this.sendGauge(type, "estab_resets", tcp.estabResets()); - this.sendGauge(type, "attempt_fails", tcp.attemptFails()); - this.sendGauge(type, "in_errs", tcp.inErrs()); - this.sendGauge(type, "out_rsts", tcp.outRsts()); + this.sendGauge(prefix, "active_opens", tcp.getActiveOpens()); + this.sendGauge(prefix, "passive_opens", tcp.getPassiveOpens()); + this.sendGauge(prefix, "curr_estab", tcp.getCurrEstab()); + this.sendGauge(prefix, "in_segs", tcp.inSegs()); + this.sendGauge(prefix, "out_segs", tcp.outSegs()); + this.sendGauge(prefix, "retrans_segs", tcp.retransSegs()); + this.sendGauge(prefix, "estab_resets", tcp.estabResets()); + this.sendGauge(prefix, "attempt_fails", tcp.attemptFails()); + this.sendGauge(prefix, "in_errs", tcp.inErrs()); + this.sendGauge(prefix, "out_rsts", tcp.outRsts()); } } private void sendNodeJvmStats(JvmStats jvmStats) { - String type = this.getPrefix("jvm"); + String prefix = this.getPrefix("jvm"); // mem - this.sendGauge(type + ".mem", "heap_used_percent", jvmStats.mem().heapUsedPercent()); - this.sendGauge(type + ".mem", "heap_used_in_bytes", jvmStats.mem().heapUsed().bytes()); - this.sendGauge(type + ".mem", "heap_committed_in_bytes", jvmStats.mem().heapCommitted().bytes()); - this.sendGauge(type + ".mem", "non_heap_used_in_bytes", jvmStats.mem().nonHeapUsed().bytes()); - this.sendGauge(type + ".mem", "non_heap_committed_in_bytes", jvmStats.mem().nonHeapCommitted().bytes()); + this.sendGauge(prefix + ".mem", "heap_used_percent", jvmStats.mem().heapUsedPercent()); + this.sendGauge(prefix + ".mem", "heap_used_in_bytes", jvmStats.mem().heapUsed().bytes()); + this.sendGauge(prefix + ".mem", "heap_committed_in_bytes", jvmStats.mem().heapCommitted().bytes()); + this.sendGauge(prefix + ".mem", "non_heap_used_in_bytes", jvmStats.mem().nonHeapUsed().bytes()); + this.sendGauge(prefix + ".mem", "non_heap_committed_in_bytes", jvmStats.mem().nonHeapCommitted().bytes()); for (JvmStats.MemoryPool memoryPool : jvmStats.mem()) { - String memoryPoolType = type + ".mem.pools." + memoryPool.name(); + String memoryPoolType = prefix + ".mem.pools." + memoryPool.name(); this.sendGauge(memoryPoolType, "max_in_bytes", memoryPool.max().bytes()); this.sendGauge(memoryPoolType, "used_in_bytes", memoryPool.used().bytes()); @@ -158,75 +158,75 @@ private void sendNodeJvmStats(JvmStats jvmStats) { } // threads - this.sendGauge(type + ".threads", "count", jvmStats.threads().count()); - this.sendGauge(type + ".threads", "peak_count", jvmStats.threads().peakCount()); + this.sendGauge(prefix + ".threads", "count", jvmStats.threads().count()); + this.sendGauge(prefix + ".threads", "peak_count", jvmStats.threads().peakCount()); // garbage collectors for (JvmStats.GarbageCollector collector : jvmStats.gc()) { - String id = type + ".gc.collectors." + collector.name(); + String gcCollectorType = prefix + ".gc.collectors." + collector.name(); - this.sendGauge(id, "collection_count", collector.collectionCount()); - this.sendGauge(id, "collection_time_in_millis", collector.collectionTime().millis()); + this.sendGauge(gcCollectorType, "collection_count", collector.collectionCount()); + this.sendGauge(gcCollectorType, "collection_time_in_millis", collector.collectionTime().millis()); } // TODO: buffer pools } private void sendNodeHttpStats(HttpStats httpStats) { - String type = this.getPrefix("http"); - this.sendGauge(type, "current_open", httpStats.getServerOpen()); - this.sendGauge(type, "total_opened", httpStats.getTotalOpen()); + String prefix = this.getPrefix("http"); + this.sendGauge(prefix, "current_open", httpStats.getServerOpen()); + this.sendGauge(prefix, "total_opened", httpStats.getTotalOpen()); } private void sendNodeFsStats(FsStats fs) { // Send total - String type = this.getPrefix("fs"); - this.sendNodeFsStatsInfo(type + ".total", fs.total()); + String prefix = this.getPrefix("fs"); + this.sendNodeFsStatsInfo(prefix + ".total", fs.total()); // Maybe send details if (this.statsdReportFsDetails) { Iterator infoIterator = fs.iterator(); while (infoIterator.hasNext()) { FsStats.Info info = infoIterator.next(); - this.sendNodeFsStatsInfo(type + ".data", info); + this.sendNodeFsStatsInfo(prefix + ".data", info); } } } - private void sendNodeFsStatsInfo(String type, FsStats.Info info) { + private void sendNodeFsStatsInfo(String prefix, FsStats.Info info) { // Construct detailed path - String typeAppend = ""; + String prefixAppend = ""; if (info.getPath() != null) - typeAppend += "." + info.getPath(); + prefixAppend += "." + info.getPath(); if (info.getMount() != null) - typeAppend += "." + info.getMount(); + prefixAppend += "." + info.getMount(); if (info.getDev() != null) - typeAppend += "." + info.getDev(); + prefixAppend += "." + info.getDev(); if (info.getAvailable().bytes() != -1) - this.sendGauge(type + typeAppend, "available_in_bytes", info.getAvailable().bytes()); + this.sendGauge(prefix + prefixAppend, "available_in_bytes", info.getAvailable().bytes()); if (info.getTotal().bytes() != -1) - this.sendGauge(type + typeAppend, "total_in_bytes", info.getTotal().bytes()); + this.sendGauge(prefix + prefixAppend, "total_in_bytes", info.getTotal().bytes()); if (info.getFree().bytes() != -1) - this.sendGauge(type + typeAppend, "free_in_bytes", info.getFree().bytes()); + this.sendGauge(prefix + prefixAppend, "free_in_bytes", info.getFree().bytes()); // disk_io_op is sum of reads and writes (use graphite functions) if (info.getDiskReads() != -1) - this.sendGauge(type + typeAppend, "disk_reads", info.getDiskReads()); + this.sendGauge(prefix + prefixAppend, "disk_reads", info.getDiskReads()); if (info.getDiskWrites() != -1) - this.sendGauge(type + typeAppend, "disk_writes", info.getDiskWrites()); + this.sendGauge(prefix + prefixAppend, "disk_writes", info.getDiskWrites()); // disk_io_size_in_bytes is sum of reads and writes (use graphite functions) if (info.getDiskReadSizeInBytes() != -1) - this.sendGauge(type + typeAppend, "disk_read_size_in_bytes", info.getDiskReadSizeInBytes()); + this.sendGauge(prefix + prefixAppend, "disk_read_size_in_bytes", info.getDiskReadSizeInBytes()); if (info.getDiskWriteSizeInBytes() != -1) - this.sendGauge(type + typeAppend, "disk_write_size_in_bytes", info.getDiskWriteSizeInBytes()); + this.sendGauge(prefix + prefixAppend, "disk_write_size_in_bytes", info.getDiskWriteSizeInBytes()); /** TODO: Find out if these stats are useful. if (info.getDiskQueue() != -1) - this.sendGauge(type + typeAppend, "disk_queue", (long) info.getDiskQueue()); + this.sendGauge(prefix + prefixAppend, "disk_queue", (long) info.getDiskQueue()); if (info.getDiskServiceTime() != -1) - this.sendGauge(type + typeAppend, "disk_service_time", (long) info.getDiskServiceTime()); + this.sendGauge(prefix + prefixAppend, "disk_service_time", (long) info.getDiskServiceTime()); */ } From 5a8bd2ab665187ef50b22674bcf6d25260375c26 Mon Sep 17 00:00:00 2001 From: Xiao Yu Date: Fri, 8 Aug 2014 12:31:13 -0400 Subject: [PATCH 34/48] Fix name in credits --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 63c3431..8721fd1 100644 --- a/README.md +++ b/README.md @@ -64,7 +64,7 @@ This plugin reports both node level and cluster level stats, the StatsD keys wil ## Credits -Heavily inspired by the excellent [metrics library](http://metrics.codehale.com) by Code Hale and its [GraphiteReporter add-on](http://metrics.codahale.com/manual/graphite/). +Heavily inspired by the excellent [metrics library](http://metrics.codahale.com) by Coda Hale and its [GraphiteReporter add-on](http://metrics.codahale.com/manual/graphite/). ## License From 6b9a0be2f156f3b073c0e4725c4928727c1dc4a8 Mon Sep 17 00:00:00 2001 From: Xiao Yu Date: Fri, 8 Aug 2014 12:37:08 -0400 Subject: [PATCH 35/48] Version Bump to v0.3.1 --- README.md | 8 +++++--- pom.xml | 4 ++-- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 8721fd1..ab4a216 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# Elasticsearch statsd plugin +# Elasticsearch StatsD Plugin This plugin creates a little push service, which regularly updates a StatsD host with indices stats and nodes stats. Index stats that apply across the entire cluster is only pushed from the elected master which node level stats are pushed from every node. @@ -11,7 +11,7 @@ The data sent to the StatsD server tries to be roughly equivalent to the [Indice To install a prepackaged plugin use the following command: ``` -bin/plugin -install statsd -url https://github.com/Automattic/elasticsearch-statsd-plugin/releases/download/v0.3/elasticsearch-statsd-0.3.zip +bin/plugin -install statsd -url https://github.com/Automattic/elasticsearch-statsd-plugin/releases/download/v0.3.1/elasticsearch-statsd-0.3.1.zip ``` You can also build your own by doing the following: @@ -20,7 +20,7 @@ You can also build your own by doing the following: git clone http://github.com/Automattic/elasticsearch-statsd-plugin.git cd elasticsearch-statsd-plugin mvn package -bin/plugin -install statsd -url file:///absolute/path/to/current/dir/target/releases/elasticsearch-statsd-0.3-SNAPSHOT.zip +bin/plugin -install statsd -url file:///absolute/path/to/current/dir/target/releases/elasticsearch-statsd-0.3.1.zip ``` @@ -64,6 +64,8 @@ This plugin reports both node level and cluster level stats, the StatsD keys wil ## Credits +This is a fork of the Swoop plugin (swoop-inc/elasticsearch-statsd-plugin@beeca78240) for multi node clusters on ES 1.x. + Heavily inspired by the excellent [metrics library](http://metrics.codahale.com) by Coda Hale and its [GraphiteReporter add-on](http://metrics.codahale.com/manual/graphite/). diff --git a/pom.xml b/pom.xml index 005e17f..ff588a9 100644 --- a/pom.xml +++ b/pom.xml @@ -6,10 +6,10 @@ 4.0.0 de.spinscale.elasticsearch elasticsearch-statsd - 0.3-SNAPSHOT + 0.3.1 jar StatsD monitoring plugin for Elasticsearch - https://github.com/swoop-inc/elasticsearch-statsd-plugin/ + https://github.com/Automattic/elasticsearch-statsd-plugin/ 1.0.1 From 5c5cc0563228bc98c9276ef29dc7d6804b232477 Mon Sep 17 00:00:00 2001 From: Xiao Yu Date: Fri, 8 Aug 2014 13:13:58 -0400 Subject: [PATCH 36/48] Update README -- fix URL --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index ab4a216..19953ca 100644 --- a/README.md +++ b/README.md @@ -64,7 +64,7 @@ This plugin reports both node level and cluster level stats, the StatsD keys wil ## Credits -This is a fork of the Swoop plugin (swoop-inc/elasticsearch-statsd-plugin@beeca78240) for multi node clusters on ES 1.x. +This is a fork of the [Swoop plugin](https://github.com/swoop-inc/elasticsearch-statsd-plugin) for multi-node clusters on ES 1.x. Heavily inspired by the excellent [metrics library](http://metrics.codahale.com) by Coda Hale and its [GraphiteReporter add-on](http://metrics.codahale.com/manual/graphite/). From e2d2bb2455acde23404633d77a7e8f897a65bb7d Mon Sep 17 00:00:00 2001 From: Xiao Yu Date: Fri, 8 Aug 2014 15:51:35 -0400 Subject: [PATCH 37/48] Upgrade Java StatsD Client\n\nNeeded to send longs not ints --- pom.xml | 2 +- .../org/elasticsearch/service/statsd/StatsdReporter.java | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pom.xml b/pom.xml index ff588a9..811ac16 100644 --- a/pom.xml +++ b/pom.xml @@ -21,7 +21,7 @@ com.timgroup java-statsd-client - 2.0.0 + 3.0.1 diff --git a/src/main/java/org/elasticsearch/service/statsd/StatsdReporter.java b/src/main/java/org/elasticsearch/service/statsd/StatsdReporter.java index 07ac3ec..cba1e83 100644 --- a/src/main/java/org/elasticsearch/service/statsd/StatsdReporter.java +++ b/src/main/java/org/elasticsearch/service/statsd/StatsdReporter.java @@ -19,15 +19,15 @@ public StatsdReporter setStatsDClient(StatsDClient statsdClient) { public abstract void run(); protected void sendGauge(String name, String valueName, long value) { - this.statsdClient.gauge(this.join(name, valueName), (int) value); + this.statsdClient.gauge(this.join(name, valueName), value); } protected void sendCount(String name, String valueName, long value) { - this.statsdClient.count(this.join(name, valueName), (int) value); + this.statsdClient.count(this.join(name, valueName), value); } protected void sendTime(String name, String valueName, long value) { - this.statsdClient.time(this.join(name, valueName), (int) value); + this.statsdClient.time(this.join(name, valueName), value); } protected String sanitizeString(String s) { From 887836c71156322d6b8e8698561a2e5fd449e635 Mon Sep 17 00:00:00 2001 From: Xiao Yu Date: Fri, 8 Aug 2014 17:56:21 -0400 Subject: [PATCH 38/48] Make tests spin up 3 nodes to distribute docs better --- .../module/statsd/test/StatsdPluginIntegrationTest.java | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/test/java/org/elasticsearch/module/statsd/test/StatsdPluginIntegrationTest.java b/src/test/java/org/elasticsearch/module/statsd/test/StatsdPluginIntegrationTest.java index b0b4fda..6c6197b 100644 --- a/src/test/java/org/elasticsearch/module/statsd/test/StatsdPluginIntegrationTest.java +++ b/src/test/java/org/elasticsearch/module/statsd/test/StatsdPluginIntegrationTest.java @@ -25,14 +25,16 @@ public class StatsdPluginIntegrationTest private String type = RandomStringGenerator.randomAlphabetic(6).toLowerCase(); private Node node_1; private Node node_2; + private Node node_3; @Before public void startStatsdMockServerAndNode() throws Exception { statsdMockServer = new StatsdMockServer(STATSD_SERVER_PORT); statsdMockServer.start(); - node_1 = createNode(clusterName, 2, STATSD_SERVER_PORT, "1s"); - node_2 = createNode(clusterName, 2, STATSD_SERVER_PORT, "1s"); + node_1 = createNode(clusterName, 4, STATSD_SERVER_PORT, "1s"); + node_2 = createNode(clusterName, 4, STATSD_SERVER_PORT, "1s"); + node_3 = createNode(clusterName, 4, STATSD_SERVER_PORT, "1s"); } @After @@ -45,6 +47,9 @@ public void stopStatsdServer() throws Exception if (!node_2.isClosed()) { node_2.close(); } + if (!node_3.isClosed()) { + node_3.close(); + } } @Test From d92279e068871da663d9823c19cee8c14a8e82d9 Mon Sep 17 00:00:00 2001 From: Xiao Yu Date: Fri, 8 Aug 2014 18:02:07 -0400 Subject: [PATCH 39/48] Use IndicesStatsRequestBuilder / IndicesStatsResponse in ES to gather index stats --- .../service/statsd/StatsdReporterIndices.java | 73 ++++++------------- .../service/statsd/StatsdService.java | 33 +++------ 2 files changed, 32 insertions(+), 74 deletions(-) diff --git a/src/main/java/org/elasticsearch/service/statsd/StatsdReporterIndices.java b/src/main/java/org/elasticsearch/service/statsd/StatsdReporterIndices.java index 8a2f4c1..55accd6 100644 --- a/src/main/java/org/elasticsearch/service/statsd/StatsdReporterIndices.java +++ b/src/main/java/org/elasticsearch/service/statsd/StatsdReporterIndices.java @@ -4,81 +4,52 @@ import java.util.List; import java.util.Map; -import org.elasticsearch.index.shard.service.IndexShard; -import org.elasticsearch.action.admin.indices.stats.CommonStats; -import org.elasticsearch.action.admin.indices.stats.CommonStatsFlags; +import org.elasticsearch.action.admin.indices.stats.*; public class StatsdReporterIndices extends StatsdReporterIndexStats { - private final List indexShards; + private final IndicesStatsResponse indicesStatsResponse; private final Boolean reportIndices; private final Boolean reportShards; - private String indexName; - private CommonStats indexStats; - private CommonStats indexTotalStats; - public StatsdReporterIndices(List indexShards, Boolean reportIndices, Boolean reportShards) { - this.indexShards = indexShards; + public StatsdReporterIndices(IndicesStatsResponse indicesStatsResponse, Boolean reportIndices, Boolean reportShards) { + this.indicesStatsResponse = indicesStatsResponse; this.reportIndices = reportIndices; this.reportShards = reportShards; } public void run() { try { - this.indexTotalStats = new CommonStats(CommonStatsFlags.ALL); + // First report totals + this.sendCommonStats( + this.buildMetricName("indices"), + this.indicesStatsResponse.getTotal() + ); - for (IndexShard indexShard : this.indexShards) { - // Create common stats for shard - CommonStats shardStats = new CommonStats(indexShard, CommonStatsFlags.ALL); + if (this.reportIndices) { + for (IndexStats indexStats : this.indicesStatsResponse.getIndices().values()) { + String indexPrefix = "index." + indexStats.getIndex(); - if (this.reportShards) { this.sendCommonStats( - this.buildMetricName("index." + this.indexName + "." + indexShard.shardId().id()), - shardStats + this.buildMetricName(indexPrefix + ".total"), + indexStats.getTotal() ); - } - if (this.reportIndices) { - this.maybeResetSums(indexShard.shardId().index().name()); - this.indexStats.add(shardStats); + if (this.reportShards) { + for (IndexShardStats indexShardStats : indexStats.getIndexShards().values()) { + this.sendCommonStats( + this.buildMetricName(indexPrefix + "." + indexShardStats.getShardId().id()), + indexShardStats.getTotal() + ); + } + } } - - this.indexTotalStats.add(shardStats); } - - // Send last index group... maybe - if (this.reportIndices) { - this.maybeResetSums(""); - } - - // Send index totals - this.sendCommonStats( - this.buildMetricName("indices"), - this.indexTotalStats - ); } catch (Exception e) { this.logException(e); } } - private void maybeResetSums(String newIndexName) { - if (this.indexName == newIndexName) { - return; // Same index do nothing - } - - // Index name changed to some other value, send the last sum - if (this.indexName != null) { - this.sendCommonStats( - this.buildMetricName("index." + this.indexName + ".total"), - this.indexStats - ); - } - - // Set new index name and reset to empty common stats - this.indexName = newIndexName; - this.indexStats = new CommonStats(CommonStatsFlags.ALL); - } - private void sendCommonStats(String prefix, CommonStats stats) { this.sendDocsStats(prefix + ".docs", stats.getDocs()); this.sendStoreStats(prefix + ".store", stats.getStore()); diff --git a/src/main/java/org/elasticsearch/service/statsd/StatsdService.java b/src/main/java/org/elasticsearch/service/statsd/StatsdService.java index 38c94b2..791f4f2 100644 --- a/src/main/java/org/elasticsearch/service/statsd/StatsdService.java +++ b/src/main/java/org/elasticsearch/service/statsd/StatsdService.java @@ -1,31 +1,26 @@ package org.elasticsearch.service.statsd; -import java.util.List; - import org.elasticsearch.ElasticsearchException; -import org.elasticsearch.action.admin.cluster.node.stats.NodeStats; import org.elasticsearch.action.admin.indices.stats.CommonStatsFlags; import org.elasticsearch.cluster.ClusterService; import org.elasticsearch.cluster.node.DiscoveryNode; -import org.elasticsearch.common.collect.Lists; import org.elasticsearch.common.component.AbstractLifecycleComponent; import org.elasticsearch.common.component.Lifecycle; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.util.concurrent.EsExecutors; -import org.elasticsearch.index.service.IndexService; -import org.elasticsearch.index.shard.service.IndexShard; import org.elasticsearch.indices.IndicesService; -import org.elasticsearch.indices.NodeIndicesStats; import org.elasticsearch.node.service.NodeService; import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.client.Client; import com.timgroup.statsd.NonBlockingStatsDClient; import com.timgroup.statsd.StatsDClient; public class StatsdService extends AbstractLifecycleComponent { + private final Client client; private final ClusterService clusterService; private final IndicesService indicesService; private final NodeService nodeService; @@ -44,8 +39,9 @@ public class StatsdService extends AbstractLifecycleComponent { private volatile boolean closed; @Inject - public StatsdService(Settings settings, ClusterService clusterService, IndicesService indicesService, NodeService nodeService) { + public StatsdService(Settings settings, Client client, ClusterService clusterService, IndicesService indicesService, NodeService nodeService) { super(settings); + this.client = client; this.clusterService = clusterService; this.indicesService = indicesService; this.nodeService = nodeService; @@ -165,7 +161,12 @@ public void run() { // Master node is the only one allowed to send cluster wide sums / stats if (state.nodes().localNodeMaster()) { StatsdReporter indicesReporter = new StatsdReporterIndices( - this.getIndexShards(StatsdService.this.indicesService), + StatsdService.this.client + .admin() // AdminClient + .indices() // IndicesAdminClient + .prepareStats() // IndicesStatsRequestBuilder + .all() // IndicesStatsRequestBuilder + .get(), // IndicesStatsResponse StatsdService.this.statsdReportIndices, StatsdService.this.statsdReportShards ); @@ -182,19 +183,5 @@ public void run() { } } } - - private List getIndexShards(IndicesService indicesService) { - List indexShards = Lists.newArrayList(); - - String[] indices = indicesService.indices().toArray(new String[] {}); - for (String indexName : indices) { - IndexService indexService = indicesService.indexServiceSafe(indexName); - for (int shardId : indexService.shardIds()) { - indexShards.add(indexService.shard(shardId)); - } - } - - return indexShards; - } } } From a4143b706abf034590c9f55cd4a1b8f396727258 Mon Sep 17 00:00:00 2001 From: Xiao Yu Date: Fri, 8 Aug 2014 18:03:13 -0400 Subject: [PATCH 40/48] Add Warmer and Percolate stats --- .../service/statsd/StatsdReporterIndexStats.java | 8 ++++++++ .../service/statsd/StatsdReporterIndices.java | 4 ++-- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/elasticsearch/service/statsd/StatsdReporterIndexStats.java b/src/main/java/org/elasticsearch/service/statsd/StatsdReporterIndexStats.java index 5796335..fdf08b1 100644 --- a/src/main/java/org/elasticsearch/service/statsd/StatsdReporterIndexStats.java +++ b/src/main/java/org/elasticsearch/service/statsd/StatsdReporterIndexStats.java @@ -19,6 +19,7 @@ import org.elasticsearch.index.percolator.stats.PercolateStats; import org.elasticsearch.search.suggest.completion.CompletionStats; import org.elasticsearch.index.engine.SegmentsStats; +import org.elasticsearch.index.warmer.WarmerStats; public abstract class StatsdReporterIndexStats extends StatsdReporter { @@ -84,6 +85,13 @@ protected void sendFlushStats(String name, FlushStats flushStats) { this.sendGauge(name, "total_time_in_millis", flushStats.getTotalTimeInMillis()); } + protected void sendWarmerStats(String name, WarmerStats warmerStats) { + if (null == warmerStats) return; + this.sendGauge(name, "current", warmerStats.current()); + this.sendGauge(name, "total", warmerStats.total()); + this.sendGauge(name, "total_time_in_millis", warmerStats.totalTimeInMillis()); + } + protected void sendFilterCacheStats(String name, FilterCacheStats filterCacheStats) { if (null == filterCacheStats) return; this.sendGauge(name, "memory_size_in_bytes", filterCacheStats.getMemorySizeInBytes()); diff --git a/src/main/java/org/elasticsearch/service/statsd/StatsdReporterIndices.java b/src/main/java/org/elasticsearch/service/statsd/StatsdReporterIndices.java index 55accd6..7543334 100644 --- a/src/main/java/org/elasticsearch/service/statsd/StatsdReporterIndices.java +++ b/src/main/java/org/elasticsearch/service/statsd/StatsdReporterIndices.java @@ -59,11 +59,11 @@ private void sendCommonStats(String prefix, CommonStats stats) { this.sendMergeStats(prefix + ".merges", stats.getMerge()); this.sendRefreshStats(prefix + ".refresh", stats.getRefresh()); this.sendFlushStats(prefix + ".flush", stats.getFlush()); - //TODO: getWarmer + this.sendWarmerStats(prefix + ".warmer", stats.getWarmer()); this.sendFilterCacheStats(prefix + ".filter_cache", stats.getFilterCache()); this.sendIdCacheStats(prefix + ".id_cache", stats.getIdCache()); this.sendFielddataCacheStats(prefix + ".fielddata", stats.getFieldData()); - //TODO: getPercolate + this.sendPercolateStats(prefix + ".percolate", stats.getPercolate()); this.sendCompletionStats(prefix + ".completion", stats.getCompletion()); this.sendSegmentsStats(prefix + ".segments", stats.getSegments()); //TODO: getTranslog From a5c3e2c314ea03fa39678674fa75e3219c108cee Mon Sep 17 00:00:00 2001 From: Xiao Yu Date: Fri, 8 Aug 2014 18:09:23 -0400 Subject: [PATCH 41/48] Version Bump --- README.md | 4 ++-- pom.xml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index ab4a216..ea82bf3 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ The data sent to the StatsD server tries to be roughly equivalent to the [Indice To install a prepackaged plugin use the following command: ``` -bin/plugin -install statsd -url https://github.com/Automattic/elasticsearch-statsd-plugin/releases/download/v0.3.1/elasticsearch-statsd-0.3.1.zip +bin/plugin -install statsd -url https://github.com/Automattic/elasticsearch-statsd-plugin/releases/download/v0.3.2/elasticsearch-statsd-0.3.2.zip ``` You can also build your own by doing the following: @@ -20,7 +20,7 @@ You can also build your own by doing the following: git clone http://github.com/Automattic/elasticsearch-statsd-plugin.git cd elasticsearch-statsd-plugin mvn package -bin/plugin -install statsd -url file:///absolute/path/to/current/dir/target/releases/elasticsearch-statsd-0.3.1.zip +bin/plugin -install statsd -url file:///absolute/path/to/current/dir/target/releases/elasticsearch-statsd-0.3.2.zip ``` diff --git a/pom.xml b/pom.xml index 811ac16..a5390d2 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ 4.0.0 de.spinscale.elasticsearch elasticsearch-statsd - 0.3.1 + 0.3.2 jar StatsD monitoring plugin for Elasticsearch https://github.com/Automattic/elasticsearch-statsd-plugin/ From 97565dc66c290daf0c90a855fd522499526938ad Mon Sep 17 00:00:00 2001 From: Xiao Yu Date: Fri, 8 Aug 2014 15:51:35 -0400 Subject: [PATCH 42/48] Upgrade Java StatsD Client\n\nNeeded to send longs not ints --- pom.xml | 2 +- .../org/elasticsearch/service/statsd/StatsdReporter.java | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pom.xml b/pom.xml index ff588a9..811ac16 100644 --- a/pom.xml +++ b/pom.xml @@ -21,7 +21,7 @@ com.timgroup java-statsd-client - 2.0.0 + 3.0.1 diff --git a/src/main/java/org/elasticsearch/service/statsd/StatsdReporter.java b/src/main/java/org/elasticsearch/service/statsd/StatsdReporter.java index 07ac3ec..cba1e83 100644 --- a/src/main/java/org/elasticsearch/service/statsd/StatsdReporter.java +++ b/src/main/java/org/elasticsearch/service/statsd/StatsdReporter.java @@ -19,15 +19,15 @@ public StatsdReporter setStatsDClient(StatsDClient statsdClient) { public abstract void run(); protected void sendGauge(String name, String valueName, long value) { - this.statsdClient.gauge(this.join(name, valueName), (int) value); + this.statsdClient.gauge(this.join(name, valueName), value); } protected void sendCount(String name, String valueName, long value) { - this.statsdClient.count(this.join(name, valueName), (int) value); + this.statsdClient.count(this.join(name, valueName), value); } protected void sendTime(String name, String valueName, long value) { - this.statsdClient.time(this.join(name, valueName), (int) value); + this.statsdClient.time(this.join(name, valueName), value); } protected String sanitizeString(String s) { From cd7ca2096c2c4ad95c0ab38c5bb3713f601f4392 Mon Sep 17 00:00:00 2001 From: Xiao Yu Date: Fri, 8 Aug 2014 17:56:21 -0400 Subject: [PATCH 43/48] Make tests spin up 3 nodes to distribute docs better --- .../module/statsd/test/StatsdPluginIntegrationTest.java | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/test/java/org/elasticsearch/module/statsd/test/StatsdPluginIntegrationTest.java b/src/test/java/org/elasticsearch/module/statsd/test/StatsdPluginIntegrationTest.java index b0b4fda..6c6197b 100644 --- a/src/test/java/org/elasticsearch/module/statsd/test/StatsdPluginIntegrationTest.java +++ b/src/test/java/org/elasticsearch/module/statsd/test/StatsdPluginIntegrationTest.java @@ -25,14 +25,16 @@ public class StatsdPluginIntegrationTest private String type = RandomStringGenerator.randomAlphabetic(6).toLowerCase(); private Node node_1; private Node node_2; + private Node node_3; @Before public void startStatsdMockServerAndNode() throws Exception { statsdMockServer = new StatsdMockServer(STATSD_SERVER_PORT); statsdMockServer.start(); - node_1 = createNode(clusterName, 2, STATSD_SERVER_PORT, "1s"); - node_2 = createNode(clusterName, 2, STATSD_SERVER_PORT, "1s"); + node_1 = createNode(clusterName, 4, STATSD_SERVER_PORT, "1s"); + node_2 = createNode(clusterName, 4, STATSD_SERVER_PORT, "1s"); + node_3 = createNode(clusterName, 4, STATSD_SERVER_PORT, "1s"); } @After @@ -45,6 +47,9 @@ public void stopStatsdServer() throws Exception if (!node_2.isClosed()) { node_2.close(); } + if (!node_3.isClosed()) { + node_3.close(); + } } @Test From ee59c957899549e34b147d2632642e073c2dcb63 Mon Sep 17 00:00:00 2001 From: Xiao Yu Date: Fri, 8 Aug 2014 18:02:07 -0400 Subject: [PATCH 44/48] Use IndicesStatsRequestBuilder / IndicesStatsResponse in ES to gather index stats --- .../service/statsd/StatsdReporterIndices.java | 73 ++++++------------- .../service/statsd/StatsdService.java | 33 +++------ 2 files changed, 32 insertions(+), 74 deletions(-) diff --git a/src/main/java/org/elasticsearch/service/statsd/StatsdReporterIndices.java b/src/main/java/org/elasticsearch/service/statsd/StatsdReporterIndices.java index 8a2f4c1..55accd6 100644 --- a/src/main/java/org/elasticsearch/service/statsd/StatsdReporterIndices.java +++ b/src/main/java/org/elasticsearch/service/statsd/StatsdReporterIndices.java @@ -4,81 +4,52 @@ import java.util.List; import java.util.Map; -import org.elasticsearch.index.shard.service.IndexShard; -import org.elasticsearch.action.admin.indices.stats.CommonStats; -import org.elasticsearch.action.admin.indices.stats.CommonStatsFlags; +import org.elasticsearch.action.admin.indices.stats.*; public class StatsdReporterIndices extends StatsdReporterIndexStats { - private final List indexShards; + private final IndicesStatsResponse indicesStatsResponse; private final Boolean reportIndices; private final Boolean reportShards; - private String indexName; - private CommonStats indexStats; - private CommonStats indexTotalStats; - public StatsdReporterIndices(List indexShards, Boolean reportIndices, Boolean reportShards) { - this.indexShards = indexShards; + public StatsdReporterIndices(IndicesStatsResponse indicesStatsResponse, Boolean reportIndices, Boolean reportShards) { + this.indicesStatsResponse = indicesStatsResponse; this.reportIndices = reportIndices; this.reportShards = reportShards; } public void run() { try { - this.indexTotalStats = new CommonStats(CommonStatsFlags.ALL); + // First report totals + this.sendCommonStats( + this.buildMetricName("indices"), + this.indicesStatsResponse.getTotal() + ); - for (IndexShard indexShard : this.indexShards) { - // Create common stats for shard - CommonStats shardStats = new CommonStats(indexShard, CommonStatsFlags.ALL); + if (this.reportIndices) { + for (IndexStats indexStats : this.indicesStatsResponse.getIndices().values()) { + String indexPrefix = "index." + indexStats.getIndex(); - if (this.reportShards) { this.sendCommonStats( - this.buildMetricName("index." + this.indexName + "." + indexShard.shardId().id()), - shardStats + this.buildMetricName(indexPrefix + ".total"), + indexStats.getTotal() ); - } - if (this.reportIndices) { - this.maybeResetSums(indexShard.shardId().index().name()); - this.indexStats.add(shardStats); + if (this.reportShards) { + for (IndexShardStats indexShardStats : indexStats.getIndexShards().values()) { + this.sendCommonStats( + this.buildMetricName(indexPrefix + "." + indexShardStats.getShardId().id()), + indexShardStats.getTotal() + ); + } + } } - - this.indexTotalStats.add(shardStats); } - - // Send last index group... maybe - if (this.reportIndices) { - this.maybeResetSums(""); - } - - // Send index totals - this.sendCommonStats( - this.buildMetricName("indices"), - this.indexTotalStats - ); } catch (Exception e) { this.logException(e); } } - private void maybeResetSums(String newIndexName) { - if (this.indexName == newIndexName) { - return; // Same index do nothing - } - - // Index name changed to some other value, send the last sum - if (this.indexName != null) { - this.sendCommonStats( - this.buildMetricName("index." + this.indexName + ".total"), - this.indexStats - ); - } - - // Set new index name and reset to empty common stats - this.indexName = newIndexName; - this.indexStats = new CommonStats(CommonStatsFlags.ALL); - } - private void sendCommonStats(String prefix, CommonStats stats) { this.sendDocsStats(prefix + ".docs", stats.getDocs()); this.sendStoreStats(prefix + ".store", stats.getStore()); diff --git a/src/main/java/org/elasticsearch/service/statsd/StatsdService.java b/src/main/java/org/elasticsearch/service/statsd/StatsdService.java index 38c94b2..791f4f2 100644 --- a/src/main/java/org/elasticsearch/service/statsd/StatsdService.java +++ b/src/main/java/org/elasticsearch/service/statsd/StatsdService.java @@ -1,31 +1,26 @@ package org.elasticsearch.service.statsd; -import java.util.List; - import org.elasticsearch.ElasticsearchException; -import org.elasticsearch.action.admin.cluster.node.stats.NodeStats; import org.elasticsearch.action.admin.indices.stats.CommonStatsFlags; import org.elasticsearch.cluster.ClusterService; import org.elasticsearch.cluster.node.DiscoveryNode; -import org.elasticsearch.common.collect.Lists; import org.elasticsearch.common.component.AbstractLifecycleComponent; import org.elasticsearch.common.component.Lifecycle; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.util.concurrent.EsExecutors; -import org.elasticsearch.index.service.IndexService; -import org.elasticsearch.index.shard.service.IndexShard; import org.elasticsearch.indices.IndicesService; -import org.elasticsearch.indices.NodeIndicesStats; import org.elasticsearch.node.service.NodeService; import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.client.Client; import com.timgroup.statsd.NonBlockingStatsDClient; import com.timgroup.statsd.StatsDClient; public class StatsdService extends AbstractLifecycleComponent { + private final Client client; private final ClusterService clusterService; private final IndicesService indicesService; private final NodeService nodeService; @@ -44,8 +39,9 @@ public class StatsdService extends AbstractLifecycleComponent { private volatile boolean closed; @Inject - public StatsdService(Settings settings, ClusterService clusterService, IndicesService indicesService, NodeService nodeService) { + public StatsdService(Settings settings, Client client, ClusterService clusterService, IndicesService indicesService, NodeService nodeService) { super(settings); + this.client = client; this.clusterService = clusterService; this.indicesService = indicesService; this.nodeService = nodeService; @@ -165,7 +161,12 @@ public void run() { // Master node is the only one allowed to send cluster wide sums / stats if (state.nodes().localNodeMaster()) { StatsdReporter indicesReporter = new StatsdReporterIndices( - this.getIndexShards(StatsdService.this.indicesService), + StatsdService.this.client + .admin() // AdminClient + .indices() // IndicesAdminClient + .prepareStats() // IndicesStatsRequestBuilder + .all() // IndicesStatsRequestBuilder + .get(), // IndicesStatsResponse StatsdService.this.statsdReportIndices, StatsdService.this.statsdReportShards ); @@ -182,19 +183,5 @@ public void run() { } } } - - private List getIndexShards(IndicesService indicesService) { - List indexShards = Lists.newArrayList(); - - String[] indices = indicesService.indices().toArray(new String[] {}); - for (String indexName : indices) { - IndexService indexService = indicesService.indexServiceSafe(indexName); - for (int shardId : indexService.shardIds()) { - indexShards.add(indexService.shard(shardId)); - } - } - - return indexShards; - } } } From 512dbc292d8f36af08d95c1bfee563de593e788e Mon Sep 17 00:00:00 2001 From: Xiao Yu Date: Fri, 8 Aug 2014 18:03:13 -0400 Subject: [PATCH 45/48] Add Warmer and Percolate stats --- .../service/statsd/StatsdReporterIndexStats.java | 8 ++++++++ .../service/statsd/StatsdReporterIndices.java | 4 ++-- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/elasticsearch/service/statsd/StatsdReporterIndexStats.java b/src/main/java/org/elasticsearch/service/statsd/StatsdReporterIndexStats.java index 5796335..fdf08b1 100644 --- a/src/main/java/org/elasticsearch/service/statsd/StatsdReporterIndexStats.java +++ b/src/main/java/org/elasticsearch/service/statsd/StatsdReporterIndexStats.java @@ -19,6 +19,7 @@ import org.elasticsearch.index.percolator.stats.PercolateStats; import org.elasticsearch.search.suggest.completion.CompletionStats; import org.elasticsearch.index.engine.SegmentsStats; +import org.elasticsearch.index.warmer.WarmerStats; public abstract class StatsdReporterIndexStats extends StatsdReporter { @@ -84,6 +85,13 @@ protected void sendFlushStats(String name, FlushStats flushStats) { this.sendGauge(name, "total_time_in_millis", flushStats.getTotalTimeInMillis()); } + protected void sendWarmerStats(String name, WarmerStats warmerStats) { + if (null == warmerStats) return; + this.sendGauge(name, "current", warmerStats.current()); + this.sendGauge(name, "total", warmerStats.total()); + this.sendGauge(name, "total_time_in_millis", warmerStats.totalTimeInMillis()); + } + protected void sendFilterCacheStats(String name, FilterCacheStats filterCacheStats) { if (null == filterCacheStats) return; this.sendGauge(name, "memory_size_in_bytes", filterCacheStats.getMemorySizeInBytes()); diff --git a/src/main/java/org/elasticsearch/service/statsd/StatsdReporterIndices.java b/src/main/java/org/elasticsearch/service/statsd/StatsdReporterIndices.java index 55accd6..7543334 100644 --- a/src/main/java/org/elasticsearch/service/statsd/StatsdReporterIndices.java +++ b/src/main/java/org/elasticsearch/service/statsd/StatsdReporterIndices.java @@ -59,11 +59,11 @@ private void sendCommonStats(String prefix, CommonStats stats) { this.sendMergeStats(prefix + ".merges", stats.getMerge()); this.sendRefreshStats(prefix + ".refresh", stats.getRefresh()); this.sendFlushStats(prefix + ".flush", stats.getFlush()); - //TODO: getWarmer + this.sendWarmerStats(prefix + ".warmer", stats.getWarmer()); this.sendFilterCacheStats(prefix + ".filter_cache", stats.getFilterCache()); this.sendIdCacheStats(prefix + ".id_cache", stats.getIdCache()); this.sendFielddataCacheStats(prefix + ".fielddata", stats.getFieldData()); - //TODO: getPercolate + this.sendPercolateStats(prefix + ".percolate", stats.getPercolate()); this.sendCompletionStats(prefix + ".completion", stats.getCompletion()); this.sendSegmentsStats(prefix + ".segments", stats.getSegments()); //TODO: getTranslog From de7dbad8a681507aefa40ad7b6bf4f2245b58fba Mon Sep 17 00:00:00 2001 From: Xiao Yu Date: Fri, 8 Aug 2014 18:09:23 -0400 Subject: [PATCH 46/48] Version Bump --- README.md | 4 ++-- pom.xml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 19953ca..9971722 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ The data sent to the StatsD server tries to be roughly equivalent to the [Indice To install a prepackaged plugin use the following command: ``` -bin/plugin -install statsd -url https://github.com/Automattic/elasticsearch-statsd-plugin/releases/download/v0.3.1/elasticsearch-statsd-0.3.1.zip +bin/plugin -install statsd -url https://github.com/Automattic/elasticsearch-statsd-plugin/releases/download/v0.3.2/elasticsearch-statsd-0.3.2.zip ``` You can also build your own by doing the following: @@ -20,7 +20,7 @@ You can also build your own by doing the following: git clone http://github.com/Automattic/elasticsearch-statsd-plugin.git cd elasticsearch-statsd-plugin mvn package -bin/plugin -install statsd -url file:///absolute/path/to/current/dir/target/releases/elasticsearch-statsd-0.3.1.zip +bin/plugin -install statsd -url file:///absolute/path/to/current/dir/target/releases/elasticsearch-statsd-0.3.2.zip ``` diff --git a/pom.xml b/pom.xml index 811ac16..a5390d2 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ 4.0.0 de.spinscale.elasticsearch elasticsearch-statsd - 0.3.1 + 0.3.2 jar StatsD monitoring plugin for Elasticsearch https://github.com/Automattic/elasticsearch-statsd-plugin/ From 1d3ec423bdd10595ec44f795fa94f30fc8d601db Mon Sep 17 00:00:00 2001 From: Xiao Yu Date: Wed, 20 Aug 2014 11:44:55 -0400 Subject: [PATCH 47/48] Send fractional values for load average to StatsD The newest version (v3.0.2) of java-statsd-client now supports doubles for guage value and gauge deltas, update ES StatsD Plugin to send doubles instead of longs for load averages. This resolves #1 --- pom.xml | 2 +- .../org/elasticsearch/service/statsd/StatsdReporter.java | 4 ++++ .../service/statsd/StatsdReporterNodeStats.java | 8 +++----- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/pom.xml b/pom.xml index a5390d2..ab00b5b 100644 --- a/pom.xml +++ b/pom.xml @@ -21,7 +21,7 @@ com.timgroup java-statsd-client - 3.0.1 + 3.0.2 diff --git a/src/main/java/org/elasticsearch/service/statsd/StatsdReporter.java b/src/main/java/org/elasticsearch/service/statsd/StatsdReporter.java index cba1e83..5afa50e 100644 --- a/src/main/java/org/elasticsearch/service/statsd/StatsdReporter.java +++ b/src/main/java/org/elasticsearch/service/statsd/StatsdReporter.java @@ -22,6 +22,10 @@ protected void sendGauge(String name, String valueName, long value) { this.statsdClient.gauge(this.join(name, valueName), value); } + protected void sendGauge(String name, String valueName, double value) { + this.statsdClient.gauge(this.join(name, valueName), value); + } + protected void sendCount(String name, String valueName, long value) { this.statsdClient.count(this.join(name, valueName), value); } diff --git a/src/main/java/org/elasticsearch/service/statsd/StatsdReporterNodeStats.java b/src/main/java/org/elasticsearch/service/statsd/StatsdReporterNodeStats.java index 141240e..62668b1 100644 --- a/src/main/java/org/elasticsearch/service/statsd/StatsdReporterNodeStats.java +++ b/src/main/java/org/elasticsearch/service/statsd/StatsdReporterNodeStats.java @@ -89,13 +89,11 @@ private void sendNodeProcessStats(ProcessStats processStats) { private void sendNodeOsStats(OsStats osStats) { String prefix = this.getPrefix("os"); - // Java client does not support doubles yet :( - // https://github.com/tim-group/java-statsd-client/issues/19 double[] loadAverage = osStats.getLoadAverage(); if (loadAverage.length > 0) { - this.sendGauge(prefix + ".load_average", "1m", (long) loadAverage[0]); - this.sendGauge(prefix + ".load_average", "5m", (long) loadAverage[1]); - this.sendGauge(prefix + ".load_average", "15m", (long) loadAverage[2]); + this.sendGauge(prefix + ".load_average", "1m", loadAverage[0]); + this.sendGauge(prefix + ".load_average", "5m", loadAverage[1]); + this.sendGauge(prefix + ".load_average", "15m", loadAverage[2]); } if (osStats.cpu() != null) { From 1f8c8e6eedbd20b1db481fe21e30760466348e53 Mon Sep 17 00:00:00 2001 From: Xiao Yu Date: Wed, 20 Aug 2014 11:51:21 -0400 Subject: [PATCH 48/48] Version bump to v0.3.3 --- README.md | 4 ++-- pom.xml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 9971722..a52f2af 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ The data sent to the StatsD server tries to be roughly equivalent to the [Indice To install a prepackaged plugin use the following command: ``` -bin/plugin -install statsd -url https://github.com/Automattic/elasticsearch-statsd-plugin/releases/download/v0.3.2/elasticsearch-statsd-0.3.2.zip +bin/plugin -install statsd -url https://github.com/Automattic/elasticsearch-statsd-plugin/releases/download/v0.3.3/elasticsearch-statsd-0.3.3.zip ``` You can also build your own by doing the following: @@ -20,7 +20,7 @@ You can also build your own by doing the following: git clone http://github.com/Automattic/elasticsearch-statsd-plugin.git cd elasticsearch-statsd-plugin mvn package -bin/plugin -install statsd -url file:///absolute/path/to/current/dir/target/releases/elasticsearch-statsd-0.3.2.zip +bin/plugin -install statsd -url file:///absolute/path/to/current/dir/target/releases/elasticsearch-statsd-0.3.3.zip ``` diff --git a/pom.xml b/pom.xml index ab00b5b..34ce60c 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ 4.0.0 de.spinscale.elasticsearch elasticsearch-statsd - 0.3.2 + 0.3.3 jar StatsD monitoring plugin for Elasticsearch https://github.com/Automattic/elasticsearch-statsd-plugin/