diff options
Diffstat (limited to '')
19 files changed, 7806 insertions, 0 deletions
diff --git a/src/go/collectors/go.d.plugin/modules/elasticsearch/README.md b/src/go/collectors/go.d.plugin/modules/elasticsearch/README.md new file mode 120000 index 000000000..8951ff7b2 --- /dev/null +++ b/src/go/collectors/go.d.plugin/modules/elasticsearch/README.md @@ -0,0 +1 @@ +integrations/elasticsearch.md
\ No newline at end of file diff --git a/src/go/collectors/go.d.plugin/modules/elasticsearch/charts.go b/src/go/collectors/go.d.plugin/modules/elasticsearch/charts.go new file mode 100644 index 000000000..1087ef65f --- /dev/null +++ b/src/go/collectors/go.d.plugin/modules/elasticsearch/charts.go @@ -0,0 +1,845 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +package elasticsearch + +import ( + "fmt" + "strings" + + "github.com/netdata/netdata/go/go.d.plugin/agent/module" +) + +const ( + prioNodeIndicesIndexingOps = module.Priority + iota + prioNodeIndicesIndexingOpsCurrent + prioNodeIndicesIndexingOpsTime + prioNodeIndicesSearchOps + prioNodeIndicesSearchOpsCurrent + prioNodeIndicesSearchOpsTime + prioNodeIndicesRefreshOps + prioNodeIndicesRefreshOpsTime + prioNodeIndicesFlushOps + prioNodeIndicesFlushOpsTime + prioNodeIndicesFieldDataMemoryUsage + prioNodeIndicesFieldDataEvictions + prioNodeIndicesSegmentsCount + prioNodeIndicesSegmentsMemoryUsageTotal + prioNodeIndicesSegmentsMemoryUsage + prioNodeIndicesTransLogOps + prioNodeIndexTransLogSize + prioNodeFileDescriptors + prioNodeJVMMemHeap + prioNodeJVMMemHeapBytes + prioNodeJVMBufferPoolsCount + prioNodeJVMBufferPoolDirectMemory + prioNodeJVMBufferPoolMappedMemory + prioNodeJVMGCCount + prioNodeJVMGCTime + prioNodeThreadPoolQueued + prioNodeThreadPoolRejected + prioNodeClusterCommunicationPackets + prioNodeClusterCommunication + prioNodeHTTPConnections + prioNodeBreakersTrips + + prioClusterStatus + prioClusterNodesCount + prioClusterShardsCount + prioClusterPendingTasks + prioClusterInFlightFetchesCount + + prioClusterIndicesCount + prioClusterIndicesShardsCount + prioClusterIndicesDocsCount + prioClusterIndicesStoreSize + prioClusterIndicesQueryCache + prioClusterNodesByRoleCount + + prioNodeIndexHealth + prioNodeIndexShardsCount + prioNodeIndexDocsCount + prioNodeIndexStoreSize +) + +var nodeChartsTmpl = module.Charts{ + nodeIndicesIndexingOpsChartTmpl.Copy(), + nodeIndicesIndexingOpsCurrentChartTmpl.Copy(), + nodeIndicesIndexingOpsTimeChartTmpl.Copy(), + + nodeIndicesSearchOpsChartTmpl.Copy(), + nodeIndicesSearchOpsCurrentChartTmpl.Copy(), + nodeIndicesSearchOpsTimeChartTmpl.Copy(), + + nodeIndicesRefreshOpsChartTmpl.Copy(), + nodeIndicesRefreshOpsTimeChartTmpl.Copy(), + + nodeIndicesFlushOpsChartTmpl.Copy(), + nodeIndicesFlushOpsTimeChartTmpl.Copy(), + + nodeIndicesFieldDataMemoryUsageChartTmpl.Copy(), + nodeIndicesFieldDataEvictionsChartTmpl.Copy(), + + nodeIndicesSegmentsCountChartTmpl.Copy(), + nodeIndicesSegmentsMemoryUsageTotalChartTmpl.Copy(), + nodeIndicesSegmentsMemoryUsageChartTmpl.Copy(), + + nodeIndicesTransLogOpsChartTmpl.Copy(), + nodeIndexTransLogSizeChartTmpl.Copy(), + + nodeFileDescriptorsChartTmpl.Copy(), + + nodeJVMMemHeapChartTmpl.Copy(), + nodeJVMMemHeapBytesChartTmpl.Copy(), + nodeJVMBufferPoolsCountChartTmpl.Copy(), + nodeJVMBufferPoolDirectMemoryChartTmpl.Copy(), + nodeJVMBufferPoolMappedMemoryChartTmpl.Copy(), + nodeJVMGCCountChartTmpl.Copy(), + nodeJVMGCTimeChartTmpl.Copy(), + + nodeThreadPoolQueuedChartTmpl.Copy(), + nodeThreadPoolRejectedChartTmpl.Copy(), + + nodeClusterCommunicationPacketsChartTmpl.Copy(), + nodeClusterCommunicationChartTmpl.Copy(), + + nodeHTTPConnectionsChartTmpl.Copy(), + + nodeBreakersTripsChartTmpl.Copy(), +} + +var ( + nodeIndicesIndexingOpsChartTmpl = module.Chart{ + ID: "node_%s_cluster_%s_indices_indexing_operations", + Title: "Indexing Operations", + Units: "operations/s", + Fam: "indices indexing", + Ctx: "elasticsearch.node_indices_indexing", + Priority: prioNodeIndicesIndexingOps, + Dims: module.Dims{ + {ID: "node_%s_indices_indexing_index_total", Name: "index", Algo: module.Incremental}, + }, + } + nodeIndicesIndexingOpsCurrentChartTmpl = module.Chart{ + ID: "node_%s_cluster_%s_indices_indexing_operations_current", + Title: "Indexing Operations Current", + Units: "operations", + Fam: "indices indexing", + Ctx: "elasticsearch.node_indices_indexing_current", + Priority: prioNodeIndicesIndexingOpsCurrent, + Dims: module.Dims{ + {ID: "node_%s_indices_indexing_index_current", Name: "index"}, + }, + } + nodeIndicesIndexingOpsTimeChartTmpl = module.Chart{ + ID: "node_%s_cluster_%s_indices_indexing_operations_time", + Title: "Time Spent On Indexing Operations", + Units: "milliseconds", + Fam: "indices indexing", + Ctx: "elasticsearch.node_indices_indexing_time", + Priority: prioNodeIndicesIndexingOpsTime, + Dims: module.Dims{ + {ID: "node_%s_indices_indexing_index_time_in_millis", Name: "index", Algo: module.Incremental}, + }, + } + + nodeIndicesSearchOpsChartTmpl = module.Chart{ + ID: "node_%s_cluster_%s_indices_search_operations", + Title: "Search Operations", + Units: "operations/s", + Fam: "indices search", + Ctx: "elasticsearch.node_indices_search", + Type: module.Stacked, + Priority: prioNodeIndicesSearchOps, + Dims: module.Dims{ + {ID: "node_%s_indices_search_query_total", Name: "queries", Algo: module.Incremental}, + {ID: "node_%s_indices_search_fetch_total", Name: "fetches", Algo: module.Incremental}, + }, + } + nodeIndicesSearchOpsCurrentChartTmpl = module.Chart{ + ID: "node_%s_cluster_%s_indices_search_operations_current", + Title: "Search Operations Current", + Units: "operations", + Fam: "indices search", + Ctx: "elasticsearch.node_indices_search_current", + Type: module.Stacked, + Priority: prioNodeIndicesSearchOpsCurrent, + Dims: module.Dims{ + {ID: "node_%s_indices_search_query_current", Name: "queries"}, + {ID: "node_%s_indices_search_fetch_current", Name: "fetches"}, + }, + } + nodeIndicesSearchOpsTimeChartTmpl = module.Chart{ + ID: "node_%s_cluster_%s_indices_search_operations_time", + Title: "Time Spent On Search Operations", + Units: "milliseconds", + Fam: "indices search", + Ctx: "elasticsearch.node_indices_search_time", + Type: module.Stacked, + Priority: prioNodeIndicesSearchOpsTime, + Dims: module.Dims{ + {ID: "node_%s_indices_search_query_time_in_millis", Name: "query", Algo: module.Incremental}, + {ID: "node_%s_indices_search_fetch_time_in_millis", Name: "fetch", Algo: module.Incremental}, + }, + } + + nodeIndicesRefreshOpsChartTmpl = module.Chart{ + ID: "node_%s_cluster_%s_indices_refresh_operations", + Title: "Refresh Operations", + Units: "operations/s", + Fam: "indices refresh", + Ctx: "elasticsearch.node_indices_refresh", + Priority: prioNodeIndicesRefreshOps, + Dims: module.Dims{ + {ID: "node_%s_indices_refresh_total", Name: "refresh", Algo: module.Incremental}, + }, + } + nodeIndicesRefreshOpsTimeChartTmpl = module.Chart{ + ID: "node_%s_cluster_%s_indices_refresh_operations_time", + Title: "Time Spent On Refresh Operations", + Units: "milliseconds", + Fam: "indices refresh", + Ctx: "elasticsearch.node_indices_refresh_time", + Priority: prioNodeIndicesRefreshOpsTime, + Dims: module.Dims{ + {ID: "node_%s_indices_refresh_total_time_in_millis", Name: "refresh", Algo: module.Incremental}, + }, + } + + nodeIndicesFlushOpsChartTmpl = module.Chart{ + ID: "node_%s_cluster_%s_indices_flush_operations", + Title: "Flush Operations", + Units: "operations/s", + Fam: "indices flush", + Ctx: "elasticsearch.node_indices_flush", + Priority: prioNodeIndicesFlushOps, + Dims: module.Dims{ + {ID: "node_%s_indices_flush_total", Name: "flush", Algo: module.Incremental}, + }, + } + nodeIndicesFlushOpsTimeChartTmpl = module.Chart{ + ID: "node_%s_cluster_%s_indices_flush_operations_time", + Title: "Time Spent On Flush Operations", + Units: "milliseconds", + Fam: "indices flush", + Ctx: "elasticsearch.node_indices_flush_time", + Priority: prioNodeIndicesFlushOpsTime, + Dims: module.Dims{ + {ID: "node_%s_indices_flush_total_time_in_millis", Name: "flush", Algo: module.Incremental}, + }, + } + + nodeIndicesFieldDataMemoryUsageChartTmpl = module.Chart{ + ID: "node_%s_cluster_%s_indices_fielddata_memory_usage", + Title: "Fielddata Cache Memory Usage", + Units: "bytes", + Fam: "indices fielddata", + Ctx: "elasticsearch.node_indices_fielddata_memory_usage", + Type: module.Area, + Priority: prioNodeIndicesFieldDataMemoryUsage, + Dims: module.Dims{ + {ID: "node_%s_indices_fielddata_memory_size_in_bytes", Name: "used"}, + }, + } + nodeIndicesFieldDataEvictionsChartTmpl = module.Chart{ + ID: "node_%s_cluster_%s_indices_fielddata_evictions", + Title: "Fielddata Evictions", + Units: "operations/s", + Fam: "indices fielddata", + Ctx: "elasticsearch.node_indices_fielddata_evictions", + Priority: prioNodeIndicesFieldDataEvictions, + Dims: module.Dims{ + {ID: "node_%s_indices_fielddata_evictions", Name: "evictions", Algo: module.Incremental}, + }, + } + + nodeIndicesSegmentsCountChartTmpl = module.Chart{ + ID: "node_%s_cluster_%s_indices_segments_count", + Title: "Segments Count", + Units: "segments", + Fam: "indices segments", + Ctx: "elasticsearch.node_indices_segments_count", + Priority: prioNodeIndicesSegmentsCount, + Dims: module.Dims{ + {ID: "node_%s_indices_segments_count", Name: "segments"}, + }, + } + nodeIndicesSegmentsMemoryUsageTotalChartTmpl = module.Chart{ + ID: "node_%s_cluster_%s_indices_segments_memory_usage_total", + Title: "Segments Memory Usage Total", + Units: "bytes", + Fam: "indices segments", + Ctx: "elasticsearch.node_indices_segments_memory_usage_total", + Priority: prioNodeIndicesSegmentsMemoryUsageTotal, + Dims: module.Dims{ + {ID: "node_%s_indices_segments_memory_in_bytes", Name: "used"}, + }, + } + nodeIndicesSegmentsMemoryUsageChartTmpl = module.Chart{ + ID: "node_%s_cluster_%s_indices_segments_memory_usage", + Title: "Segments Memory Usage", + Units: "bytes", + Fam: "indices segments", + Ctx: "elasticsearch.node_indices_segments_memory_usage", + Type: module.Stacked, + Priority: prioNodeIndicesSegmentsMemoryUsage, + Dims: module.Dims{ + {ID: "node_%s_indices_segments_terms_memory_in_bytes", Name: "terms"}, + {ID: "node_%s_indices_segments_stored_fields_memory_in_bytes", Name: "stored_fields"}, + {ID: "node_%s_indices_segments_term_vectors_memory_in_bytes", Name: "term_vectors"}, + {ID: "node_%s_indices_segments_norms_memory_in_bytes", Name: "norms"}, + {ID: "node_%s_indices_segments_points_memory_in_bytes", Name: "points"}, + {ID: "node_%s_indices_segments_doc_values_memory_in_bytes", Name: "doc_values"}, + {ID: "node_%s_indices_segments_index_writer_memory_in_bytes", Name: "index_writer"}, + {ID: "node_%s_indices_segments_version_map_memory_in_bytes", Name: "version_map"}, + {ID: "node_%s_indices_segments_fixed_bit_set_memory_in_bytes", Name: "fixed_bit_set"}, + }, + } + + nodeIndicesTransLogOpsChartTmpl = module.Chart{ + ID: "node_%s_cluster_%s_indices_translog_operations", + Title: "Translog Operations", + Units: "operations", + Fam: "indices translog", + Ctx: "elasticsearch.node_indices_translog_operations", + Type: module.Area, + Priority: prioNodeIndicesTransLogOps, + Dims: module.Dims{ + {ID: "node_%s_indices_translog_operations", Name: "total"}, + {ID: "node_%s_indices_translog_uncommitted_operations", Name: "uncommitted"}, + }, + } + nodeIndexTransLogSizeChartTmpl = module.Chart{ + ID: "node_%s_cluster_%s_index_translog_size", + Title: "Translog Size", + Units: "bytes", + Fam: "indices translog", + Ctx: "elasticsearch.node_indices_translog_size", + Type: module.Area, + Priority: prioNodeIndexTransLogSize, + Dims: module.Dims{ + {ID: "node_%s_indices_translog_size_in_bytes", Name: "total"}, + {ID: "node_%s_indices_translog_uncommitted_size_in_bytes", Name: "uncommitted"}, + }, + } + + nodeFileDescriptorsChartTmpl = module.Chart{ + ID: "node_%s_cluster_%s_file_descriptors", + Title: "Process File Descriptors", + Units: "fd", + Fam: "process", + Ctx: "elasticsearch.node_file_descriptors", + Priority: prioNodeFileDescriptors, + Dims: module.Dims{ + {ID: "node_%s_process_open_file_descriptors", Name: "open"}, + }, + } + + nodeJVMMemHeapChartTmpl = module.Chart{ + ID: "node_%s_cluster_%s_jvm_mem_heap", + Title: "JVM Heap Percentage Currently in Use", + Units: "percentage", + Fam: "jvm", + Ctx: "elasticsearch.node_jvm_heap", + Type: module.Area, + Priority: prioNodeJVMMemHeap, + Dims: module.Dims{ + {ID: "node_%s_jvm_mem_heap_used_percent", Name: "inuse"}, + }, + } + nodeJVMMemHeapBytesChartTmpl = module.Chart{ + ID: "node_%s_cluster_%s_jvm_mem_heap_bytes", + Title: "JVM Heap Commit And Usage", + Units: "bytes", + Fam: "jvm", + Ctx: "elasticsearch.node_jvm_heap_bytes", + Type: module.Area, + Priority: prioNodeJVMMemHeapBytes, + Dims: module.Dims{ + {ID: "node_%s_jvm_mem_heap_committed_in_bytes", Name: "committed"}, + {ID: "node_%s_jvm_mem_heap_used_in_bytes", Name: "used"}, + }, + } + nodeJVMBufferPoolsCountChartTmpl = module.Chart{ + ID: "node_%s_cluster_%s_jvm_buffer_pools_count", + Title: "JVM Buffer Pools Count", + Units: "pools", + Fam: "jvm", + Ctx: "elasticsearch.node_jvm_buffer_pools_count", + Priority: prioNodeJVMBufferPoolsCount, + Dims: module.Dims{ + {ID: "node_%s_jvm_buffer_pools_direct_count", Name: "direct"}, + {ID: "node_%s_jvm_buffer_pools_mapped_count", Name: "mapped"}, + }, + } + nodeJVMBufferPoolDirectMemoryChartTmpl = module.Chart{ + ID: "node_%s_cluster_%s_jvm_buffer_pool_direct_memory", + Title: "JVM Buffer Pool Direct Memory", + Units: "bytes", + Fam: "jvm", + Ctx: "elasticsearch.node_jvm_buffer_pool_direct_memory", + Type: module.Area, + Priority: prioNodeJVMBufferPoolDirectMemory, + Dims: module.Dims{ + {ID: "node_%s_jvm_buffer_pools_direct_total_capacity_in_bytes", Name: "total"}, + {ID: "node_%s_jvm_buffer_pools_direct_used_in_bytes", Name: "used"}, + }, + } + nodeJVMBufferPoolMappedMemoryChartTmpl = module.Chart{ + ID: "node_%s_cluster_%s_jvm_buffer_pool_mapped_memory", + Title: "JVM Buffer Pool Mapped Memory", + Units: "bytes", + Fam: "jvm", + Ctx: "elasticsearch.node_jvm_buffer_pool_mapped_memory", + Type: module.Area, + Priority: prioNodeJVMBufferPoolMappedMemory, + Dims: module.Dims{ + {ID: "node_%s_jvm_buffer_pools_mapped_total_capacity_in_bytes", Name: "total"}, + {ID: "node_%s_jvm_buffer_pools_mapped_used_in_bytes", Name: "used"}, + }, + } + nodeJVMGCCountChartTmpl = module.Chart{ + ID: "node_%s_cluster_%s_jvm_gc_count", + Title: "JVM Garbage Collections", + Units: "gc/s", + Fam: "jvm", + Ctx: "elasticsearch.node_jvm_gc_count", + Type: module.Stacked, + Priority: prioNodeJVMGCCount, + Dims: module.Dims{ + {ID: "node_%s_jvm_gc_collectors_young_collection_count", Name: "young", Algo: module.Incremental}, + {ID: "node_%s_jvm_gc_collectors_old_collection_count", Name: "old", Algo: module.Incremental}, + }, + } + nodeJVMGCTimeChartTmpl = module.Chart{ + ID: "node_%s_cluster_%s_jvm_gc_time", + Title: "JVM Time Spent On Garbage Collections", + Units: "milliseconds", + Fam: "jvm", + Ctx: "elasticsearch.node_jvm_gc_time", + Type: module.Stacked, + Priority: prioNodeJVMGCTime, + Dims: module.Dims{ + {ID: "node_%s_jvm_gc_collectors_young_collection_time_in_millis", Name: "young", Algo: module.Incremental}, + {ID: "node_%s_jvm_gc_collectors_old_collection_time_in_millis", Name: "old", Algo: module.Incremental}, + }, + } + + nodeThreadPoolQueuedChartTmpl = module.Chart{ + ID: "node_%s_cluster_%s_thread_pool_queued", + Title: "Thread Pool Queued Threads Count", + Units: "threads", + Fam: "thread pool", + Ctx: "elasticsearch.node_thread_pool_queued", + Type: module.Stacked, + Priority: prioNodeThreadPoolQueued, + Dims: module.Dims{ + {ID: "node_%s_thread_pool_generic_queue", Name: "generic"}, + {ID: "node_%s_thread_pool_search_queue", Name: "search"}, + {ID: "node_%s_thread_pool_search_throttled_queue", Name: "search_throttled"}, + {ID: "node_%s_thread_pool_get_queue", Name: "get"}, + {ID: "node_%s_thread_pool_analyze_queue", Name: "analyze"}, + {ID: "node_%s_thread_pool_write_queue", Name: "write"}, + {ID: "node_%s_thread_pool_snapshot_queue", Name: "snapshot"}, + {ID: "node_%s_thread_pool_warmer_queue", Name: "warmer"}, + {ID: "node_%s_thread_pool_refresh_queue", Name: "refresh"}, + {ID: "node_%s_thread_pool_listener_queue", Name: "listener"}, + {ID: "node_%s_thread_pool_fetch_shard_started_queue", Name: "fetch_shard_started"}, + {ID: "node_%s_thread_pool_fetch_shard_store_queue", Name: "fetch_shard_store"}, + {ID: "node_%s_thread_pool_flush_queue", Name: "flush"}, + {ID: "node_%s_thread_pool_force_merge_queue", Name: "force_merge"}, + {ID: "node_%s_thread_pool_management_queue", Name: "management"}, + }, + } + nodeThreadPoolRejectedChartTmpl = module.Chart{ + ID: "node_%s_cluster_%s_thread_pool_rejected", + Title: "Thread Pool Rejected Threads Count", + Units: "threads", + Fam: "thread pool", + Ctx: "elasticsearch.node_thread_pool_rejected", + Type: module.Stacked, + Priority: prioNodeThreadPoolRejected, + Dims: module.Dims{ + {ID: "node_%s_thread_pool_generic_rejected", Name: "generic"}, + {ID: "node_%s_thread_pool_search_rejected", Name: "search"}, + {ID: "node_%s_thread_pool_search_throttled_rejected", Name: "search_throttled"}, + {ID: "node_%s_thread_pool_get_rejected", Name: "get"}, + {ID: "node_%s_thread_pool_analyze_rejected", Name: "analyze"}, + {ID: "node_%s_thread_pool_write_rejected", Name: "write"}, + {ID: "node_%s_thread_pool_snapshot_rejected", Name: "snapshot"}, + {ID: "node_%s_thread_pool_warmer_rejected", Name: "warmer"}, + {ID: "node_%s_thread_pool_refresh_rejected", Name: "refresh"}, + {ID: "node_%s_thread_pool_listener_rejected", Name: "listener"}, + {ID: "node_%s_thread_pool_fetch_shard_started_rejected", Name: "fetch_shard_started"}, + {ID: "node_%s_thread_pool_fetch_shard_store_rejected", Name: "fetch_shard_store"}, + {ID: "node_%s_thread_pool_flush_rejected", Name: "flush"}, + {ID: "node_%s_thread_pool_force_merge_rejected", Name: "force_merge"}, + {ID: "node_%s_thread_pool_management_rejected", Name: "management"}, + }, + } + + nodeClusterCommunicationPacketsChartTmpl = module.Chart{ + ID: "node_%s_cluster_%s_cluster_communication_packets", + Title: "Node Cluster Communication", + Units: "pps", + Fam: "transport", + Ctx: "elasticsearch.node_cluster_communication_packets", + Priority: prioNodeClusterCommunicationPackets, + Dims: module.Dims{ + {ID: "node_%s_transport_rx_count", Name: "received", Algo: module.Incremental}, + {ID: "node_%s_transport_tx_count", Name: "sent", Mul: -1, Algo: module.Incremental}, + }, + } + nodeClusterCommunicationChartTmpl = module.Chart{ + ID: "node_%s_cluster_%s_cluster_communication_traffic", + Title: "Cluster Communication Bandwidth", + Units: "bytes/s", + Fam: "transport", + Ctx: "elasticsearch.node_cluster_communication_traffic", + Priority: prioNodeClusterCommunication, + Dims: module.Dims{ + {ID: "node_%s_transport_rx_size_in_bytes", Name: "received", Algo: module.Incremental}, + {ID: "node_%s_transport_tx_size_in_bytes", Name: "sent", Mul: -1, Algo: module.Incremental}, + }, + } + + nodeHTTPConnectionsChartTmpl = module.Chart{ + ID: "node_%s_cluster_%s_http_connections", + Title: "HTTP Connections", + Units: "connections", + Fam: "http", + Ctx: "elasticsearch.node_http_connections", + Priority: prioNodeHTTPConnections, + Dims: module.Dims{ + {ID: "node_%s_http_current_open", Name: "open"}, + }, + } + + nodeBreakersTripsChartTmpl = module.Chart{ + ID: "node_%s_cluster_%s_breakers_trips", + Title: "Circuit Breaker Trips Count", + Units: "trips/s", + Fam: "circuit breakers", + Ctx: "elasticsearch.node_breakers_trips", + Type: module.Stacked, + Priority: prioNodeBreakersTrips, + Dims: module.Dims{ + {ID: "node_%s_breakers_request_tripped", Name: "requests", Algo: module.Incremental}, + {ID: "node_%s_breakers_fielddata_tripped", Name: "fielddata", Algo: module.Incremental}, + {ID: "node_%s_breakers_in_flight_requests_tripped", Name: "in_flight_requests", Algo: module.Incremental}, + {ID: "node_%s_breakers_model_inference_tripped", Name: "model_inference", Algo: module.Incremental}, + {ID: "node_%s_breakers_accounting_tripped", Name: "accounting", Algo: module.Incremental}, + {ID: "node_%s_breakers_parent_tripped", Name: "parent", Algo: module.Incremental}, + }, + } +) + +var clusterHealthChartsTmpl = module.Charts{ + clusterStatusChartTmpl.Copy(), + clusterNodesCountChartTmpl.Copy(), + clusterShardsCountChartTmpl.Copy(), + clusterPendingTasksChartTmpl.Copy(), + clusterInFlightFetchesCountChartTmpl.Copy(), +} + +var ( + clusterStatusChartTmpl = module.Chart{ + ID: "cluster_%s_status", + Title: "Cluster Status", + Units: "status", + Fam: "cluster health", + Ctx: "elasticsearch.cluster_health_status", + Priority: prioClusterStatus, + Dims: module.Dims{ + {ID: "cluster_status_green", Name: "green"}, + {ID: "cluster_status_red", Name: "red"}, + {ID: "cluster_status_yellow", Name: "yellow"}, + }, + } + clusterNodesCountChartTmpl = module.Chart{ + ID: "cluster_%s_number_of_nodes", + Title: "Cluster Nodes Count", + Units: "nodes", + Fam: "cluster health", + Ctx: "elasticsearch.cluster_number_of_nodes", + Priority: prioClusterNodesCount, + Dims: module.Dims{ + {ID: "cluster_number_of_nodes", Name: "nodes"}, + {ID: "cluster_number_of_data_nodes", Name: "data_nodes"}, + }, + } + clusterShardsCountChartTmpl = module.Chart{ + ID: "cluster_%s_shards_count", + Title: "Cluster Shards Count", + Units: "shards", + Fam: "cluster health", + Ctx: "elasticsearch.cluster_shards_count", + Priority: prioClusterShardsCount, + Dims: module.Dims{ + {ID: "cluster_active_primary_shards", Name: "active_primary"}, + {ID: "cluster_active_shards", Name: "active"}, + {ID: "cluster_relocating_shards", Name: "relocating"}, + {ID: "cluster_initializing_shards", Name: "initializing"}, + {ID: "cluster_unassigned_shards", Name: "unassigned"}, + {ID: "cluster_delayed_unassigned_shards", Name: "delayed_unassigned"}, + }, + } + clusterPendingTasksChartTmpl = module.Chart{ + ID: "cluster_%s_pending_tasks", + Title: "Cluster Pending Tasks", + Units: "tasks", + Fam: "cluster health", + Ctx: "elasticsearch.cluster_pending_tasks", + Priority: prioClusterPendingTasks, + Dims: module.Dims{ + {ID: "cluster_number_of_pending_tasks", Name: "pending"}, + }, + } + clusterInFlightFetchesCountChartTmpl = module.Chart{ + ID: "cluster_%s_number_of_in_flight_fetch", + Title: "Cluster Unfinished Fetches", + Units: "fetches", + Fam: "cluster health", + Ctx: "elasticsearch.cluster_number_of_in_flight_fetch", + Priority: prioClusterInFlightFetchesCount, + Dims: module.Dims{ + {ID: "cluster_number_of_in_flight_fetch", Name: "in_flight_fetch"}, + }, + } +) + +var clusterStatsChartsTmpl = module.Charts{ + clusterIndicesCountChartTmpl.Copy(), + clusterIndicesShardsCountChartTmpl.Copy(), + clusterIndicesDocsCountChartTmpl.Copy(), + clusterIndicesStoreSizeChartTmpl.Copy(), + clusterIndicesQueryCacheChartTmpl.Copy(), + clusterNodesByRoleCountChartTmpl.Copy(), +} + +var ( + clusterIndicesCountChartTmpl = module.Chart{ + ID: "cluster_%s_indices_count", + Title: "Cluster Indices Count", + Units: "indices", + Fam: "cluster stats", + Ctx: "elasticsearch.cluster_indices_count", + Priority: prioClusterIndicesCount, + Dims: module.Dims{ + {ID: "cluster_indices_count", Name: "indices"}, + }, + } + clusterIndicesShardsCountChartTmpl = module.Chart{ + ID: "cluster_%s_indices_shards_count", + Title: "Cluster Indices Shards Count", + Units: "shards", + Fam: "cluster stats", + Ctx: "elasticsearch.cluster_indices_shards_count", + Priority: prioClusterIndicesShardsCount, + Dims: module.Dims{ + {ID: "cluster_indices_shards_total", Name: "total"}, + {ID: "cluster_indices_shards_primaries", Name: "primaries"}, + {ID: "cluster_indices_shards_replication", Name: "replication"}, + }, + } + clusterIndicesDocsCountChartTmpl = module.Chart{ + ID: "cluster_%s_indices_docs_count", + Title: "Cluster Indices Docs Count", + Units: "docs", + Fam: "cluster stats", + Ctx: "elasticsearch.cluster_indices_docs_count", + Priority: prioClusterIndicesDocsCount, + Dims: module.Dims{ + {ID: "cluster_indices_docs_count", Name: "docs"}, + }, + } + clusterIndicesStoreSizeChartTmpl = module.Chart{ + ID: "cluster_%s_indices_store_size", + Title: "Cluster Indices Store Size", + Units: "bytes", + Fam: "cluster stats", + Ctx: "elasticsearch.cluster_indices_store_size", + Priority: prioClusterIndicesStoreSize, + Dims: module.Dims{ + {ID: "cluster_indices_store_size_in_bytes", Name: "size"}, + }, + } + clusterIndicesQueryCacheChartTmpl = module.Chart{ + ID: "cluster_%s_indices_query_cache", + Title: "Cluster Indices Query Cache", + Units: "events/s", + Fam: "cluster stats", + Ctx: "elasticsearch.cluster_indices_query_cache", + Type: module.Stacked, + Priority: prioClusterIndicesQueryCache, + Dims: module.Dims{ + {ID: "cluster_indices_query_cache_hit_count", Name: "hit", Algo: module.Incremental}, + {ID: "cluster_indices_query_cache_miss_count", Name: "miss", Algo: module.Incremental}, + }, + } + clusterNodesByRoleCountChartTmpl = module.Chart{ + ID: "cluster_%s_nodes_by_role_count", + Title: "Cluster Nodes By Role Count", + Units: "nodes", + Fam: "cluster stats", + Ctx: "elasticsearch.cluster_nodes_by_role_count", + Priority: prioClusterNodesByRoleCount, + Dims: module.Dims{ + {ID: "cluster_nodes_count_coordinating_only", Name: "coordinating_only"}, + {ID: "cluster_nodes_count_data", Name: "data"}, + {ID: "cluster_nodes_count_data_cold", Name: "data_cold"}, + {ID: "cluster_nodes_count_data_content", Name: "data_content"}, + {ID: "cluster_nodes_count_data_frozen", Name: "data_frozen"}, + {ID: "cluster_nodes_count_data_hot", Name: "data_hot"}, + {ID: "cluster_nodes_count_data_warm", Name: "data_warm"}, + {ID: "cluster_nodes_count_ingest", Name: "ingest"}, + {ID: "cluster_nodes_count_master", Name: "master"}, + {ID: "cluster_nodes_count_ml", Name: "ml"}, + {ID: "cluster_nodes_count_remote_cluster_client", Name: "remote_cluster_client"}, + {ID: "cluster_nodes_count_voting_only", Name: "voting_only"}, + }, + } +) + +var nodeIndexChartsTmpl = module.Charts{ + nodeIndexHealthChartTmpl.Copy(), + nodeIndexShardsCountChartTmpl.Copy(), + nodeIndexDocsCountChartTmpl.Copy(), + nodeIndexStoreSizeChartTmpl.Copy(), +} + +var ( + nodeIndexHealthChartTmpl = module.Chart{ + ID: "node_index_%s_cluster_%s_health", + Title: "Index Health", + Units: "status", + Fam: "index stats", + Ctx: "elasticsearch.node_index_health", + Priority: prioNodeIndexHealth, + Dims: module.Dims{ + {ID: "node_index_%s_stats_health_green", Name: "green"}, + {ID: "node_index_%s_stats_health_red", Name: "red"}, + {ID: "node_index_%s_stats_health_yellow", Name: "yellow"}, + }, + } + nodeIndexShardsCountChartTmpl = module.Chart{ + ID: "node_index_%s_cluster_%s_shards_count", + Title: "Index Shards Count", + Units: "shards", + Fam: "index stats", + Ctx: "elasticsearch.node_index_shards_count", + Priority: prioNodeIndexShardsCount, + Dims: module.Dims{ + {ID: "node_index_%s_stats_shards_count", Name: "shards"}, + }, + } + nodeIndexDocsCountChartTmpl = module.Chart{ + ID: "node_index_%s_cluster_%s_docs_count", + Title: "Index Docs Count", + Units: "docs", + Fam: "index stats", + Ctx: "elasticsearch.node_index_docs_count", + Priority: prioNodeIndexDocsCount, + Dims: module.Dims{ + {ID: "node_index_%s_stats_docs_count", Name: "docs"}, + }, + } + nodeIndexStoreSizeChartTmpl = module.Chart{ + ID: "node_index_%s_cluster_%s_store_size", + Title: "Index Store Size", + Units: "bytes", + Fam: "index stats", + Ctx: "elasticsearch.node_index_store_size", + Priority: prioNodeIndexStoreSize, + Dims: module.Dims{ + {ID: "node_index_%s_stats_store_size_in_bytes", Name: "store_size"}, + }, + } +) + +func (es *Elasticsearch) addClusterStatsCharts() { + charts := clusterStatsChartsTmpl.Copy() + + for _, chart := range *charts { + chart.ID = fmt.Sprintf(chart.ID, es.clusterName) + chart.Labels = []module.Label{ + {Key: "cluster_name", Value: es.clusterName}, + } + } + + if err := es.charts.Add(*charts...); err != nil { + es.Warning(err) + } +} + +func (es *Elasticsearch) addClusterHealthCharts() { + charts := clusterHealthChartsTmpl.Copy() + + for _, chart := range *charts { + chart.ID = fmt.Sprintf(chart.ID, es.clusterName) + chart.Labels = []module.Label{ + {Key: "cluster_name", Value: es.clusterName}, + } + } + + if err := es.charts.Add(*charts...); err != nil { + es.Warning(err) + } +} + +func (es *Elasticsearch) addNodeCharts(nodeID string, node *esNodeStats) { + charts := nodeChartsTmpl.Copy() + + for _, chart := range *charts { + chart.ID = fmt.Sprintf(chart.ID, nodeID, es.clusterName) + chart.Labels = []module.Label{ + {Key: "cluster_name", Value: es.clusterName}, + {Key: "node_name", Value: node.Name}, + {Key: "host", Value: node.Host}, + } + for _, dim := range chart.Dims { + dim.ID = fmt.Sprintf(dim.ID, nodeID) + } + } + + if err := es.Charts().Add(*charts...); err != nil { + es.Warning(err) + } +} + +func (es *Elasticsearch) removeNodeCharts(nodeID string) { + px := fmt.Sprintf("node_%s_cluster_%s_", nodeID, es.clusterName) + es.removeCharts(px) +} + +func (es *Elasticsearch) addIndexCharts(index string) { + charts := nodeIndexChartsTmpl.Copy() + + for _, chart := range *charts { + chart.ID = fmt.Sprintf(chart.ID, index, es.clusterName) + chart.Labels = []module.Label{ + {Key: "cluster_name", Value: es.clusterName}, + {Key: "index", Value: index}, + } + for _, dim := range chart.Dims { + dim.ID = fmt.Sprintf(dim.ID, index) + } + } + + if err := es.Charts().Add(*charts...); err != nil { + es.Warning(err) + } +} + +func (es *Elasticsearch) removeIndexCharts(index string) { + px := fmt.Sprintf("node_index_%s_cluster_%s_", index, es.clusterName) + es.removeCharts(px) +} + +func (es *Elasticsearch) removeCharts(prefix string) { + for _, chart := range *es.Charts() { + if strings.HasPrefix(chart.ID, prefix) { + chart.MarkRemove() + chart.MarkNotCreated() + } + } +} diff --git a/src/go/collectors/go.d.plugin/modules/elasticsearch/collect.go b/src/go/collectors/go.d.plugin/modules/elasticsearch/collect.go new file mode 100644 index 000000000..eb3ffb351 --- /dev/null +++ b/src/go/collectors/go.d.plugin/modules/elasticsearch/collect.go @@ -0,0 +1,308 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +package elasticsearch + +import ( + "encoding/json" + "errors" + "fmt" + "io" + "math" + "net/http" + "strconv" + "strings" + "sync" + + "github.com/netdata/netdata/go/go.d.plugin/pkg/stm" + "github.com/netdata/netdata/go/go.d.plugin/pkg/web" +) + +const ( + urlPathLocalNodeStats = "/_nodes/_local/stats" + urlPathNodesStats = "/_nodes/stats" + urlPathIndicesStats = "/_cat/indices" + urlPathClusterHealth = "/_cluster/health" + urlPathClusterStats = "/_cluster/stats" +) + +func (es *Elasticsearch) collect() (map[string]int64, error) { + if es.clusterName == "" { + name, err := es.getClusterName() + if err != nil { + return nil, err + } + es.clusterName = name + } + + ms := es.scrapeElasticsearch() + if ms.empty() { + return nil, nil + } + + mx := make(map[string]int64) + + es.collectNodesStats(mx, ms) + es.collectClusterHealth(mx, ms) + es.collectClusterStats(mx, ms) + es.collectLocalIndicesStats(mx, ms) + + return mx, nil +} + +func (es *Elasticsearch) collectNodesStats(mx map[string]int64, ms *esMetrics) { + if !ms.hasNodesStats() { + return + } + + seen := make(map[string]bool) + + for nodeID, node := range ms.NodesStats.Nodes { + seen[nodeID] = true + + if !es.nodes[nodeID] { + es.nodes[nodeID] = true + es.addNodeCharts(nodeID, node) + } + + merge(mx, stm.ToMap(node), "node_"+nodeID) + } + + for nodeID := range es.nodes { + if !seen[nodeID] { + delete(es.nodes, nodeID) + es.removeNodeCharts(nodeID) + } + } +} + +func (es *Elasticsearch) collectClusterHealth(mx map[string]int64, ms *esMetrics) { + if !ms.hasClusterHealth() { + return + } + + es.addClusterHealthChartsOnce.Do(es.addClusterHealthCharts) + + merge(mx, stm.ToMap(ms.ClusterHealth), "cluster") + + mx["cluster_status_green"] = boolToInt(ms.ClusterHealth.Status == "green") + mx["cluster_status_yellow"] = boolToInt(ms.ClusterHealth.Status == "yellow") + mx["cluster_status_red"] = boolToInt(ms.ClusterHealth.Status == "red") +} + +func (es *Elasticsearch) collectClusterStats(mx map[string]int64, ms *esMetrics) { + if !ms.hasClusterStats() { + return + } + + es.addClusterStatsChartsOnce.Do(es.addClusterStatsCharts) + + merge(mx, stm.ToMap(ms.ClusterStats), "cluster") +} + +func (es *Elasticsearch) collectLocalIndicesStats(mx map[string]int64, ms *esMetrics) { + if !ms.hasLocalIndicesStats() { + return + } + + seen := make(map[string]bool) + + for _, v := range ms.LocalIndicesStats { + seen[v.Index] = true + + if !es.indices[v.Index] { + es.indices[v.Index] = true + es.addIndexCharts(v.Index) + } + + px := fmt.Sprintf("node_index_%s_stats_", v.Index) + + mx[px+"health_green"] = boolToInt(v.Health == "green") + mx[px+"health_yellow"] = boolToInt(v.Health == "yellow") + mx[px+"health_red"] = boolToInt(v.Health == "red") + mx[px+"shards_count"] = strToInt(v.Rep) + mx[px+"docs_count"] = strToInt(v.DocsCount) + mx[px+"store_size_in_bytes"] = convertIndexStoreSizeToBytes(v.StoreSize) + } + + for index := range es.indices { + if !seen[index] { + delete(es.indices, index) + es.removeIndexCharts(index) + } + } +} + +func (es *Elasticsearch) scrapeElasticsearch() *esMetrics { + ms := &esMetrics{} + wg := &sync.WaitGroup{} + + if es.DoNodeStats { + wg.Add(1) + go func() { defer wg.Done(); es.scrapeNodesStats(ms) }() + } + if es.DoClusterHealth { + wg.Add(1) + go func() { defer wg.Done(); es.scrapeClusterHealth(ms) }() + } + if es.DoClusterStats { + wg.Add(1) + go func() { defer wg.Done(); es.scrapeClusterStats(ms) }() + } + if !es.ClusterMode && es.DoIndicesStats { + wg.Add(1) + go func() { defer wg.Done(); es.scrapeLocalIndicesStats(ms) }() + } + wg.Wait() + + return ms +} + +func (es *Elasticsearch) scrapeNodesStats(ms *esMetrics) { + req, _ := web.NewHTTPRequest(es.Request) + if es.ClusterMode { + req.URL.Path = urlPathNodesStats + } else { + req.URL.Path = urlPathLocalNodeStats + } + + var stats esNodesStats + if err := es.doOKDecode(req, &stats); err != nil { + es.Warning(err) + return + } + + ms.NodesStats = &stats +} + +func (es *Elasticsearch) scrapeClusterHealth(ms *esMetrics) { + req, _ := web.NewHTTPRequest(es.Request) + req.URL.Path = urlPathClusterHealth + + var health esClusterHealth + if err := es.doOKDecode(req, &health); err != nil { + es.Warning(err) + return + } + + ms.ClusterHealth = &health +} + +func (es *Elasticsearch) scrapeClusterStats(ms *esMetrics) { + req, _ := web.NewHTTPRequest(es.Request) + req.URL.Path = urlPathClusterStats + + var stats esClusterStats + if err := es.doOKDecode(req, &stats); err != nil { + es.Warning(err) + return + } + + ms.ClusterStats = &stats +} + +func (es *Elasticsearch) scrapeLocalIndicesStats(ms *esMetrics) { + req, _ := web.NewHTTPRequest(es.Request) + req.URL.Path = urlPathIndicesStats + req.URL.RawQuery = "local=true&format=json" + + var stats []esIndexStats + if err := es.doOKDecode(req, &stats); err != nil { + es.Warning(err) + return + } + + ms.LocalIndicesStats = removeSystemIndices(stats) +} + +func (es *Elasticsearch) getClusterName() (string, error) { + req, _ := web.NewHTTPRequest(es.Request) + + var info struct { + ClusterName string `json:"cluster_name"` + } + + if err := es.doOKDecode(req, &info); err != nil { + return "", err + } + + if info.ClusterName == "" { + return "", errors.New("empty cluster name") + } + + return info.ClusterName, nil +} + +func (es *Elasticsearch) doOKDecode(req *http.Request, in interface{}) error { + resp, err := es.httpClient.Do(req) + if err != nil { + return fmt.Errorf("error on HTTP request '%s': %v", req.URL, err) + } + defer closeBody(resp) + + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("'%s' returned HTTP status code: %d", req.URL, resp.StatusCode) + } + + if err := json.NewDecoder(resp.Body).Decode(in); err != nil { + return fmt.Errorf("error on decoding response from '%s': %v", req.URL, err) + } + return nil +} + +func closeBody(resp *http.Response) { + if resp != nil && resp.Body != nil { + _, _ = io.Copy(io.Discard, resp.Body) + _ = resp.Body.Close() + } +} + +func convertIndexStoreSizeToBytes(size string) int64 { + var num float64 + switch { + case strings.HasSuffix(size, "kb"): + num, _ = strconv.ParseFloat(size[:len(size)-2], 64) + num *= math.Pow(1024, 1) + case strings.HasSuffix(size, "mb"): + num, _ = strconv.ParseFloat(size[:len(size)-2], 64) + num *= math.Pow(1024, 2) + case strings.HasSuffix(size, "gb"): + num, _ = strconv.ParseFloat(size[:len(size)-2], 64) + num *= math.Pow(1024, 3) + case strings.HasSuffix(size, "tb"): + num, _ = strconv.ParseFloat(size[:len(size)-2], 64) + num *= math.Pow(1024, 4) + case strings.HasSuffix(size, "b"): + num, _ = strconv.ParseFloat(size[:len(size)-1], 64) + } + return int64(num) +} + +func strToInt(s string) int64 { + v, _ := strconv.Atoi(s) + return int64(v) +} + +func boolToInt(v bool) int64 { + if v { + return 1 + } + return 0 +} + +func removeSystemIndices(indices []esIndexStats) []esIndexStats { + var i int + for _, index := range indices { + if strings.HasPrefix(index.Index, ".") { + continue + } + indices[i] = index + i++ + } + return indices[:i] +} + +func merge(dst, src map[string]int64, prefix string) { + for k, v := range src { + dst[prefix+"_"+k] = v + } +} diff --git a/src/go/collectors/go.d.plugin/modules/elasticsearch/config_schema.json b/src/go/collectors/go.d.plugin/modules/elasticsearch/config_schema.json new file mode 100644 index 000000000..bc4d4eaab --- /dev/null +++ b/src/go/collectors/go.d.plugin/modules/elasticsearch/config_schema.json @@ -0,0 +1,198 @@ +{ + "jsonSchema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Elasticsearch collector configuration.", + "type": "object", + "properties": { + "update_every": { + "title": "Update every", + "description": "Data collection interval, measured in seconds.", + "type": "integer", + "minimum": 1, + "default": 5 + }, + "url": { + "title": "URL", + "description": "The base URL of the Elasticsearch cluster.", + "type": "string", + "default": "http://127.0.0.1:9200", + "format": "uri" + }, + "timeout": { + "title": "Timeout", + "description": "The timeout in seconds for the HTTP request.", + "type": "number", + "minimum": 0.5, + "default": 2 + }, + "not_follow_redirects": { + "title": "Not follow redirects", + "description": "If set, the client will not follow HTTP redirects automatically.", + "type": "boolean" + }, + "cluster_mode": { + "title": "Cluster mode", + "description": "If set, metrics will be collected for all nodes in the Elasticsearch cluster; otherwise, only for the local node where the collector is running.", + "type": "boolean", + "default": false + }, + "collect_node_stats": { + "title": "Collect node stats", + "description": "Collect metrics about individual [nodes in the cluster](https://www.elastic.co/guide/en/elasticsearch/reference/current/cluster-nodes-stats.html).", + "type": "boolean", + "default": true + }, + "collect_cluster_health": { + "title": "Collect cluster health", + "description": "Collect metrics about the overall [health of the cluster](https://www.elastic.co/guide/en/elasticsearch/reference/current/cluster-health.html).", + "type": "boolean", + "default": true + }, + "collect_cluster_stats": { + "title": "Collect cluster stats", + "description": "Collect high-level [cluster statistics](https://www.elastic.co/guide/en/elasticsearch/reference/current/cluster-stats.html).", + "type": "boolean", + "default": true + }, + "collect_indices_stats": { + "title": "Collect indices stats", + "description": "Collect metrics about individual [indices in the cluster](https://www.elastic.co/guide/en/elasticsearch/reference/current/cat-indices.html).", + "type": "boolean", + "default": false + }, + "username": { + "title": "Username", + "description": "The username for basic authentication.", + "type": "string", + "sensitive": true + }, + "password": { + "title": "Password", + "description": "The password for basic authentication.", + "type": "string", + "sensitive": true + }, + "proxy_url": { + "title": "Proxy URL", + "description": "The URL of the proxy server.", + "type": "string" + }, + "proxy_username": { + "title": "Proxy username", + "description": "The username for proxy authentication.", + "type": "string", + "sensitive": true + }, + "proxy_password": { + "title": "Proxy password", + "description": "The password for proxy authentication.", + "type": "string", + "sensitive": true + }, + "headers": { + "title": "Headers", + "description": "Additional HTTP headers to include in the request.", + "type": [ + "object", + "null" + ], + "additionalProperties": { + "type": "string" + } + }, + "tls_skip_verify": { + "title": "Skip TLS verification", + "description": "If set, TLS certificate verification will be skipped.", + "type": "boolean" + }, + "tls_ca": { + "title": "TLS CA", + "description": "The path to the CA certificate file for TLS verification.", + "type": "string", + "pattern": "^$|^/" + }, + "tls_cert": { + "title": "TLS certificate", + "description": "The path to the client certificate file for TLS authentication.", + "type": "string", + "pattern": "^$|^/" + }, + "tls_key": { + "title": "TLS key", + "description": "The path to the client key file for TLS authentication.", + "type": "string", + "pattern": "^$|^/" + } + }, + "required": [ + "url" + ], + "additionalProperties": false, + "patternProperties": { + "^name$": {} + } + }, + "uiSchema": { + "ui:flavour": "tabs", + "ui:options": { + "tabs": [ + { + "title": "Base", + "fields": [ + "update_every", + "url", + "timeout", + "not_follow_redirects", + "cluster_mode", + "collect_node_stats", + "collect_cluster_health", + "collect_cluster_stats", + "collect_indices_stats" + ] + }, + { + "title": "Auth", + "fields": [ + "username", + "password" + ] + }, + { + "title": "TLS", + "fields": [ + "tls_skip_verify", + "tls_ca", + "tls_cert", + "tls_key" + ] + }, + { + "title": "Proxy", + "fields": [ + "proxy_url", + "proxy_username", + "proxy_password" + ] + }, + { + "title": "Headers", + "fields": [ + "headers" + ] + } + ] + }, + "uiOptions": { + "fullPage": true + }, + "timeout": { + "ui:help": "Accepts decimals for precise control (e.g., type 1.5 for 1.5 seconds)." + }, + "password": { + "ui:widget": "password" + }, + "proxy_password": { + "ui:widget": "password" + } + } +} diff --git a/src/go/collectors/go.d.plugin/modules/elasticsearch/elasticsearch.go b/src/go/collectors/go.d.plugin/modules/elasticsearch/elasticsearch.go new file mode 100644 index 000000000..a2b7b529f --- /dev/null +++ b/src/go/collectors/go.d.plugin/modules/elasticsearch/elasticsearch.go @@ -0,0 +1,135 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +package elasticsearch + +import ( + _ "embed" + "errors" + "net/http" + "sync" + "time" + + "github.com/netdata/netdata/go/go.d.plugin/agent/module" + "github.com/netdata/netdata/go/go.d.plugin/pkg/web" +) + +//go:embed "config_schema.json" +var configSchema string + +func init() { + module.Register("elasticsearch", module.Creator{ + JobConfigSchema: configSchema, + Defaults: module.Defaults{ + UpdateEvery: 5, + }, + Create: func() module.Module { return New() }, + }) +} + +func New() *Elasticsearch { + return &Elasticsearch{ + Config: Config{ + HTTP: web.HTTP{ + Request: web.Request{ + URL: "http://127.0.0.1:9200", + }, + Client: web.Client{ + Timeout: web.Duration(time.Second * 2), + }, + }, + ClusterMode: false, + + DoNodeStats: true, + DoClusterStats: true, + DoClusterHealth: true, + DoIndicesStats: false, + }, + + charts: &module.Charts{}, + addClusterHealthChartsOnce: &sync.Once{}, + addClusterStatsChartsOnce: &sync.Once{}, + nodes: make(map[string]bool), + indices: make(map[string]bool), + } +} + +type Config struct { + web.HTTP `yaml:",inline" json:""` + UpdateEvery int `yaml:"update_every" json:"update_every"` + ClusterMode bool `yaml:"cluster_mode" json:"cluster_mode"` + DoNodeStats bool `yaml:"collect_node_stats" json:"collect_node_stats"` + DoClusterHealth bool `yaml:"collect_cluster_health" json:"collect_cluster_health"` + DoClusterStats bool `yaml:"collect_cluster_stats" json:"collect_cluster_stats"` + DoIndicesStats bool `yaml:"collect_indices_stats" json:"collect_indices_stats"` +} + +type Elasticsearch struct { + module.Base + Config `yaml:",inline" json:""` + + charts *module.Charts + addClusterHealthChartsOnce *sync.Once + addClusterStatsChartsOnce *sync.Once + + httpClient *http.Client + + clusterName string + nodes map[string]bool + indices map[string]bool +} + +func (es *Elasticsearch) Configuration() any { + return es.Config +} + +func (es *Elasticsearch) Init() error { + err := es.validateConfig() + if err != nil { + es.Errorf("check configuration: %v", err) + return err + } + + httpClient, err := es.initHTTPClient() + if err != nil { + es.Errorf("init HTTP client: %v", err) + return err + } + es.httpClient = httpClient + + return nil +} + +func (es *Elasticsearch) Check() error { + mx, err := es.collect() + if err != nil { + es.Error(err) + return err + } + if len(mx) == 0 { + return errors.New("no metrics collected") + + } + return nil +} + +func (es *Elasticsearch) Charts() *module.Charts { + return es.charts +} + +func (es *Elasticsearch) Collect() map[string]int64 { + mx, err := es.collect() + if err != nil { + es.Error(err) + } + + if len(mx) == 0 { + return nil + } + return mx +} + +func (es *Elasticsearch) Cleanup() { + if es.httpClient != nil { + es.httpClient.CloseIdleConnections() + } +} diff --git a/src/go/collectors/go.d.plugin/modules/elasticsearch/elasticsearch_test.go b/src/go/collectors/go.d.plugin/modules/elasticsearch/elasticsearch_test.go new file mode 100644 index 000000000..dc4817336 --- /dev/null +++ b/src/go/collectors/go.d.plugin/modules/elasticsearch/elasticsearch_test.go @@ -0,0 +1,743 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +package elasticsearch + +import ( + "github.com/netdata/netdata/go/go.d.plugin/agent/module" + "net/http" + "net/http/httptest" + "os" + "testing" + + "github.com/netdata/netdata/go/go.d.plugin/pkg/tlscfg" + "github.com/netdata/netdata/go/go.d.plugin/pkg/web" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +var ( + dataConfigJSON, _ = os.ReadFile("testdata/config.json") + dataConfigYAML, _ = os.ReadFile("testdata/config.yaml") + + dataVer842NodesLocalStats, _ = os.ReadFile("testdata/v8.4.2/nodes_local_stats.json") + dataVer842NodesStats, _ = os.ReadFile("testdata/v8.4.2/nodes_stats.json") + dataVer842ClusterHealth, _ = os.ReadFile("testdata/v8.4.2/cluster_health.json") + dataVer842ClusterStats, _ = os.ReadFile("testdata/v8.4.2/cluster_stats.json") + dataVer842CatIndicesStats, _ = os.ReadFile("testdata/v8.4.2/cat_indices_stats.json") + dataVer842Info, _ = os.ReadFile("testdata/v8.4.2/info.json") +) + +func Test_testDataIsValid(t *testing.T) { + for name, data := range map[string][]byte{ + "dataConfigJSON": dataConfigJSON, + "dataConfigYAML": dataConfigYAML, + "dataVer842NodesLocalStats": dataVer842NodesLocalStats, + "dataVer842NodesStats": dataVer842NodesStats, + "dataVer842ClusterHealth": dataVer842ClusterHealth, + "dataVer842ClusterStats": dataVer842ClusterStats, + "dataVer842CatIndicesStats": dataVer842CatIndicesStats, + "dataVer842Info": dataVer842Info, + } { + require.NotNil(t, data, name) + } +} + +func TestElasticsearch_ConfigurationSerialize(t *testing.T) { + module.TestConfigurationSerialize(t, &Elasticsearch{}, dataConfigJSON, dataConfigYAML) +} + +func TestElasticsearch_Init(t *testing.T) { + tests := map[string]struct { + config Config + wantFail bool + }{ + "default": { + config: New().Config, + }, + "all stats": { + config: Config{ + HTTP: web.HTTP{ + Request: web.Request{URL: "http://127.0.0.1:38001"}, + }, + DoNodeStats: true, + DoClusterHealth: true, + DoClusterStats: true, + DoIndicesStats: true, + }, + }, + "only node_stats": { + config: Config{ + HTTP: web.HTTP{ + Request: web.Request{URL: "http://127.0.0.1:38001"}, + }, + DoNodeStats: true, + DoClusterHealth: false, + DoClusterStats: false, + DoIndicesStats: false, + }, + }, + "URL not set": { + wantFail: true, + config: Config{ + HTTP: web.HTTP{ + Request: web.Request{URL: ""}, + }}, + }, + "invalid TLSCA": { + wantFail: true, + config: Config{ + HTTP: web.HTTP{ + Client: web.Client{ + TLSConfig: tlscfg.TLSConfig{TLSCA: "testdata/tls"}, + }, + }}, + }, + "all API calls are disabled": { + wantFail: true, + config: Config{ + HTTP: web.HTTP{ + Request: web.Request{URL: "http://127.0.0.1:38001"}, + }, + DoNodeStats: false, + DoClusterHealth: false, + DoClusterStats: false, + DoIndicesStats: false, + }, + }, + } + + for name, test := range tests { + t.Run(name, func(t *testing.T) { + es := New() + es.Config = test.config + + if test.wantFail { + assert.Error(t, es.Init()) + } else { + assert.NoError(t, es.Init()) + } + }) + } +} + +func TestElasticsearch_Check(t *testing.T) { + tests := map[string]struct { + prepare func(*testing.T) (es *Elasticsearch, cleanup func()) + wantFail bool + }{ + "valid data": {prepare: prepareElasticsearchValidData}, + "invalid data": {prepare: prepareElasticsearchInvalidData, wantFail: true}, + "404": {prepare: prepareElasticsearch404, wantFail: true}, + "connection refused": {prepare: prepareElasticsearchConnectionRefused, wantFail: true}, + } + + for name, test := range tests { + t.Run(name, func(t *testing.T) { + es, cleanup := test.prepare(t) + defer cleanup() + + if test.wantFail { + assert.Error(t, es.Check()) + } else { + assert.NoError(t, es.Check()) + } + }) + } +} + +func TestElasticsearch_Charts(t *testing.T) { + assert.NotNil(t, New().Charts()) +} + +func TestElasticsearch_Cleanup(t *testing.T) { + assert.NotPanics(t, New().Cleanup) +} + +func TestElasticsearch_Collect(t *testing.T) { + tests := map[string]struct { + prepare func() *Elasticsearch + wantCollected map[string]int64 + wantCharts int + }{ + "v842: all nodes stats": { + prepare: func() *Elasticsearch { + es := New() + es.ClusterMode = true + es.DoNodeStats = true + es.DoClusterHealth = false + es.DoClusterStats = false + es.DoIndicesStats = false + return es + }, + wantCharts: len(nodeChartsTmpl) * 3, + wantCollected: map[string]int64{ + "node_Klg1CjgMTouentQcJlRGuA_breakers_accounting_tripped": 0, + "node_Klg1CjgMTouentQcJlRGuA_breakers_fielddata_tripped": 0, + "node_Klg1CjgMTouentQcJlRGuA_breakers_in_flight_requests_tripped": 0, + "node_Klg1CjgMTouentQcJlRGuA_breakers_model_inference_tripped": 0, + "node_Klg1CjgMTouentQcJlRGuA_breakers_parent_tripped": 0, + "node_Klg1CjgMTouentQcJlRGuA_breakers_request_tripped": 0, + "node_Klg1CjgMTouentQcJlRGuA_http_current_open": 75, + "node_Klg1CjgMTouentQcJlRGuA_indices_fielddata_evictions": 0, + "node_Klg1CjgMTouentQcJlRGuA_indices_fielddata_memory_size_in_bytes": 600, + "node_Klg1CjgMTouentQcJlRGuA_indices_flush_total": 35130, + "node_Klg1CjgMTouentQcJlRGuA_indices_flush_total_time_in_millis": 22204637, + "node_Klg1CjgMTouentQcJlRGuA_indices_indexing_index_current": 0, + "node_Klg1CjgMTouentQcJlRGuA_indices_indexing_index_time_in_millis": 1100012973, + "node_Klg1CjgMTouentQcJlRGuA_indices_indexing_index_total": 3667364815, + "node_Klg1CjgMTouentQcJlRGuA_indices_refresh_total": 7720800, + "node_Klg1CjgMTouentQcJlRGuA_indices_refresh_total_time_in_millis": 94297737, + "node_Klg1CjgMTouentQcJlRGuA_indices_search_fetch_current": 0, + "node_Klg1CjgMTouentQcJlRGuA_indices_search_fetch_time_in_millis": 21316723, + "node_Klg1CjgMTouentQcJlRGuA_indices_search_fetch_total": 42642621, + "node_Klg1CjgMTouentQcJlRGuA_indices_search_query_current": 0, + "node_Klg1CjgMTouentQcJlRGuA_indices_search_query_time_in_millis": 51262303, + "node_Klg1CjgMTouentQcJlRGuA_indices_search_query_total": 166820275, + "node_Klg1CjgMTouentQcJlRGuA_indices_segments_count": 320, + "node_Klg1CjgMTouentQcJlRGuA_indices_segments_doc_values_memory_in_bytes": 0, + "node_Klg1CjgMTouentQcJlRGuA_indices_segments_fixed_bit_set_memory_in_bytes": 1904, + "node_Klg1CjgMTouentQcJlRGuA_indices_segments_index_writer_memory_in_bytes": 262022568, + "node_Klg1CjgMTouentQcJlRGuA_indices_segments_memory_in_bytes": 0, + "node_Klg1CjgMTouentQcJlRGuA_indices_segments_norms_memory_in_bytes": 0, + "node_Klg1CjgMTouentQcJlRGuA_indices_segments_points_memory_in_bytes": 0, + "node_Klg1CjgMTouentQcJlRGuA_indices_segments_stored_fields_memory_in_bytes": 0, + "node_Klg1CjgMTouentQcJlRGuA_indices_segments_term_vectors_memory_in_bytes": 0, + "node_Klg1CjgMTouentQcJlRGuA_indices_segments_terms_memory_in_bytes": 0, + "node_Klg1CjgMTouentQcJlRGuA_indices_segments_version_map_memory_in_bytes": 49200018, + "node_Klg1CjgMTouentQcJlRGuA_indices_translog_operations": 352376, + "node_Klg1CjgMTouentQcJlRGuA_indices_translog_size_in_bytes": 447695989, + "node_Klg1CjgMTouentQcJlRGuA_indices_translog_uncommitted_operations": 352376, + "node_Klg1CjgMTouentQcJlRGuA_indices_translog_uncommitted_size_in_bytes": 447695989, + "node_Klg1CjgMTouentQcJlRGuA_jvm_buffer_pools_direct_count": 94, + "node_Klg1CjgMTouentQcJlRGuA_jvm_buffer_pools_direct_total_capacity_in_bytes": 4654848, + "node_Klg1CjgMTouentQcJlRGuA_jvm_buffer_pools_direct_used_in_bytes": 4654850, + "node_Klg1CjgMTouentQcJlRGuA_jvm_buffer_pools_mapped_count": 858, + "node_Klg1CjgMTouentQcJlRGuA_jvm_buffer_pools_mapped_total_capacity_in_bytes": 103114998135, + "node_Klg1CjgMTouentQcJlRGuA_jvm_buffer_pools_mapped_used_in_bytes": 103114998135, + "node_Klg1CjgMTouentQcJlRGuA_jvm_gc_collectors_old_collection_count": 0, + "node_Klg1CjgMTouentQcJlRGuA_jvm_gc_collectors_old_collection_time_in_millis": 0, + "node_Klg1CjgMTouentQcJlRGuA_jvm_gc_collectors_young_collection_count": 78652, + "node_Klg1CjgMTouentQcJlRGuA_jvm_gc_collectors_young_collection_time_in_millis": 6014274, + "node_Klg1CjgMTouentQcJlRGuA_jvm_mem_heap_committed_in_bytes": 7864320000, + "node_Klg1CjgMTouentQcJlRGuA_jvm_mem_heap_used_in_bytes": 5059735552, + "node_Klg1CjgMTouentQcJlRGuA_jvm_mem_heap_used_percent": 64, + "node_Klg1CjgMTouentQcJlRGuA_process_max_file_descriptors": 1048576, + "node_Klg1CjgMTouentQcJlRGuA_process_open_file_descriptors": 1156, + "node_Klg1CjgMTouentQcJlRGuA_thread_pool_analyze_queue": 0, + "node_Klg1CjgMTouentQcJlRGuA_thread_pool_analyze_rejected": 0, + "node_Klg1CjgMTouentQcJlRGuA_thread_pool_fetch_shard_started_queue": 0, + "node_Klg1CjgMTouentQcJlRGuA_thread_pool_fetch_shard_started_rejected": 0, + "node_Klg1CjgMTouentQcJlRGuA_thread_pool_fetch_shard_store_queue": 0, + "node_Klg1CjgMTouentQcJlRGuA_thread_pool_fetch_shard_store_rejected": 0, + "node_Klg1CjgMTouentQcJlRGuA_thread_pool_flush_queue": 0, + "node_Klg1CjgMTouentQcJlRGuA_thread_pool_flush_rejected": 0, + "node_Klg1CjgMTouentQcJlRGuA_thread_pool_force_merge_queue": 0, + "node_Klg1CjgMTouentQcJlRGuA_thread_pool_force_merge_rejected": 0, + "node_Klg1CjgMTouentQcJlRGuA_thread_pool_generic_queue": 0, + "node_Klg1CjgMTouentQcJlRGuA_thread_pool_generic_rejected": 0, + "node_Klg1CjgMTouentQcJlRGuA_thread_pool_get_queue": 0, + "node_Klg1CjgMTouentQcJlRGuA_thread_pool_get_rejected": 0, + "node_Klg1CjgMTouentQcJlRGuA_thread_pool_listener_queue": 0, + "node_Klg1CjgMTouentQcJlRGuA_thread_pool_listener_rejected": 0, + "node_Klg1CjgMTouentQcJlRGuA_thread_pool_management_queue": 0, + "node_Klg1CjgMTouentQcJlRGuA_thread_pool_management_rejected": 0, + "node_Klg1CjgMTouentQcJlRGuA_thread_pool_refresh_queue": 0, + "node_Klg1CjgMTouentQcJlRGuA_thread_pool_refresh_rejected": 0, + "node_Klg1CjgMTouentQcJlRGuA_thread_pool_search_queue": 0, + "node_Klg1CjgMTouentQcJlRGuA_thread_pool_search_rejected": 0, + "node_Klg1CjgMTouentQcJlRGuA_thread_pool_search_throttled_queue": 0, + "node_Klg1CjgMTouentQcJlRGuA_thread_pool_search_throttled_rejected": 0, + "node_Klg1CjgMTouentQcJlRGuA_thread_pool_snapshot_queue": 0, + "node_Klg1CjgMTouentQcJlRGuA_thread_pool_snapshot_rejected": 0, + "node_Klg1CjgMTouentQcJlRGuA_thread_pool_warmer_queue": 0, + "node_Klg1CjgMTouentQcJlRGuA_thread_pool_warmer_rejected": 0, + "node_Klg1CjgMTouentQcJlRGuA_thread_pool_write_queue": 0, + "node_Klg1CjgMTouentQcJlRGuA_thread_pool_write_rejected": 0, + "node_Klg1CjgMTouentQcJlRGuA_transport_rx_count": 1300324276, + "node_Klg1CjgMTouentQcJlRGuA_transport_rx_size_in_bytes": 1789333458217, + "node_Klg1CjgMTouentQcJlRGuA_transport_tx_count": 1300324275, + "node_Klg1CjgMTouentQcJlRGuA_transport_tx_size_in_bytes": 2927487680282, + "node_k_AifYMWQTykjUq3pgE_-w_breakers_accounting_tripped": 0, + "node_k_AifYMWQTykjUq3pgE_-w_breakers_fielddata_tripped": 0, + "node_k_AifYMWQTykjUq3pgE_-w_breakers_in_flight_requests_tripped": 0, + "node_k_AifYMWQTykjUq3pgE_-w_breakers_model_inference_tripped": 0, + "node_k_AifYMWQTykjUq3pgE_-w_breakers_parent_tripped": 0, + "node_k_AifYMWQTykjUq3pgE_-w_breakers_request_tripped": 0, + "node_k_AifYMWQTykjUq3pgE_-w_http_current_open": 14, + "node_k_AifYMWQTykjUq3pgE_-w_indices_fielddata_evictions": 0, + "node_k_AifYMWQTykjUq3pgE_-w_indices_fielddata_memory_size_in_bytes": 0, + "node_k_AifYMWQTykjUq3pgE_-w_indices_flush_total": 0, + "node_k_AifYMWQTykjUq3pgE_-w_indices_flush_total_time_in_millis": 0, + "node_k_AifYMWQTykjUq3pgE_-w_indices_indexing_index_current": 0, + "node_k_AifYMWQTykjUq3pgE_-w_indices_indexing_index_time_in_millis": 0, + "node_k_AifYMWQTykjUq3pgE_-w_indices_indexing_index_total": 0, + "node_k_AifYMWQTykjUq3pgE_-w_indices_refresh_total": 0, + "node_k_AifYMWQTykjUq3pgE_-w_indices_refresh_total_time_in_millis": 0, + "node_k_AifYMWQTykjUq3pgE_-w_indices_search_fetch_current": 0, + "node_k_AifYMWQTykjUq3pgE_-w_indices_search_fetch_time_in_millis": 0, + "node_k_AifYMWQTykjUq3pgE_-w_indices_search_fetch_total": 0, + "node_k_AifYMWQTykjUq3pgE_-w_indices_search_query_current": 0, + "node_k_AifYMWQTykjUq3pgE_-w_indices_search_query_time_in_millis": 0, + "node_k_AifYMWQTykjUq3pgE_-w_indices_search_query_total": 0, + "node_k_AifYMWQTykjUq3pgE_-w_indices_segments_count": 0, + "node_k_AifYMWQTykjUq3pgE_-w_indices_segments_doc_values_memory_in_bytes": 0, + "node_k_AifYMWQTykjUq3pgE_-w_indices_segments_fixed_bit_set_memory_in_bytes": 0, + "node_k_AifYMWQTykjUq3pgE_-w_indices_segments_index_writer_memory_in_bytes": 0, + "node_k_AifYMWQTykjUq3pgE_-w_indices_segments_memory_in_bytes": 0, + "node_k_AifYMWQTykjUq3pgE_-w_indices_segments_norms_memory_in_bytes": 0, + "node_k_AifYMWQTykjUq3pgE_-w_indices_segments_points_memory_in_bytes": 0, + "node_k_AifYMWQTykjUq3pgE_-w_indices_segments_stored_fields_memory_in_bytes": 0, + "node_k_AifYMWQTykjUq3pgE_-w_indices_segments_term_vectors_memory_in_bytes": 0, + "node_k_AifYMWQTykjUq3pgE_-w_indices_segments_terms_memory_in_bytes": 0, + "node_k_AifYMWQTykjUq3pgE_-w_indices_segments_version_map_memory_in_bytes": 0, + "node_k_AifYMWQTykjUq3pgE_-w_indices_translog_operations": 0, + "node_k_AifYMWQTykjUq3pgE_-w_indices_translog_size_in_bytes": 0, + "node_k_AifYMWQTykjUq3pgE_-w_indices_translog_uncommitted_operations": 0, + "node_k_AifYMWQTykjUq3pgE_-w_indices_translog_uncommitted_size_in_bytes": 0, + "node_k_AifYMWQTykjUq3pgE_-w_jvm_buffer_pools_direct_count": 19, + "node_k_AifYMWQTykjUq3pgE_-w_jvm_buffer_pools_direct_total_capacity_in_bytes": 2142214, + "node_k_AifYMWQTykjUq3pgE_-w_jvm_buffer_pools_direct_used_in_bytes": 2142216, + "node_k_AifYMWQTykjUq3pgE_-w_jvm_buffer_pools_mapped_count": 0, + "node_k_AifYMWQTykjUq3pgE_-w_jvm_buffer_pools_mapped_total_capacity_in_bytes": 0, + "node_k_AifYMWQTykjUq3pgE_-w_jvm_buffer_pools_mapped_used_in_bytes": 0, + "node_k_AifYMWQTykjUq3pgE_-w_jvm_gc_collectors_old_collection_count": 0, + "node_k_AifYMWQTykjUq3pgE_-w_jvm_gc_collectors_old_collection_time_in_millis": 0, + "node_k_AifYMWQTykjUq3pgE_-w_jvm_gc_collectors_young_collection_count": 342994, + "node_k_AifYMWQTykjUq3pgE_-w_jvm_gc_collectors_young_collection_time_in_millis": 768917, + "node_k_AifYMWQTykjUq3pgE_-w_jvm_mem_heap_committed_in_bytes": 281018368, + "node_k_AifYMWQTykjUq3pgE_-w_jvm_mem_heap_used_in_bytes": 178362704, + "node_k_AifYMWQTykjUq3pgE_-w_jvm_mem_heap_used_percent": 63, + "node_k_AifYMWQTykjUq3pgE_-w_process_max_file_descriptors": 1048576, + "node_k_AifYMWQTykjUq3pgE_-w_process_open_file_descriptors": 557, + "node_k_AifYMWQTykjUq3pgE_-w_thread_pool_analyze_queue": 0, + "node_k_AifYMWQTykjUq3pgE_-w_thread_pool_analyze_rejected": 0, + "node_k_AifYMWQTykjUq3pgE_-w_thread_pool_fetch_shard_started_queue": 0, + "node_k_AifYMWQTykjUq3pgE_-w_thread_pool_fetch_shard_started_rejected": 0, + "node_k_AifYMWQTykjUq3pgE_-w_thread_pool_fetch_shard_store_queue": 0, + "node_k_AifYMWQTykjUq3pgE_-w_thread_pool_fetch_shard_store_rejected": 0, + "node_k_AifYMWQTykjUq3pgE_-w_thread_pool_flush_queue": 0, + "node_k_AifYMWQTykjUq3pgE_-w_thread_pool_flush_rejected": 0, + "node_k_AifYMWQTykjUq3pgE_-w_thread_pool_force_merge_queue": 0, + "node_k_AifYMWQTykjUq3pgE_-w_thread_pool_force_merge_rejected": 0, + "node_k_AifYMWQTykjUq3pgE_-w_thread_pool_generic_queue": 0, + "node_k_AifYMWQTykjUq3pgE_-w_thread_pool_generic_rejected": 0, + "node_k_AifYMWQTykjUq3pgE_-w_thread_pool_get_queue": 0, + "node_k_AifYMWQTykjUq3pgE_-w_thread_pool_get_rejected": 0, + "node_k_AifYMWQTykjUq3pgE_-w_thread_pool_listener_queue": 0, + "node_k_AifYMWQTykjUq3pgE_-w_thread_pool_listener_rejected": 0, + "node_k_AifYMWQTykjUq3pgE_-w_thread_pool_management_queue": 0, + "node_k_AifYMWQTykjUq3pgE_-w_thread_pool_management_rejected": 0, + "node_k_AifYMWQTykjUq3pgE_-w_thread_pool_refresh_queue": 0, + "node_k_AifYMWQTykjUq3pgE_-w_thread_pool_refresh_rejected": 0, + "node_k_AifYMWQTykjUq3pgE_-w_thread_pool_search_queue": 0, + "node_k_AifYMWQTykjUq3pgE_-w_thread_pool_search_rejected": 0, + "node_k_AifYMWQTykjUq3pgE_-w_thread_pool_search_throttled_queue": 0, + "node_k_AifYMWQTykjUq3pgE_-w_thread_pool_search_throttled_rejected": 0, + "node_k_AifYMWQTykjUq3pgE_-w_thread_pool_snapshot_queue": 0, + "node_k_AifYMWQTykjUq3pgE_-w_thread_pool_snapshot_rejected": 0, + "node_k_AifYMWQTykjUq3pgE_-w_thread_pool_warmer_queue": 0, + "node_k_AifYMWQTykjUq3pgE_-w_thread_pool_warmer_rejected": 0, + "node_k_AifYMWQTykjUq3pgE_-w_thread_pool_write_queue": 0, + "node_k_AifYMWQTykjUq3pgE_-w_thread_pool_write_rejected": 0, + "node_k_AifYMWQTykjUq3pgE_-w_transport_rx_count": 107632996, + "node_k_AifYMWQTykjUq3pgE_-w_transport_rx_size_in_bytes": 180620082152, + "node_k_AifYMWQTykjUq3pgE_-w_transport_tx_count": 107633007, + "node_k_AifYMWQTykjUq3pgE_-w_transport_tx_size_in_bytes": 420999501235, + "node_tk_U7GMCRkCG4FoOvusrng_breakers_accounting_tripped": 0, + "node_tk_U7GMCRkCG4FoOvusrng_breakers_fielddata_tripped": 0, + "node_tk_U7GMCRkCG4FoOvusrng_breakers_in_flight_requests_tripped": 0, + "node_tk_U7GMCRkCG4FoOvusrng_breakers_model_inference_tripped": 0, + "node_tk_U7GMCRkCG4FoOvusrng_breakers_parent_tripped": 93, + "node_tk_U7GMCRkCG4FoOvusrng_breakers_request_tripped": 1, + "node_tk_U7GMCRkCG4FoOvusrng_http_current_open": 84, + "node_tk_U7GMCRkCG4FoOvusrng_indices_fielddata_evictions": 0, + "node_tk_U7GMCRkCG4FoOvusrng_indices_fielddata_memory_size_in_bytes": 0, + "node_tk_U7GMCRkCG4FoOvusrng_indices_flush_total": 67895, + "node_tk_U7GMCRkCG4FoOvusrng_indices_flush_total_time_in_millis": 81917283, + "node_tk_U7GMCRkCG4FoOvusrng_indices_indexing_index_current": 0, + "node_tk_U7GMCRkCG4FoOvusrng_indices_indexing_index_time_in_millis": 1244633519, + "node_tk_U7GMCRkCG4FoOvusrng_indices_indexing_index_total": 6550378755, + "node_tk_U7GMCRkCG4FoOvusrng_indices_refresh_total": 12359783, + "node_tk_U7GMCRkCG4FoOvusrng_indices_refresh_total_time_in_millis": 300152615, + "node_tk_U7GMCRkCG4FoOvusrng_indices_search_fetch_current": 0, + "node_tk_U7GMCRkCG4FoOvusrng_indices_search_fetch_time_in_millis": 24517851, + "node_tk_U7GMCRkCG4FoOvusrng_indices_search_fetch_total": 25105951, + "node_tk_U7GMCRkCG4FoOvusrng_indices_search_query_current": 0, + "node_tk_U7GMCRkCG4FoOvusrng_indices_search_query_time_in_millis": 158980385, + "node_tk_U7GMCRkCG4FoOvusrng_indices_search_query_total": 157912598, + "node_tk_U7GMCRkCG4FoOvusrng_indices_segments_count": 291, + "node_tk_U7GMCRkCG4FoOvusrng_indices_segments_doc_values_memory_in_bytes": 0, + "node_tk_U7GMCRkCG4FoOvusrng_indices_segments_fixed_bit_set_memory_in_bytes": 55672, + "node_tk_U7GMCRkCG4FoOvusrng_indices_segments_index_writer_memory_in_bytes": 57432664, + "node_tk_U7GMCRkCG4FoOvusrng_indices_segments_memory_in_bytes": 0, + "node_tk_U7GMCRkCG4FoOvusrng_indices_segments_norms_memory_in_bytes": 0, + "node_tk_U7GMCRkCG4FoOvusrng_indices_segments_points_memory_in_bytes": 0, + "node_tk_U7GMCRkCG4FoOvusrng_indices_segments_stored_fields_memory_in_bytes": 0, + "node_tk_U7GMCRkCG4FoOvusrng_indices_segments_term_vectors_memory_in_bytes": 0, + "node_tk_U7GMCRkCG4FoOvusrng_indices_segments_terms_memory_in_bytes": 0, + "node_tk_U7GMCRkCG4FoOvusrng_indices_segments_version_map_memory_in_bytes": 568, + "node_tk_U7GMCRkCG4FoOvusrng_indices_translog_operations": 1449698, + "node_tk_U7GMCRkCG4FoOvusrng_indices_translog_size_in_bytes": 1214204014, + "node_tk_U7GMCRkCG4FoOvusrng_indices_translog_uncommitted_operations": 1449698, + "node_tk_U7GMCRkCG4FoOvusrng_indices_translog_uncommitted_size_in_bytes": 1214204014, + "node_tk_U7GMCRkCG4FoOvusrng_jvm_buffer_pools_direct_count": 90, + "node_tk_U7GMCRkCG4FoOvusrng_jvm_buffer_pools_direct_total_capacity_in_bytes": 4571711, + "node_tk_U7GMCRkCG4FoOvusrng_jvm_buffer_pools_direct_used_in_bytes": 4571713, + "node_tk_U7GMCRkCG4FoOvusrng_jvm_buffer_pools_mapped_count": 831, + "node_tk_U7GMCRkCG4FoOvusrng_jvm_buffer_pools_mapped_total_capacity_in_bytes": 99844219805, + "node_tk_U7GMCRkCG4FoOvusrng_jvm_buffer_pools_mapped_used_in_bytes": 99844219805, + "node_tk_U7GMCRkCG4FoOvusrng_jvm_gc_collectors_old_collection_count": 1, + "node_tk_U7GMCRkCG4FoOvusrng_jvm_gc_collectors_old_collection_time_in_millis": 796, + "node_tk_U7GMCRkCG4FoOvusrng_jvm_gc_collectors_young_collection_count": 139959, + "node_tk_U7GMCRkCG4FoOvusrng_jvm_gc_collectors_young_collection_time_in_millis": 3581668, + "node_tk_U7GMCRkCG4FoOvusrng_jvm_mem_heap_committed_in_bytes": 7864320000, + "node_tk_U7GMCRkCG4FoOvusrng_jvm_mem_heap_used_in_bytes": 1884124192, + "node_tk_U7GMCRkCG4FoOvusrng_jvm_mem_heap_used_percent": 23, + "node_tk_U7GMCRkCG4FoOvusrng_process_max_file_descriptors": 1048576, + "node_tk_U7GMCRkCG4FoOvusrng_process_open_file_descriptors": 1180, + "node_tk_U7GMCRkCG4FoOvusrng_thread_pool_analyze_queue": 0, + "node_tk_U7GMCRkCG4FoOvusrng_thread_pool_analyze_rejected": 0, + "node_tk_U7GMCRkCG4FoOvusrng_thread_pool_fetch_shard_started_queue": 0, + "node_tk_U7GMCRkCG4FoOvusrng_thread_pool_fetch_shard_started_rejected": 0, + "node_tk_U7GMCRkCG4FoOvusrng_thread_pool_fetch_shard_store_queue": 0, + "node_tk_U7GMCRkCG4FoOvusrng_thread_pool_fetch_shard_store_rejected": 0, + "node_tk_U7GMCRkCG4FoOvusrng_thread_pool_flush_queue": 0, + "node_tk_U7GMCRkCG4FoOvusrng_thread_pool_flush_rejected": 0, + "node_tk_U7GMCRkCG4FoOvusrng_thread_pool_force_merge_queue": 0, + "node_tk_U7GMCRkCG4FoOvusrng_thread_pool_force_merge_rejected": 0, + "node_tk_U7GMCRkCG4FoOvusrng_thread_pool_generic_queue": 0, + "node_tk_U7GMCRkCG4FoOvusrng_thread_pool_generic_rejected": 0, + "node_tk_U7GMCRkCG4FoOvusrng_thread_pool_get_queue": 0, + "node_tk_U7GMCRkCG4FoOvusrng_thread_pool_get_rejected": 0, + "node_tk_U7GMCRkCG4FoOvusrng_thread_pool_listener_queue": 0, + "node_tk_U7GMCRkCG4FoOvusrng_thread_pool_listener_rejected": 0, + "node_tk_U7GMCRkCG4FoOvusrng_thread_pool_management_queue": 0, + "node_tk_U7GMCRkCG4FoOvusrng_thread_pool_management_rejected": 0, + "node_tk_U7GMCRkCG4FoOvusrng_thread_pool_refresh_queue": 0, + "node_tk_U7GMCRkCG4FoOvusrng_thread_pool_refresh_rejected": 0, + "node_tk_U7GMCRkCG4FoOvusrng_thread_pool_search_queue": 0, + "node_tk_U7GMCRkCG4FoOvusrng_thread_pool_search_rejected": 0, + "node_tk_U7GMCRkCG4FoOvusrng_thread_pool_search_throttled_queue": 0, + "node_tk_U7GMCRkCG4FoOvusrng_thread_pool_search_throttled_rejected": 0, + "node_tk_U7GMCRkCG4FoOvusrng_thread_pool_snapshot_queue": 0, + "node_tk_U7GMCRkCG4FoOvusrng_thread_pool_snapshot_rejected": 0, + "node_tk_U7GMCRkCG4FoOvusrng_thread_pool_warmer_queue": 0, + "node_tk_U7GMCRkCG4FoOvusrng_thread_pool_warmer_rejected": 0, + "node_tk_U7GMCRkCG4FoOvusrng_thread_pool_write_queue": 0, + "node_tk_U7GMCRkCG4FoOvusrng_thread_pool_write_rejected": 0, + "node_tk_U7GMCRkCG4FoOvusrng_transport_rx_count": 2167879292, + "node_tk_U7GMCRkCG4FoOvusrng_transport_rx_size_in_bytes": 4905919297323, + "node_tk_U7GMCRkCG4FoOvusrng_transport_tx_count": 2167879293, + "node_tk_U7GMCRkCG4FoOvusrng_transport_tx_size_in_bytes": 2964638852652, + }, + }, + "v842: local node stats": { + prepare: func() *Elasticsearch { + es := New() + es.DoNodeStats = true + es.DoClusterHealth = false + es.DoClusterStats = false + es.DoIndicesStats = false + return es + }, + wantCharts: len(nodeChartsTmpl), + wantCollected: map[string]int64{ + "node_Klg1CjgMTouentQcJlRGuA_breakers_accounting_tripped": 0, + "node_Klg1CjgMTouentQcJlRGuA_breakers_fielddata_tripped": 0, + "node_Klg1CjgMTouentQcJlRGuA_breakers_in_flight_requests_tripped": 0, + "node_Klg1CjgMTouentQcJlRGuA_breakers_model_inference_tripped": 0, + "node_Klg1CjgMTouentQcJlRGuA_breakers_parent_tripped": 0, + "node_Klg1CjgMTouentQcJlRGuA_breakers_request_tripped": 0, + "node_Klg1CjgMTouentQcJlRGuA_http_current_open": 73, + "node_Klg1CjgMTouentQcJlRGuA_indices_fielddata_evictions": 0, + "node_Klg1CjgMTouentQcJlRGuA_indices_fielddata_memory_size_in_bytes": 600, + "node_Klg1CjgMTouentQcJlRGuA_indices_flush_total": 35134, + "node_Klg1CjgMTouentQcJlRGuA_indices_flush_total_time_in_millis": 22213090, + "node_Klg1CjgMTouentQcJlRGuA_indices_indexing_index_current": 1, + "node_Klg1CjgMTouentQcJlRGuA_indices_indexing_index_time_in_millis": 1100149051, + "node_Klg1CjgMTouentQcJlRGuA_indices_indexing_index_total": 3667793202, + "node_Klg1CjgMTouentQcJlRGuA_indices_refresh_total": 7721472, + "node_Klg1CjgMTouentQcJlRGuA_indices_refresh_total_time_in_millis": 94304142, + "node_Klg1CjgMTouentQcJlRGuA_indices_search_fetch_current": 0, + "node_Klg1CjgMTouentQcJlRGuA_indices_search_fetch_time_in_millis": 21316820, + "node_Klg1CjgMTouentQcJlRGuA_indices_search_fetch_total": 42645288, + "node_Klg1CjgMTouentQcJlRGuA_indices_search_query_current": 0, + "node_Klg1CjgMTouentQcJlRGuA_indices_search_query_time_in_millis": 51265805, + "node_Klg1CjgMTouentQcJlRGuA_indices_search_query_total": 166823028, + "node_Klg1CjgMTouentQcJlRGuA_indices_segments_count": 307, + "node_Klg1CjgMTouentQcJlRGuA_indices_segments_doc_values_memory_in_bytes": 0, + "node_Klg1CjgMTouentQcJlRGuA_indices_segments_fixed_bit_set_memory_in_bytes": 2008, + "node_Klg1CjgMTouentQcJlRGuA_indices_segments_index_writer_memory_in_bytes": 240481008, + "node_Klg1CjgMTouentQcJlRGuA_indices_segments_memory_in_bytes": 0, + "node_Klg1CjgMTouentQcJlRGuA_indices_segments_norms_memory_in_bytes": 0, + "node_Klg1CjgMTouentQcJlRGuA_indices_segments_points_memory_in_bytes": 0, + "node_Klg1CjgMTouentQcJlRGuA_indices_segments_stored_fields_memory_in_bytes": 0, + "node_Klg1CjgMTouentQcJlRGuA_indices_segments_term_vectors_memory_in_bytes": 0, + "node_Klg1CjgMTouentQcJlRGuA_indices_segments_terms_memory_in_bytes": 0, + "node_Klg1CjgMTouentQcJlRGuA_indices_segments_version_map_memory_in_bytes": 44339216, + "node_Klg1CjgMTouentQcJlRGuA_indices_translog_operations": 362831, + "node_Klg1CjgMTouentQcJlRGuA_indices_translog_size_in_bytes": 453491882, + "node_Klg1CjgMTouentQcJlRGuA_indices_translog_uncommitted_operations": 362831, + "node_Klg1CjgMTouentQcJlRGuA_indices_translog_uncommitted_size_in_bytes": 453491882, + "node_Klg1CjgMTouentQcJlRGuA_jvm_buffer_pools_direct_count": 94, + "node_Klg1CjgMTouentQcJlRGuA_jvm_buffer_pools_direct_total_capacity_in_bytes": 4654848, + "node_Klg1CjgMTouentQcJlRGuA_jvm_buffer_pools_direct_used_in_bytes": 4654850, + "node_Klg1CjgMTouentQcJlRGuA_jvm_buffer_pools_mapped_count": 844, + "node_Klg1CjgMTouentQcJlRGuA_jvm_buffer_pools_mapped_total_capacity_in_bytes": 103411995802, + "node_Klg1CjgMTouentQcJlRGuA_jvm_buffer_pools_mapped_used_in_bytes": 103411995802, + "node_Klg1CjgMTouentQcJlRGuA_jvm_gc_collectors_old_collection_count": 0, + "node_Klg1CjgMTouentQcJlRGuA_jvm_gc_collectors_old_collection_time_in_millis": 0, + "node_Klg1CjgMTouentQcJlRGuA_jvm_gc_collectors_young_collection_count": 78661, + "node_Klg1CjgMTouentQcJlRGuA_jvm_gc_collectors_young_collection_time_in_millis": 6014901, + "node_Klg1CjgMTouentQcJlRGuA_jvm_mem_heap_committed_in_bytes": 7864320000, + "node_Klg1CjgMTouentQcJlRGuA_jvm_mem_heap_used_in_bytes": 4337402488, + "node_Klg1CjgMTouentQcJlRGuA_jvm_mem_heap_used_percent": 55, + "node_Klg1CjgMTouentQcJlRGuA_process_max_file_descriptors": 1048576, + "node_Klg1CjgMTouentQcJlRGuA_process_open_file_descriptors": 1149, + "node_Klg1CjgMTouentQcJlRGuA_thread_pool_analyze_queue": 0, + "node_Klg1CjgMTouentQcJlRGuA_thread_pool_analyze_rejected": 0, + "node_Klg1CjgMTouentQcJlRGuA_thread_pool_fetch_shard_started_queue": 0, + "node_Klg1CjgMTouentQcJlRGuA_thread_pool_fetch_shard_started_rejected": 0, + "node_Klg1CjgMTouentQcJlRGuA_thread_pool_fetch_shard_store_queue": 0, + "node_Klg1CjgMTouentQcJlRGuA_thread_pool_fetch_shard_store_rejected": 0, + "node_Klg1CjgMTouentQcJlRGuA_thread_pool_flush_queue": 0, + "node_Klg1CjgMTouentQcJlRGuA_thread_pool_flush_rejected": 0, + "node_Klg1CjgMTouentQcJlRGuA_thread_pool_force_merge_queue": 0, + "node_Klg1CjgMTouentQcJlRGuA_thread_pool_force_merge_rejected": 0, + "node_Klg1CjgMTouentQcJlRGuA_thread_pool_generic_queue": 0, + "node_Klg1CjgMTouentQcJlRGuA_thread_pool_generic_rejected": 0, + "node_Klg1CjgMTouentQcJlRGuA_thread_pool_get_queue": 0, + "node_Klg1CjgMTouentQcJlRGuA_thread_pool_get_rejected": 0, + "node_Klg1CjgMTouentQcJlRGuA_thread_pool_listener_queue": 0, + "node_Klg1CjgMTouentQcJlRGuA_thread_pool_listener_rejected": 0, + "node_Klg1CjgMTouentQcJlRGuA_thread_pool_management_queue": 0, + "node_Klg1CjgMTouentQcJlRGuA_thread_pool_management_rejected": 0, + "node_Klg1CjgMTouentQcJlRGuA_thread_pool_refresh_queue": 0, + "node_Klg1CjgMTouentQcJlRGuA_thread_pool_refresh_rejected": 0, + "node_Klg1CjgMTouentQcJlRGuA_thread_pool_search_queue": 0, + "node_Klg1CjgMTouentQcJlRGuA_thread_pool_search_rejected": 0, + "node_Klg1CjgMTouentQcJlRGuA_thread_pool_search_throttled_queue": 0, + "node_Klg1CjgMTouentQcJlRGuA_thread_pool_search_throttled_rejected": 0, + "node_Klg1CjgMTouentQcJlRGuA_thread_pool_snapshot_queue": 0, + "node_Klg1CjgMTouentQcJlRGuA_thread_pool_snapshot_rejected": 0, + "node_Klg1CjgMTouentQcJlRGuA_thread_pool_warmer_queue": 0, + "node_Klg1CjgMTouentQcJlRGuA_thread_pool_warmer_rejected": 0, + "node_Klg1CjgMTouentQcJlRGuA_thread_pool_write_queue": 0, + "node_Klg1CjgMTouentQcJlRGuA_thread_pool_write_rejected": 0, + "node_Klg1CjgMTouentQcJlRGuA_transport_rx_count": 1300468666, + "node_Klg1CjgMTouentQcJlRGuA_transport_rx_size_in_bytes": 1789647854011, + "node_Klg1CjgMTouentQcJlRGuA_transport_tx_count": 1300468665, + "node_Klg1CjgMTouentQcJlRGuA_transport_tx_size_in_bytes": 2927853534431, + }, + }, + "v842: only cluster_health": { + prepare: func() *Elasticsearch { + es := New() + es.DoNodeStats = false + es.DoClusterHealth = true + es.DoClusterStats = false + es.DoIndicesStats = false + return es + }, + wantCharts: len(clusterHealthChartsTmpl), + wantCollected: map[string]int64{ + "cluster_active_primary_shards": 97, + "cluster_active_shards": 194, + "cluster_active_shards_percent_as_number": 100, + "cluster_delayed_unassigned_shards": 0, + "cluster_initializing_shards": 0, + "cluster_number_of_data_nodes": 2, + "cluster_number_of_in_flight_fetch": 0, + "cluster_number_of_nodes": 3, + "cluster_number_of_pending_tasks": 0, + "cluster_relocating_shards": 0, + "cluster_status_green": 1, + "cluster_status_red": 0, + "cluster_status_yellow": 0, + "cluster_unassigned_shards": 0, + }, + }, + "v842: only cluster_stats": { + prepare: func() *Elasticsearch { + es := New() + es.DoNodeStats = false + es.DoClusterHealth = false + es.DoClusterStats = true + es.DoIndicesStats = false + return es + }, + wantCharts: len(clusterStatsChartsTmpl), + wantCollected: map[string]int64{ + "cluster_indices_count": 97, + "cluster_indices_docs_count": 402750703, + "cluster_indices_query_cache_hit_count": 96838726, + "cluster_indices_query_cache_miss_count": 587768226, + "cluster_indices_shards_primaries": 97, + "cluster_indices_shards_replication": 1, + "cluster_indices_shards_total": 194, + "cluster_indices_store_size_in_bytes": 380826136962, + "cluster_nodes_count_coordinating_only": 0, + "cluster_nodes_count_data": 0, + "cluster_nodes_count_data_cold": 0, + "cluster_nodes_count_data_content": 2, + "cluster_nodes_count_data_frozen": 0, + "cluster_nodes_count_data_hot": 2, + "cluster_nodes_count_data_warm": 0, + "cluster_nodes_count_ingest": 2, + "cluster_nodes_count_master": 3, + "cluster_nodes_count_ml": 0, + "cluster_nodes_count_remote_cluster_client": 2, + "cluster_nodes_count_total": 3, + "cluster_nodes_count_transform": 2, + "cluster_nodes_count_voting_only": 1, + }, + }, + "v842: only indices_stats": { + prepare: func() *Elasticsearch { + es := New() + es.DoNodeStats = false + es.DoClusterHealth = false + es.DoClusterStats = false + es.DoIndicesStats = true + return es + }, + wantCharts: len(nodeIndexChartsTmpl) * 3, + wantCollected: map[string]int64{ + "node_index_my-index-000001_stats_docs_count": 1, + "node_index_my-index-000001_stats_health_green": 0, + "node_index_my-index-000001_stats_health_red": 0, + "node_index_my-index-000001_stats_health_yellow": 1, + "node_index_my-index-000001_stats_shards_count": 1, + "node_index_my-index-000001_stats_store_size_in_bytes": 208, + "node_index_my-index-000002_stats_docs_count": 1, + "node_index_my-index-000002_stats_health_green": 0, + "node_index_my-index-000002_stats_health_red": 0, + "node_index_my-index-000002_stats_health_yellow": 1, + "node_index_my-index-000002_stats_shards_count": 1, + "node_index_my-index-000002_stats_store_size_in_bytes": 208, + "node_index_my-index-000003_stats_docs_count": 1, + "node_index_my-index-000003_stats_health_green": 0, + "node_index_my-index-000003_stats_health_red": 0, + "node_index_my-index-000003_stats_health_yellow": 1, + "node_index_my-index-000003_stats_shards_count": 1, + "node_index_my-index-000003_stats_store_size_in_bytes": 208, + }, + }, + } + + for name, test := range tests { + t.Run(name, func(t *testing.T) { + es, cleanup := prepareElasticsearch(t, test.prepare) + defer cleanup() + + var mx map[string]int64 + for i := 0; i < 10; i++ { + mx = es.Collect() + } + + //m := mx + //l := make([]string, 0) + //for k := range m { + // l = append(l, k) + //} + //sort.Strings(l) + //for _, value := range l { + // fmt.Println(fmt.Sprintf("\"%s\": %d,", value, m[value])) + //} + //return + + assert.Equal(t, test.wantCollected, mx) + assert.Len(t, *es.Charts(), test.wantCharts) + ensureCollectedHasAllChartsDimsVarsIDs(t, es, mx) + }) + } +} + +func ensureCollectedHasAllChartsDimsVarsIDs(t *testing.T, es *Elasticsearch, collected map[string]int64) { + for _, chart := range *es.Charts() { + if chart.Obsolete { + continue + } + for _, dim := range chart.Dims { + _, ok := collected[dim.ID] + assert.Truef(t, ok, "collected metrics has no data for dim '%s' chart '%s'", dim.ID, chart.ID) + } + for _, v := range chart.Vars { + _, ok := collected[v.ID] + assert.Truef(t, ok, "collected metrics has no data for var '%s' chart '%s'", v.ID, chart.ID) + } + } +} + +func prepareElasticsearch(t *testing.T, createES func() *Elasticsearch) (es *Elasticsearch, cleanup func()) { + t.Helper() + srv := prepareElasticsearchEndpoint() + + es = createES() + es.URL = srv.URL + require.NoError(t, es.Init()) + + return es, srv.Close +} + +func prepareElasticsearchValidData(t *testing.T) (es *Elasticsearch, cleanup func()) { + return prepareElasticsearch(t, New) +} + +func prepareElasticsearchInvalidData(t *testing.T) (*Elasticsearch, func()) { + t.Helper() + srv := httptest.NewServer(http.HandlerFunc( + func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte("hello and\n goodbye")) + })) + es := New() + es.URL = srv.URL + require.NoError(t, es.Init()) + + return es, srv.Close +} + +func prepareElasticsearch404(t *testing.T) (*Elasticsearch, func()) { + t.Helper() + srv := httptest.NewServer(http.HandlerFunc( + func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusNotFound) + })) + es := New() + es.URL = srv.URL + require.NoError(t, es.Init()) + + return es, srv.Close +} + +func prepareElasticsearchConnectionRefused(t *testing.T) (*Elasticsearch, func()) { + t.Helper() + es := New() + es.URL = "http://127.0.0.1:38001" + require.NoError(t, es.Init()) + + return es, func() {} +} + +func prepareElasticsearchEndpoint() *httptest.Server { + return httptest.NewServer(http.HandlerFunc( + func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case urlPathNodesStats: + _, _ = w.Write(dataVer842NodesStats) + case urlPathLocalNodeStats: + _, _ = w.Write(dataVer842NodesLocalStats) + case urlPathClusterHealth: + _, _ = w.Write(dataVer842ClusterHealth) + case urlPathClusterStats: + _, _ = w.Write(dataVer842ClusterStats) + case urlPathIndicesStats: + _, _ = w.Write(dataVer842CatIndicesStats) + case "/": + _, _ = w.Write(dataVer842Info) + default: + w.WriteHeader(http.StatusNotFound) + } + })) +} diff --git a/src/go/collectors/go.d.plugin/modules/elasticsearch/init.go b/src/go/collectors/go.d.plugin/modules/elasticsearch/init.go new file mode 100644 index 000000000..955d9c3a2 --- /dev/null +++ b/src/go/collectors/go.d.plugin/modules/elasticsearch/init.go @@ -0,0 +1,27 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +package elasticsearch + +import ( + "errors" + "net/http" + + "github.com/netdata/netdata/go/go.d.plugin/pkg/web" +) + +func (es *Elasticsearch) validateConfig() error { + if es.URL == "" { + return errors.New("URL not set") + } + if !(es.DoNodeStats || es.DoClusterHealth || es.DoClusterStats || es.DoIndicesStats) { + return errors.New("all API calls are disabled") + } + if _, err := web.NewHTTPRequest(es.Request); err != nil { + return err + } + return nil +} + +func (es *Elasticsearch) initHTTPClient() (*http.Client, error) { + return web.NewHTTPClient(es.Client) +} diff --git a/src/go/collectors/go.d.plugin/modules/elasticsearch/integrations/elasticsearch.md b/src/go/collectors/go.d.plugin/modules/elasticsearch/integrations/elasticsearch.md new file mode 100644 index 000000000..9978bf073 --- /dev/null +++ b/src/go/collectors/go.d.plugin/modules/elasticsearch/integrations/elasticsearch.md @@ -0,0 +1,343 @@ +<!--startmeta +custom_edit_url: "https://github.com/netdata/netdata/edit/master/src/go/collectors/go.d.plugin/modules/elasticsearch/integrations/elasticsearch.md" +meta_yaml: "https://github.com/netdata/netdata/edit/master/src/go/collectors/go.d.plugin/modules/elasticsearch/metadata.yaml" +sidebar_label: "Elasticsearch" +learn_status: "Published" +learn_rel_path: "Collecting Metrics/Search Engines" +most_popular: True +message: "DO NOT EDIT THIS FILE DIRECTLY, IT IS GENERATED BY THE COLLECTOR'S metadata.yaml FILE" +endmeta--> + +# Elasticsearch + + +<img src="https://netdata.cloud/img/elasticsearch.svg" width="150"/> + + +Plugin: go.d.plugin +Module: elasticsearch + +<img src="https://img.shields.io/badge/maintained%20by-Netdata-%2300ab44" /> + +## Overview + +This collector monitors the performance and health of the Elasticsearch cluster. + + +It uses [Cluster APIs](https://www.elastic.co/guide/en/elasticsearch/reference/current/cluster.html) to collect metrics. + +Used endpoints: + +| Endpoint | Description | API | +|------------------------|----------------------|-------------------------------------------------------------------------------------------------------------| +| `/` | Node info | | +| `/_nodes/stats` | Nodes metrics | [Nodes stats API](https://www.elastic.co/guide/en/elasticsearch/reference/current/cluster-nodes-stats.html) | +| `/_nodes/_local/stats` | Local node metrics | [Nodes stats API](https://www.elastic.co/guide/en/elasticsearch/reference/current/cluster-nodes-stats.html) | +| `/_cluster/health` | Cluster health stats | [Cluster health API](https://www.elastic.co/guide/en/elasticsearch/reference/current/cluster-health.html) | +| `/_cluster/stats` | Cluster metrics | [Cluster stats API](https://www.elastic.co/guide/en/elasticsearch/reference/current/cluster-stats.html) | + + +This collector is supported on all platforms. + +This collector supports collecting metrics from multiple instances of this integration, including remote instances. + + +### Default Behavior + +#### Auto-Detection + +By default, it detects instances running on localhost by attempting to connect to port 9200: + +- http://127.0.0.1:9200 +- https://127.0.0.1:9200 + + +#### Limits + +By default, this collector monitors only the node it is connected to. To monitor all cluster nodes, set the `cluster_mode` configuration option to `yes`. + + +#### Performance Impact + +The default configuration for this integration is not expected to impose a significant performance impact on the system. + + +## Metrics + +Metrics grouped by *scope*. + +The scope defines the instance that the metric belongs to. An instance is uniquely identified by a set of labels. + + + +### Per node + +These metrics refer to the cluster node. + +Labels: + +| Label | Description | +|:-----------|:----------------| +| cluster_name | Name of the cluster. Based on the [Cluster name setting](https://www.elastic.co/guide/en/elasticsearch/reference/current/important-settings.html#cluster-name). | +| node_name | Human-readable identifier for the node. Based on the [Node name setting](https://www.elastic.co/guide/en/elasticsearch/reference/current/important-settings.html#node-name). | +| host | Network host for the node, based on the [Network host setting](https://www.elastic.co/guide/en/elasticsearch/reference/current/important-settings.html#network.host). | + +Metrics: + +| Metric | Dimensions | Unit | +|:------|:----------|:----| +| elasticsearch.node_indices_indexing | index | operations/s | +| elasticsearch.node_indices_indexing_current | index | operations | +| elasticsearch.node_indices_indexing_time | index | milliseconds | +| elasticsearch.node_indices_search | queries, fetches | operations/s | +| elasticsearch.node_indices_search_current | queries, fetches | operations | +| elasticsearch.node_indices_search_time | queries, fetches | milliseconds | +| elasticsearch.node_indices_refresh | refresh | operations/s | +| elasticsearch.node_indices_refresh_time | refresh | milliseconds | +| elasticsearch.node_indices_flush | flush | operations/s | +| elasticsearch.node_indices_flush_time | flush | milliseconds | +| elasticsearch.node_indices_fielddata_memory_usage | used | bytes | +| elasticsearch.node_indices_fielddata_evictions | evictions | operations/s | +| elasticsearch.node_indices_segments_count | segments | segments | +| elasticsearch.node_indices_segments_memory_usage_total | used | bytes | +| elasticsearch.node_indices_segments_memory_usage | terms, stored_fields, term_vectors, norms, points, doc_values, index_writer, version_map, fixed_bit_set | bytes | +| elasticsearch.node_indices_translog_operations | total, uncommitted | operations | +| elasticsearch.node_indices_translog_size | total, uncommitted | bytes | +| elasticsearch.node_file_descriptors | open | fd | +| elasticsearch.node_jvm_heap | inuse | percentage | +| elasticsearch.node_jvm_heap_bytes | committed, used | bytes | +| elasticsearch.node_jvm_buffer_pools_count | direct, mapped | pools | +| elasticsearch.node_jvm_buffer_pool_direct_memory | total, used | bytes | +| elasticsearch.node_jvm_buffer_pool_mapped_memory | total, used | bytes | +| elasticsearch.node_jvm_gc_count | young, old | gc/s | +| elasticsearch.node_jvm_gc_time | young, old | milliseconds | +| elasticsearch.node_thread_pool_queued | generic, search, search_throttled, get, analyze, write, snapshot, warmer, refresh, listener, fetch_shard_started, fetch_shard_store, flush, force_merge, management | threads | +| elasticsearch.node_thread_pool_rejected | generic, search, search_throttled, get, analyze, write, snapshot, warmer, refresh, listener, fetch_shard_started, fetch_shard_store, flush, force_merge, management | threads | +| elasticsearch.node_cluster_communication_packets | received, sent | pps | +| elasticsearch.node_cluster_communication_traffic | received, sent | bytes/s | +| elasticsearch.node_http_connections | open | connections | +| elasticsearch.node_breakers_trips | requests, fielddata, in_flight_requests, model_inference, accounting, parent | trips/s | + +### Per cluster + +These metrics refer to the cluster. + +Labels: + +| Label | Description | +|:-----------|:----------------| +| cluster_name | Name of the cluster. Based on the [Cluster name setting](https://www.elastic.co/guide/en/elasticsearch/reference/current/important-settings.html#cluster-name). | + +Metrics: + +| Metric | Dimensions | Unit | +|:------|:----------|:----| +| elasticsearch.cluster_health_status | green, yellow, red | status | +| elasticsearch.cluster_number_of_nodes | nodes, data_nodes | nodes | +| elasticsearch.cluster_shards_count | active_primary, active, relocating, initializing, unassigned, delayed_unaasigned | shards | +| elasticsearch.cluster_pending_tasks | pending | tasks | +| elasticsearch.cluster_number_of_in_flight_fetch | in_flight_fetch | fetches | +| elasticsearch.cluster_indices_count | indices | indices | +| elasticsearch.cluster_indices_shards_count | total, primaries, replication | shards | +| elasticsearch.cluster_indices_docs_count | docs | docs | +| elasticsearch.cluster_indices_store_size | size | bytes | +| elasticsearch.cluster_indices_query_cache | hit, miss | events/s | +| elasticsearch.cluster_nodes_by_role_count | coordinating_only, data, data_cold, data_content, data_frozen, data_hot, data_warm, ingest, master, ml, remote_cluster_client, voting_only | nodes | + +### Per index + +These metrics refer to the index. + +Labels: + +| Label | Description | +|:-----------|:----------------| +| cluster_name | Name of the cluster. Based on the [Cluster name setting](https://www.elastic.co/guide/en/elasticsearch/reference/current/important-settings.html#cluster-name). | +| index | Name of the index. | + +Metrics: + +| Metric | Dimensions | Unit | +|:------|:----------|:----| +| elasticsearch.node_index_health | green, yellow, red | status | +| elasticsearch.node_index_shards_count | shards | shards | +| elasticsearch.node_index_docs_count | docs | docs | +| elasticsearch.node_index_store_size | store_size | bytes | + + + +## Alerts + + +The following alerts are available: + +| Alert name | On metric | Description | +|:------------|:----------|:------------| +| [ elasticsearch_node_indices_search_time_query ](https://github.com/netdata/netdata/blob/master/src/health/health.d/elasticsearch.conf) | elasticsearch.node_indices_search_time | search performance is degraded, queries run slowly. | +| [ elasticsearch_node_indices_search_time_fetch ](https://github.com/netdata/netdata/blob/master/src/health/health.d/elasticsearch.conf) | elasticsearch.node_indices_search_time | search performance is degraded, fetches run slowly. | +| [ elasticsearch_cluster_health_status_red ](https://github.com/netdata/netdata/blob/master/src/health/health.d/elasticsearch.conf) | elasticsearch.cluster_health_status | cluster health status is red. | +| [ elasticsearch_cluster_health_status_yellow ](https://github.com/netdata/netdata/blob/master/src/health/health.d/elasticsearch.conf) | elasticsearch.cluster_health_status | cluster health status is yellow. | +| [ elasticsearch_node_index_health_red ](https://github.com/netdata/netdata/blob/master/src/health/health.d/elasticsearch.conf) | elasticsearch.node_index_health | node index $label:index health status is red. | + + +## Setup + +### Prerequisites + +No action required. + +### Configuration + +#### File + +The configuration file name for this integration is `go.d/elasticsearch.conf`. + + +You can edit the configuration file using the `edit-config` script from the +Netdata [config directory](https://github.com/netdata/netdata/blob/master/docs/netdata-agent/configuration.md#the-netdata-config-directory). + +```bash +cd /etc/netdata 2>/dev/null || cd /opt/netdata/etc/netdata +sudo ./edit-config go.d/elasticsearch.conf +``` +#### Options + +The following options can be defined globally: update_every, autodetection_retry. + + +<details><summary>Config options</summary> + +| Name | Description | Default | Required | +|:----|:-----------|:-------|:--------:| +| update_every | Data collection frequency. | 5 | no | +| autodetection_retry | Recheck interval in seconds. Zero means no recheck will be scheduled. | 0 | no | +| url | Server URL. | http://127.0.0.1:9200 | yes | +| cluster_mode | Controls whether to collect metrics for all nodes in the cluster or only for the local node. | false | no | +| collect_node_stats | Controls whether to collect nodes metrics. | true | no | +| collect_cluster_health | Controls whether to collect cluster health metrics. | true | no | +| collect_cluster_stats | Controls whether to collect cluster stats metrics. | true | no | +| collect_indices_stats | Controls whether to collect indices metrics. | false | no | +| timeout | HTTP request timeout. | 2 | no | +| username | Username for basic HTTP authentication. | | no | +| password | Password for basic HTTP authentication. | | no | +| proxy_url | Proxy URL. | | no | +| proxy_username | Username for proxy basic HTTP authentication. | | no | +| proxy_password | Password for proxy basic HTTP authentication. | | no | +| method | HTTP request method. | GET | no | +| body | HTTP request body. | | no | +| headers | HTTP request headers. | | no | +| not_follow_redirects | Redirect handling policy. Controls whether the client follows redirects. | no | no | +| tls_skip_verify | Server certificate chain and hostname validation policy. Controls whether the client performs this check. | no | no | +| tls_ca | Certification authority that the client uses when verifying the server's certificates. | | no | +| tls_cert | Client TLS certificate. | | no | +| tls_key | Client TLS key. | | no | + +</details> + +#### Examples + +##### Basic single node mode + +A basic example configuration. + +```yaml +jobs: + - name: local + url: http://127.0.0.1:9200 + +``` +##### Cluster mode + +Cluster mode example configuration. + +<details><summary>Config</summary> + +```yaml +jobs: + - name: local + url: http://127.0.0.1:9200 + cluster_mode: yes + +``` +</details> + +##### HTTP authentication + +Basic HTTP authentication. + +<details><summary>Config</summary> + +```yaml +jobs: + - name: local + url: http://127.0.0.1:9200 + username: username + password: password + +``` +</details> + +##### HTTPS with self-signed certificate + +Elasticsearch with enabled HTTPS and self-signed certificate. + +<details><summary>Config</summary> + +```yaml +jobs: + - name: local + url: https://127.0.0.1:9200 + tls_skip_verify: yes + +``` +</details> + +##### Multi-instance + +> **Note**: When you define multiple jobs, their names must be unique. + +Collecting metrics from local and remote instances. + + +<details><summary>Config</summary> + +```yaml +jobs: + - name: local + url: http://127.0.0.1:9200 + + - name: remote + url: http://192.0.2.1:9200 + +``` +</details> + + + +## Troubleshooting + +### Debug Mode + +To troubleshoot issues with the `elasticsearch` collector, run the `go.d.plugin` with the debug option enabled. The output +should give you clues as to why the collector isn't working. + +- Navigate to the `plugins.d` directory, usually at `/usr/libexec/netdata/plugins.d/`. If that's not the case on + your system, open `netdata.conf` and look for the `plugins` setting under `[directories]`. + + ```bash + cd /usr/libexec/netdata/plugins.d/ + ``` + +- Switch to the `netdata` user. + + ```bash + sudo -u netdata -s + ``` + +- Run the `go.d.plugin` to debug the collector: + + ```bash + ./go.d.plugin -d -m elasticsearch + ``` + + diff --git a/src/go/collectors/go.d.plugin/modules/elasticsearch/integrations/opensearch.md b/src/go/collectors/go.d.plugin/modules/elasticsearch/integrations/opensearch.md new file mode 100644 index 000000000..c0755e4e4 --- /dev/null +++ b/src/go/collectors/go.d.plugin/modules/elasticsearch/integrations/opensearch.md @@ -0,0 +1,343 @@ +<!--startmeta +custom_edit_url: "https://github.com/netdata/netdata/edit/master/src/go/collectors/go.d.plugin/modules/elasticsearch/integrations/opensearch.md" +meta_yaml: "https://github.com/netdata/netdata/edit/master/src/go/collectors/go.d.plugin/modules/elasticsearch/metadata.yaml" +sidebar_label: "OpenSearch" +learn_status: "Published" +learn_rel_path: "Collecting Metrics/Search Engines" +most_popular: True +message: "DO NOT EDIT THIS FILE DIRECTLY, IT IS GENERATED BY THE COLLECTOR'S metadata.yaml FILE" +endmeta--> + +# OpenSearch + + +<img src="https://netdata.cloud/img/opensearch.svg" width="150"/> + + +Plugin: go.d.plugin +Module: elasticsearch + +<img src="https://img.shields.io/badge/maintained%20by-Netdata-%2300ab44" /> + +## Overview + +This collector monitors the performance and health of the Elasticsearch cluster. + + +It uses [Cluster APIs](https://www.elastic.co/guide/en/elasticsearch/reference/current/cluster.html) to collect metrics. + +Used endpoints: + +| Endpoint | Description | API | +|------------------------|----------------------|-------------------------------------------------------------------------------------------------------------| +| `/` | Node info | | +| `/_nodes/stats` | Nodes metrics | [Nodes stats API](https://www.elastic.co/guide/en/elasticsearch/reference/current/cluster-nodes-stats.html) | +| `/_nodes/_local/stats` | Local node metrics | [Nodes stats API](https://www.elastic.co/guide/en/elasticsearch/reference/current/cluster-nodes-stats.html) | +| `/_cluster/health` | Cluster health stats | [Cluster health API](https://www.elastic.co/guide/en/elasticsearch/reference/current/cluster-health.html) | +| `/_cluster/stats` | Cluster metrics | [Cluster stats API](https://www.elastic.co/guide/en/elasticsearch/reference/current/cluster-stats.html) | + + +This collector is supported on all platforms. + +This collector supports collecting metrics from multiple instances of this integration, including remote instances. + + +### Default Behavior + +#### Auto-Detection + +By default, it detects instances running on localhost by attempting to connect to port 9200: + +- http://127.0.0.1:9200 +- https://127.0.0.1:9200 + + +#### Limits + +By default, this collector monitors only the node it is connected to. To monitor all cluster nodes, set the `cluster_mode` configuration option to `yes`. + + +#### Performance Impact + +The default configuration for this integration is not expected to impose a significant performance impact on the system. + + +## Metrics + +Metrics grouped by *scope*. + +The scope defines the instance that the metric belongs to. An instance is uniquely identified by a set of labels. + + + +### Per node + +These metrics refer to the cluster node. + +Labels: + +| Label | Description | +|:-----------|:----------------| +| cluster_name | Name of the cluster. Based on the [Cluster name setting](https://www.elastic.co/guide/en/elasticsearch/reference/current/important-settings.html#cluster-name). | +| node_name | Human-readable identifier for the node. Based on the [Node name setting](https://www.elastic.co/guide/en/elasticsearch/reference/current/important-settings.html#node-name). | +| host | Network host for the node, based on the [Network host setting](https://www.elastic.co/guide/en/elasticsearch/reference/current/important-settings.html#network.host). | + +Metrics: + +| Metric | Dimensions | Unit | +|:------|:----------|:----| +| elasticsearch.node_indices_indexing | index | operations/s | +| elasticsearch.node_indices_indexing_current | index | operations | +| elasticsearch.node_indices_indexing_time | index | milliseconds | +| elasticsearch.node_indices_search | queries, fetches | operations/s | +| elasticsearch.node_indices_search_current | queries, fetches | operations | +| elasticsearch.node_indices_search_time | queries, fetches | milliseconds | +| elasticsearch.node_indices_refresh | refresh | operations/s | +| elasticsearch.node_indices_refresh_time | refresh | milliseconds | +| elasticsearch.node_indices_flush | flush | operations/s | +| elasticsearch.node_indices_flush_time | flush | milliseconds | +| elasticsearch.node_indices_fielddata_memory_usage | used | bytes | +| elasticsearch.node_indices_fielddata_evictions | evictions | operations/s | +| elasticsearch.node_indices_segments_count | segments | segments | +| elasticsearch.node_indices_segments_memory_usage_total | used | bytes | +| elasticsearch.node_indices_segments_memory_usage | terms, stored_fields, term_vectors, norms, points, doc_values, index_writer, version_map, fixed_bit_set | bytes | +| elasticsearch.node_indices_translog_operations | total, uncommitted | operations | +| elasticsearch.node_indices_translog_size | total, uncommitted | bytes | +| elasticsearch.node_file_descriptors | open | fd | +| elasticsearch.node_jvm_heap | inuse | percentage | +| elasticsearch.node_jvm_heap_bytes | committed, used | bytes | +| elasticsearch.node_jvm_buffer_pools_count | direct, mapped | pools | +| elasticsearch.node_jvm_buffer_pool_direct_memory | total, used | bytes | +| elasticsearch.node_jvm_buffer_pool_mapped_memory | total, used | bytes | +| elasticsearch.node_jvm_gc_count | young, old | gc/s | +| elasticsearch.node_jvm_gc_time | young, old | milliseconds | +| elasticsearch.node_thread_pool_queued | generic, search, search_throttled, get, analyze, write, snapshot, warmer, refresh, listener, fetch_shard_started, fetch_shard_store, flush, force_merge, management | threads | +| elasticsearch.node_thread_pool_rejected | generic, search, search_throttled, get, analyze, write, snapshot, warmer, refresh, listener, fetch_shard_started, fetch_shard_store, flush, force_merge, management | threads | +| elasticsearch.node_cluster_communication_packets | received, sent | pps | +| elasticsearch.node_cluster_communication_traffic | received, sent | bytes/s | +| elasticsearch.node_http_connections | open | connections | +| elasticsearch.node_breakers_trips | requests, fielddata, in_flight_requests, model_inference, accounting, parent | trips/s | + +### Per cluster + +These metrics refer to the cluster. + +Labels: + +| Label | Description | +|:-----------|:----------------| +| cluster_name | Name of the cluster. Based on the [Cluster name setting](https://www.elastic.co/guide/en/elasticsearch/reference/current/important-settings.html#cluster-name). | + +Metrics: + +| Metric | Dimensions | Unit | +|:------|:----------|:----| +| elasticsearch.cluster_health_status | green, yellow, red | status | +| elasticsearch.cluster_number_of_nodes | nodes, data_nodes | nodes | +| elasticsearch.cluster_shards_count | active_primary, active, relocating, initializing, unassigned, delayed_unaasigned | shards | +| elasticsearch.cluster_pending_tasks | pending | tasks | +| elasticsearch.cluster_number_of_in_flight_fetch | in_flight_fetch | fetches | +| elasticsearch.cluster_indices_count | indices | indices | +| elasticsearch.cluster_indices_shards_count | total, primaries, replication | shards | +| elasticsearch.cluster_indices_docs_count | docs | docs | +| elasticsearch.cluster_indices_store_size | size | bytes | +| elasticsearch.cluster_indices_query_cache | hit, miss | events/s | +| elasticsearch.cluster_nodes_by_role_count | coordinating_only, data, data_cold, data_content, data_frozen, data_hot, data_warm, ingest, master, ml, remote_cluster_client, voting_only | nodes | + +### Per index + +These metrics refer to the index. + +Labels: + +| Label | Description | +|:-----------|:----------------| +| cluster_name | Name of the cluster. Based on the [Cluster name setting](https://www.elastic.co/guide/en/elasticsearch/reference/current/important-settings.html#cluster-name). | +| index | Name of the index. | + +Metrics: + +| Metric | Dimensions | Unit | +|:------|:----------|:----| +| elasticsearch.node_index_health | green, yellow, red | status | +| elasticsearch.node_index_shards_count | shards | shards | +| elasticsearch.node_index_docs_count | docs | docs | +| elasticsearch.node_index_store_size | store_size | bytes | + + + +## Alerts + + +The following alerts are available: + +| Alert name | On metric | Description | +|:------------|:----------|:------------| +| [ elasticsearch_node_indices_search_time_query ](https://github.com/netdata/netdata/blob/master/src/health/health.d/elasticsearch.conf) | elasticsearch.node_indices_search_time | search performance is degraded, queries run slowly. | +| [ elasticsearch_node_indices_search_time_fetch ](https://github.com/netdata/netdata/blob/master/src/health/health.d/elasticsearch.conf) | elasticsearch.node_indices_search_time | search performance is degraded, fetches run slowly. | +| [ elasticsearch_cluster_health_status_red ](https://github.com/netdata/netdata/blob/master/src/health/health.d/elasticsearch.conf) | elasticsearch.cluster_health_status | cluster health status is red. | +| [ elasticsearch_cluster_health_status_yellow ](https://github.com/netdata/netdata/blob/master/src/health/health.d/elasticsearch.conf) | elasticsearch.cluster_health_status | cluster health status is yellow. | +| [ elasticsearch_node_index_health_red ](https://github.com/netdata/netdata/blob/master/src/health/health.d/elasticsearch.conf) | elasticsearch.node_index_health | node index $label:index health status is red. | + + +## Setup + +### Prerequisites + +No action required. + +### Configuration + +#### File + +The configuration file name for this integration is `go.d/elasticsearch.conf`. + + +You can edit the configuration file using the `edit-config` script from the +Netdata [config directory](https://github.com/netdata/netdata/blob/master/docs/netdata-agent/configuration.md#the-netdata-config-directory). + +```bash +cd /etc/netdata 2>/dev/null || cd /opt/netdata/etc/netdata +sudo ./edit-config go.d/elasticsearch.conf +``` +#### Options + +The following options can be defined globally: update_every, autodetection_retry. + + +<details><summary>Config options</summary> + +| Name | Description | Default | Required | +|:----|:-----------|:-------|:--------:| +| update_every | Data collection frequency. | 5 | no | +| autodetection_retry | Recheck interval in seconds. Zero means no recheck will be scheduled. | 0 | no | +| url | Server URL. | http://127.0.0.1:9200 | yes | +| cluster_mode | Controls whether to collect metrics for all nodes in the cluster or only for the local node. | false | no | +| collect_node_stats | Controls whether to collect nodes metrics. | true | no | +| collect_cluster_health | Controls whether to collect cluster health metrics. | true | no | +| collect_cluster_stats | Controls whether to collect cluster stats metrics. | true | no | +| collect_indices_stats | Controls whether to collect indices metrics. | false | no | +| timeout | HTTP request timeout. | 2 | no | +| username | Username for basic HTTP authentication. | | no | +| password | Password for basic HTTP authentication. | | no | +| proxy_url | Proxy URL. | | no | +| proxy_username | Username for proxy basic HTTP authentication. | | no | +| proxy_password | Password for proxy basic HTTP authentication. | | no | +| method | HTTP request method. | GET | no | +| body | HTTP request body. | | no | +| headers | HTTP request headers. | | no | +| not_follow_redirects | Redirect handling policy. Controls whether the client follows redirects. | no | no | +| tls_skip_verify | Server certificate chain and hostname validation policy. Controls whether the client performs this check. | no | no | +| tls_ca | Certification authority that the client uses when verifying the server's certificates. | | no | +| tls_cert | Client TLS certificate. | | no | +| tls_key | Client TLS key. | | no | + +</details> + +#### Examples + +##### Basic single node mode + +A basic example configuration. + +```yaml +jobs: + - name: local + url: http://127.0.0.1:9200 + +``` +##### Cluster mode + +Cluster mode example configuration. + +<details><summary>Config</summary> + +```yaml +jobs: + - name: local + url: http://127.0.0.1:9200 + cluster_mode: yes + +``` +</details> + +##### HTTP authentication + +Basic HTTP authentication. + +<details><summary>Config</summary> + +```yaml +jobs: + - name: local + url: http://127.0.0.1:9200 + username: username + password: password + +``` +</details> + +##### HTTPS with self-signed certificate + +Elasticsearch with enabled HTTPS and self-signed certificate. + +<details><summary>Config</summary> + +```yaml +jobs: + - name: local + url: https://127.0.0.1:9200 + tls_skip_verify: yes + +``` +</details> + +##### Multi-instance + +> **Note**: When you define multiple jobs, their names must be unique. + +Collecting metrics from local and remote instances. + + +<details><summary>Config</summary> + +```yaml +jobs: + - name: local + url: http://127.0.0.1:9200 + + - name: remote + url: http://192.0.2.1:9200 + +``` +</details> + + + +## Troubleshooting + +### Debug Mode + +To troubleshoot issues with the `elasticsearch` collector, run the `go.d.plugin` with the debug option enabled. The output +should give you clues as to why the collector isn't working. + +- Navigate to the `plugins.d` directory, usually at `/usr/libexec/netdata/plugins.d/`. If that's not the case on + your system, open `netdata.conf` and look for the `plugins` setting under `[directories]`. + + ```bash + cd /usr/libexec/netdata/plugins.d/ + ``` + +- Switch to the `netdata` user. + + ```bash + sudo -u netdata -s + ``` + +- Run the `go.d.plugin` to debug the collector: + + ```bash + ./go.d.plugin -d -m elasticsearch + ``` + + diff --git a/src/go/collectors/go.d.plugin/modules/elasticsearch/metadata.yaml b/src/go/collectors/go.d.plugin/modules/elasticsearch/metadata.yaml new file mode 100644 index 000000000..9ee892948 --- /dev/null +++ b/src/go/collectors/go.d.plugin/modules/elasticsearch/metadata.yaml @@ -0,0 +1,634 @@ +plugin_name: go.d.plugin +modules: + - &module + meta: &meta + id: collector-go.d.plugin-elasticsearch + module_name: elasticsearch + plugin_name: go.d.plugin + monitored_instance: + name: Elasticsearch + link: https://www.elastic.co/elasticsearch/ + icon_filename: elasticsearch.svg + categories: + - data-collection.search-engines + keywords: + - elastic + - elasticsearch + - opensearch + - search engine + related_resources: + integrations: + list: + - plugin_name: apps.plugin + module_name: apps + - plugin_name: cgroups.plugin + module_name: cgroups + info_provided_to_referring_integrations: + description: "" + most_popular: true + overview: + data_collection: + metrics_description: | + This collector monitors the performance and health of the Elasticsearch cluster. + method_description: | + It uses [Cluster APIs](https://www.elastic.co/guide/en/elasticsearch/reference/current/cluster.html) to collect metrics. + + Used endpoints: + + | Endpoint | Description | API | + |------------------------|----------------------|-------------------------------------------------------------------------------------------------------------| + | `/` | Node info | | + | `/_nodes/stats` | Nodes metrics | [Nodes stats API](https://www.elastic.co/guide/en/elasticsearch/reference/current/cluster-nodes-stats.html) | + | `/_nodes/_local/stats` | Local node metrics | [Nodes stats API](https://www.elastic.co/guide/en/elasticsearch/reference/current/cluster-nodes-stats.html) | + | `/_cluster/health` | Cluster health stats | [Cluster health API](https://www.elastic.co/guide/en/elasticsearch/reference/current/cluster-health.html) | + | `/_cluster/stats` | Cluster metrics | [Cluster stats API](https://www.elastic.co/guide/en/elasticsearch/reference/current/cluster-stats.html) | + supported_platforms: + include: [] + exclude: [] + multi_instance: true + additional_permissions: + description: "" + default_behavior: + auto_detection: + description: | + By default, it detects instances running on localhost by attempting to connect to port 9200: + + - http://127.0.0.1:9200 + - https://127.0.0.1:9200 + limits: + description: | + By default, this collector monitors only the node it is connected to. To monitor all cluster nodes, set the `cluster_mode` configuration option to `yes`. + performance_impact: + description: "" + setup: + prerequisites: + list: [] + configuration: + file: + name: "go.d/elasticsearch.conf" + options: + description: | + The following options can be defined globally: update_every, autodetection_retry. + folding: + title: Config options + enabled: true + list: + - name: update_every + description: Data collection frequency. + default_value: 5 + required: false + - name: autodetection_retry + description: Recheck interval in seconds. Zero means no recheck will be scheduled. + default_value: 0 + required: false + - name: url + description: Server URL. + default_value: http://127.0.0.1:9200 + required: true + - name: cluster_mode + description: Controls whether to collect metrics for all nodes in the cluster or only for the local node. + default_value: "false" + required: false + - name: collect_node_stats + description: Controls whether to collect nodes metrics. + default_value: "true" + required: false + - name: collect_cluster_health + description: Controls whether to collect cluster health metrics. + default_value: "true" + required: false + - name: collect_cluster_stats + description: Controls whether to collect cluster stats metrics. + default_value: "true" + required: false + - name: collect_indices_stats + description: Controls whether to collect indices metrics. + default_value: "false" + required: false + - name: timeout + description: HTTP request timeout. + default_value: 2 + required: false + - name: username + description: Username for basic HTTP authentication. + default_value: "" + required: false + - name: password + description: Password for basic HTTP authentication. + default_value: "" + required: false + - name: proxy_url + description: Proxy URL. + default_value: "" + required: false + - name: proxy_username + description: Username for proxy basic HTTP authentication. + default_value: "" + required: false + - name: proxy_password + description: Password for proxy basic HTTP authentication. + default_value: "" + required: false + - name: method + description: HTTP request method. + default_value: "GET" + required: false + - name: body + description: HTTP request body. + default_value: "" + required: false + - name: headers + description: HTTP request headers. + default_value: "" + required: false + - name: not_follow_redirects + description: Redirect handling policy. Controls whether the client follows redirects. + default_value: no + required: false + - name: tls_skip_verify + description: Server certificate chain and hostname validation policy. Controls whether the client performs this check. + default_value: no + required: false + - name: tls_ca + description: Certification authority that the client uses when verifying the server's certificates. + default_value: "" + required: false + - name: tls_cert + description: Client TLS certificate. + default_value: "" + required: false + - name: tls_key + description: Client TLS key. + default_value: "" + required: false + examples: + folding: + title: Config + enabled: true + list: + - name: Basic single node mode + description: A basic example configuration. + folding: + enabled: false + config: | + jobs: + - name: local + url: http://127.0.0.1:9200 + - name: Cluster mode + description: Cluster mode example configuration. + config: | + jobs: + - name: local + url: http://127.0.0.1:9200 + cluster_mode: yes + - name: HTTP authentication + description: Basic HTTP authentication. + config: | + jobs: + - name: local + url: http://127.0.0.1:9200 + username: username + password: password + - name: HTTPS with self-signed certificate + description: Elasticsearch with enabled HTTPS and self-signed certificate. + config: | + jobs: + - name: local + url: https://127.0.0.1:9200 + tls_skip_verify: yes + - name: Multi-instance + description: | + > **Note**: When you define multiple jobs, their names must be unique. + + Collecting metrics from local and remote instances. + config: | + jobs: + - name: local + url: http://127.0.0.1:9200 + + - name: remote + url: http://192.0.2.1:9200 + troubleshooting: + problems: + list: [] + alerts: + - name: elasticsearch_node_indices_search_time_query + link: https://github.com/netdata/netdata/blob/master/src/health/health.d/elasticsearch.conf + metric: elasticsearch.node_indices_search_time + info: search performance is degraded, queries run slowly. + - name: elasticsearch_node_indices_search_time_fetch + link: https://github.com/netdata/netdata/blob/master/src/health/health.d/elasticsearch.conf + metric: elasticsearch.node_indices_search_time + info: search performance is degraded, fetches run slowly. + - name: elasticsearch_cluster_health_status_red + link: https://github.com/netdata/netdata/blob/master/src/health/health.d/elasticsearch.conf + metric: elasticsearch.cluster_health_status + info: cluster health status is red. + - name: elasticsearch_cluster_health_status_yellow + link: https://github.com/netdata/netdata/blob/master/src/health/health.d/elasticsearch.conf + metric: elasticsearch.cluster_health_status + info: cluster health status is yellow. + - name: elasticsearch_node_index_health_red + link: https://github.com/netdata/netdata/blob/master/src/health/health.d/elasticsearch.conf + metric: elasticsearch.node_index_health + info: node index $label:index health status is red. + metrics: + folding: + title: Metrics + enabled: false + description: "" + availability: [] + scopes: + - name: node + description: These metrics refer to the cluster node. + labels: + - name: cluster_name + description: | + Name of the cluster. Based on the [Cluster name setting](https://www.elastic.co/guide/en/elasticsearch/reference/current/important-settings.html#cluster-name). + - name: node_name + description: | + Human-readable identifier for the node. Based on the [Node name setting](https://www.elastic.co/guide/en/elasticsearch/reference/current/important-settings.html#node-name). + - name: host + description: | + Network host for the node, based on the [Network host setting](https://www.elastic.co/guide/en/elasticsearch/reference/current/important-settings.html#network.host). + metrics: + - name: elasticsearch.node_indices_indexing + description: Indexing Operations + unit: operations/s + chart_type: line + dimensions: + - name: index + - name: elasticsearch.node_indices_indexing_current + description: Indexing Operations Current + unit: operations + chart_type: line + dimensions: + - name: index + - name: elasticsearch.node_indices_indexing_time + description: Time Spent On Indexing Operations + unit: milliseconds + chart_type: line + dimensions: + - name: index + - name: elasticsearch.node_indices_search + description: Search Operations + unit: operations/s + chart_type: stacked + dimensions: + - name: queries + - name: fetches + - name: elasticsearch.node_indices_search_current + description: Search Operations Current + unit: operations + chart_type: stacked + dimensions: + - name: queries + - name: fetches + - name: elasticsearch.node_indices_search_time + description: node_indices_search_time + unit: milliseconds + chart_type: stacked + dimensions: + - name: queries + - name: fetches + - name: elasticsearch.node_indices_refresh + description: Refresh Operations + unit: operations/s + chart_type: line + dimensions: + - name: refresh + - name: elasticsearch.node_indices_refresh_time + description: Time Spent On Refresh Operations + unit: milliseconds + chart_type: line + dimensions: + - name: refresh + - name: elasticsearch.node_indices_flush + description: Flush Operations + unit: operations/s + chart_type: line + dimensions: + - name: flush + - name: elasticsearch.node_indices_flush_time + description: Time Spent On Flush Operations + unit: milliseconds + chart_type: line + dimensions: + - name: flush + - name: elasticsearch.node_indices_fielddata_memory_usage + description: Fielddata Cache Memory Usage + unit: bytes + chart_type: area + dimensions: + - name: used + - name: elasticsearch.node_indices_fielddata_evictions + description: Fielddata Evictions + unit: operations/s + chart_type: line + dimensions: + - name: evictions + - name: elasticsearch.node_indices_segments_count + description: Segments Count + unit: segments + chart_type: line + dimensions: + - name: segments + - name: elasticsearch.node_indices_segments_memory_usage_total + description: Segments Memory Usage Total + unit: bytes + chart_type: line + dimensions: + - name: used + - name: elasticsearch.node_indices_segments_memory_usage + description: Segments Memory Usage + unit: bytes + chart_type: stacked + dimensions: + - name: terms + - name: stored_fields + - name: term_vectors + - name: norms + - name: points + - name: doc_values + - name: index_writer + - name: version_map + - name: fixed_bit_set + - name: elasticsearch.node_indices_translog_operations + description: Translog Operations + unit: operations + chart_type: area + dimensions: + - name: total + - name: uncommitted + - name: elasticsearch.node_indices_translog_size + description: Translog Size + unit: bytes + chart_type: area + dimensions: + - name: total + - name: uncommitted + - name: elasticsearch.node_file_descriptors + description: Process File Descriptors + unit: fd + chart_type: line + dimensions: + - name: open + - name: elasticsearch.node_jvm_heap + description: JVM Heap Percentage Currently in Use + unit: percentage + chart_type: area + dimensions: + - name: inuse + - name: elasticsearch.node_jvm_heap_bytes + description: JVM Heap Commit And Usage + unit: bytes + chart_type: area + dimensions: + - name: committed + - name: used + - name: elasticsearch.node_jvm_buffer_pools_count + description: JVM Buffer Pools Count + unit: pools + chart_type: line + dimensions: + - name: direct + - name: mapped + - name: elasticsearch.node_jvm_buffer_pool_direct_memory + description: JVM Buffer Pool Direct Memory + unit: bytes + chart_type: area + dimensions: + - name: total + - name: used + - name: elasticsearch.node_jvm_buffer_pool_mapped_memory + description: JVM Buffer Pool Mapped Memory + unit: bytes + chart_type: area + dimensions: + - name: total + - name: used + - name: elasticsearch.node_jvm_gc_count + description: JVM Garbage Collections + unit: gc/s + chart_type: stacked + dimensions: + - name: young + - name: old + - name: elasticsearch.node_jvm_gc_time + description: JVM Time Spent On Garbage Collections + unit: milliseconds + chart_type: stacked + dimensions: + - name: young + - name: old + - name: elasticsearch.node_thread_pool_queued + description: Thread Pool Queued Threads Count + unit: threads + chart_type: stacked + dimensions: + - name: generic + - name: search + - name: search_throttled + - name: get + - name: analyze + - name: write + - name: snapshot + - name: warmer + - name: refresh + - name: listener + - name: fetch_shard_started + - name: fetch_shard_store + - name: flush + - name: force_merge + - name: management + - name: elasticsearch.node_thread_pool_rejected + description: Thread Pool Rejected Threads Count + unit: threads + chart_type: stacked + dimensions: + - name: generic + - name: search + - name: search_throttled + - name: get + - name: analyze + - name: write + - name: snapshot + - name: warmer + - name: refresh + - name: listener + - name: fetch_shard_started + - name: fetch_shard_store + - name: flush + - name: force_merge + - name: management + - name: elasticsearch.node_cluster_communication_packets + description: Cluster Communication + unit: pps + chart_type: line + dimensions: + - name: received + - name: sent + - name: elasticsearch.node_cluster_communication_traffic + description: Cluster Communication Bandwidth + unit: bytes/s + chart_type: line + dimensions: + - name: received + - name: sent + - name: elasticsearch.node_http_connections + description: HTTP Connections + unit: connections + chart_type: line + dimensions: + - name: open + - name: elasticsearch.node_breakers_trips + description: Circuit Breaker Trips Count + unit: trips/s + chart_type: stacked + dimensions: + - name: requests + - name: fielddata + - name: in_flight_requests + - name: model_inference + - name: accounting + - name: parent + - name: cluster + description: These metrics refer to the cluster. + labels: + - name: cluster_name + description: | + Name of the cluster. Based on the [Cluster name setting](https://www.elastic.co/guide/en/elasticsearch/reference/current/important-settings.html#cluster-name). + metrics: + - name: elasticsearch.cluster_health_status + description: Cluster Status + unit: status + chart_type: line + dimensions: + - name: green + - name: yellow + - name: red + - name: elasticsearch.cluster_number_of_nodes + description: Cluster Nodes Count + unit: nodes + chart_type: line + dimensions: + - name: nodes + - name: data_nodes + - name: elasticsearch.cluster_shards_count + description: Cluster Shards Count + unit: shards + chart_type: line + dimensions: + - name: active_primary + - name: active + - name: relocating + - name: initializing + - name: unassigned + - name: delayed_unaasigned + - name: elasticsearch.cluster_pending_tasks + description: Cluster Pending Tasks + unit: tasks + chart_type: line + dimensions: + - name: pending + - name: elasticsearch.cluster_number_of_in_flight_fetch + description: Cluster Unfinished Fetches + unit: fetches + chart_type: line + dimensions: + - name: in_flight_fetch + - name: elasticsearch.cluster_indices_count + description: Cluster Indices Count + unit: indices + chart_type: line + dimensions: + - name: indices + - name: elasticsearch.cluster_indices_shards_count + description: Cluster Indices Shards Count + unit: shards + chart_type: line + dimensions: + - name: total + - name: primaries + - name: replication + - name: elasticsearch.cluster_indices_docs_count + description: Cluster Indices Docs Count + unit: docs + chart_type: line + dimensions: + - name: docs + - name: elasticsearch.cluster_indices_store_size + description: Cluster Indices Store Size + unit: bytes + chart_type: line + dimensions: + - name: size + - name: elasticsearch.cluster_indices_query_cache + description: Cluster Indices Query Cache + unit: events/s + chart_type: line + dimensions: + - name: hit + - name: miss + - name: elasticsearch.cluster_nodes_by_role_count + description: Cluster Nodes By Role Count + unit: nodes + chart_type: line + dimensions: + - name: coordinating_only + - name: data + - name: data_cold + - name: data_content + - name: data_frozen + - name: data_hot + - name: data_warm + - name: ingest + - name: master + - name: ml + - name: remote_cluster_client + - name: voting_only + - name: index + description: These metrics refer to the index. + labels: + - name: cluster_name + description: | + Name of the cluster. Based on the [Cluster name setting](https://www.elastic.co/guide/en/elasticsearch/reference/current/important-settings.html#cluster-name). + - name: index + description: Name of the index. + metrics: + - name: elasticsearch.node_index_health + description: Index Health + unit: status + chart_type: line + dimensions: + - name: green + - name: yellow + - name: red + - name: elasticsearch.node_index_shards_count + description: Index Shards Count + unit: shards + chart_type: line + dimensions: + - name: shards + - name: elasticsearch.node_index_docs_count + description: Index Docs Count + unit: docs + chart_type: line + dimensions: + - name: docs + - name: elasticsearch.node_index_store_size + description: Index Store Size + unit: bytes + chart_type: line + dimensions: + - name: store_size + - <<: *module + meta: + <<: *meta + id: collector-go.d.plugin-opensearch + monitored_instance: + name: OpenSearch + link: https://opensearch.org/ + icon_filename: opensearch.svg + categories: + - data-collection.search-engines diff --git a/src/go/collectors/go.d.plugin/modules/elasticsearch/metrics.go b/src/go/collectors/go.d.plugin/modules/elasticsearch/metrics.go new file mode 100644 index 000000000..e838dc643 --- /dev/null +++ b/src/go/collectors/go.d.plugin/modules/elasticsearch/metrics.go @@ -0,0 +1,277 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +package elasticsearch + +// https://www.elastic.co/guide/en/elasticsearch/reference/current/docker.html + +type esMetrics struct { + // https://www.elastic.co/guide/en/elasticsearch/reference/current/cluster-nodes-stats.html + NodesStats *esNodesStats + // https://www.elastic.co/guide/en/elasticsearch/reference/current/cluster-health.html + ClusterHealth *esClusterHealth + // https://www.elastic.co/guide/en/elasticsearch/reference/current/cluster-stats.html + ClusterStats *esClusterStats + // https://www.elastic.co/guide/en/elasticsearch/reference/current/cat-indices.html + LocalIndicesStats []esIndexStats +} + +func (m esMetrics) empty() bool { + switch { + case m.hasNodesStats(), m.hasClusterHealth(), m.hasClusterStats(), m.hasLocalIndicesStats(): + return false + } + return true +} + +func (m esMetrics) hasNodesStats() bool { return m.NodesStats != nil && len(m.NodesStats.Nodes) > 0 } +func (m esMetrics) hasClusterHealth() bool { return m.ClusterHealth != nil } +func (m esMetrics) hasClusterStats() bool { return m.ClusterStats != nil } +func (m esMetrics) hasLocalIndicesStats() bool { return len(m.LocalIndicesStats) > 0 } + +type ( + esNodesStats struct { + ClusterName string `json:"cluster_name"` + Nodes map[string]*esNodeStats `json:"nodes"` + } + esNodeStats struct { + Name string + Host string + Indices struct { + Indexing struct { + IndexTotal float64 `stm:"index_total" json:"index_total"` + IndexCurrent float64 `stm:"index_current" json:"index_current"` + IndexTimeInMillis float64 `stm:"index_time_in_millis" json:"index_time_in_millis"` + } `stm:"indexing"` + Search struct { + FetchTotal float64 `stm:"fetch_total" json:"fetch_total"` + FetchCurrent float64 `stm:"fetch_current" json:"fetch_current"` + FetchTimeInMillis float64 `stm:"fetch_time_in_millis" json:"fetch_time_in_millis"` + QueryTotal float64 `stm:"query_total" json:"query_total"` + QueryCurrent float64 `stm:"query_current" json:"query_current"` + QueryTimeInMillis float64 `stm:"query_time_in_millis" json:"query_time_in_millis"` + } `stm:"search"` + Refresh struct { + Total float64 `stm:"total"` + TimeInMillis float64 `stm:"total_time_in_millis" json:"total_time_in_millis"` + } `stm:"refresh"` + Flush struct { + Total float64 `stm:"total"` + TimeInMillis float64 `stm:"total_time_in_millis" json:"total_time_in_millis"` + } `stm:"flush"` + FieldData struct { + MemorySizeInBytes float64 `stm:"memory_size_in_bytes" json:"memory_size_in_bytes"` + Evictions float64 `stm:"evictions"` + } `stm:"fielddata"` + Segments struct { + Count float64 `stm:"count" json:"count"` + MemoryInBytes float64 `stm:"memory_in_bytes" json:"memory_in_bytes"` + TermsMemoryInBytes float64 `stm:"terms_memory_in_bytes" json:"terms_memory_in_bytes"` + StoredFieldsMemoryInBytes float64 `stm:"stored_fields_memory_in_bytes" json:"stored_fields_memory_in_bytes"` + TermVectorsMemoryInBytes float64 `stm:"term_vectors_memory_in_bytes" json:"term_vectors_memory_in_bytes"` + NormsMemoryInBytes float64 `stm:"norms_memory_in_bytes" json:"norms_memory_in_bytes"` + PointsMemoryInBytes float64 `stm:"points_memory_in_bytes" json:"points_memory_in_bytes"` + DocValuesMemoryInBytes float64 `stm:"doc_values_memory_in_bytes" json:"doc_values_memory_in_bytes"` + IndexWriterMemoryInBytes float64 `stm:"index_writer_memory_in_bytes" json:"index_writer_memory_in_bytes"` + VersionMapMemoryInBytes float64 `stm:"version_map_memory_in_bytes" json:"version_map_memory_in_bytes"` + FixedBitSetMemoryInBytes float64 `stm:"fixed_bit_set_memory_in_bytes" json:"fixed_bit_set_memory_in_bytes"` + } `stm:"segments"` + Translog struct { + Operations float64 `stm:"operations"` + SizeInBytes float64 `stm:"size_in_bytes" json:"size_in_bytes"` + UncommittedOperations float64 `stm:"uncommitted_operations" json:"uncommitted_operations"` + UncommittedSizeInBytes float64 `stm:"uncommitted_size_in_bytes" json:"uncommitted_size_in_bytes"` + } `stm:"translog"` + } `stm:"indices"` + Process struct { + OpenFileDescriptors float64 `stm:"open_file_descriptors" json:"open_file_descriptors"` + MaxFileDescriptors float64 `stm:"max_file_descriptors" json:"max_file_descriptors"` + } `stm:"process"` + JVM struct { + Mem struct { + HeapUsedPercent float64 `stm:"heap_used_percent" json:"heap_used_percent"` + HeapUsedInBytes float64 `stm:"heap_used_in_bytes" json:"heap_used_in_bytes"` + HeapCommittedInBytes float64 `stm:"heap_committed_in_bytes" json:"heap_committed_in_bytes"` + } `stm:"mem"` + GC struct { + Collectors struct { + Young struct { + CollectionCount float64 `stm:"collection_count" json:"collection_count"` + CollectionTimeInMillis float64 `stm:"collection_time_in_millis" json:"collection_time_in_millis"` + } `stm:"young"` + Old struct { + CollectionCount float64 `stm:"collection_count" json:"collection_count"` + CollectionTimeInMillis float64 `stm:"collection_time_in_millis" json:"collection_time_in_millis"` + } `stm:"old"` + } `stm:"collectors"` + } `stm:"gc"` + BufferPools struct { + Mapped struct { + Count float64 `stm:"count"` + UsedInBytes float64 `stm:"used_in_bytes" json:"used_in_bytes"` + TotalCapacityInBytes float64 `stm:"total_capacity_in_bytes" json:"total_capacity_in_bytes"` + } `stm:"mapped"` + Direct struct { + Count float64 `stm:"count"` + UsedInBytes float64 `stm:"used_in_bytes" json:"used_in_bytes"` + TotalCapacityInBytes float64 `stm:"total_capacity_in_bytes" json:"total_capacity_in_bytes"` + } `stm:"direct"` + } `stm:"buffer_pools" json:"buffer_pools"` + } `stm:"jvm"` + // https://www.elastic.co/guide/en/elasticsearch/reference/current/modules-threadpool.html + ThreadPool struct { + Generic struct { + Queue float64 `stm:"queue"` + Rejected float64 `stm:"rejected"` + } `stm:"generic"` + Search struct { + Queue float64 `stm:"queue"` + Rejected float64 `stm:"rejected"` + } `stm:"search"` + SearchThrottled struct { + Queue float64 `stm:"queue"` + Rejected float64 `stm:"rejected"` + } `stm:"search_throttled" json:"search_throttled"` + Get struct { + Queue float64 `stm:"queue"` + Rejected float64 `stm:"rejected"` + } `stm:"get"` + Analyze struct { + Queue float64 `stm:"queue"` + Rejected float64 `stm:"rejected"` + } `stm:"analyze"` + Write struct { + Queue float64 `stm:"queue"` + Rejected float64 `stm:"rejected"` + } `stm:"write"` + Snapshot struct { + Queue float64 `stm:"queue"` + Rejected float64 `stm:"rejected"` + } `stm:"snapshot"` + Warmer struct { + Queue float64 `stm:"queue"` + Rejected float64 `stm:"rejected"` + } `stm:"warmer"` + Refresh struct { + Queue float64 `stm:"queue"` + Rejected float64 `stm:"rejected"` + } `stm:"refresh"` + Listener struct { + Queue float64 `stm:"queue"` + Rejected float64 `stm:"rejected"` + } `stm:"listener"` + FetchShardStarted struct { + Queue float64 `stm:"queue"` + Rejected float64 `stm:"rejected"` + } `stm:"fetch_shard_started" json:"fetch_shard_started"` + FetchShardStore struct { + Queue float64 `stm:"queue"` + Rejected float64 `stm:"rejected"` + } `stm:"fetch_shard_store" json:"fetch_shard_store"` + Flush struct { + Queue float64 `stm:"queue"` + Rejected float64 `stm:"rejected"` + } `stm:"flush"` + ForceMerge struct { + Queue float64 `stm:"queue"` + Rejected float64 `stm:"rejected"` + } `stm:"force_merge" json:"force_merge"` + Management struct { + Queue float64 `stm:"queue"` + Rejected float64 `stm:"rejected"` + } `stm:"management"` + } `stm:"thread_pool" json:"thread_pool"` + Transport struct { + RxCount float64 `stm:"rx_count" json:"rx_count"` + RxSizeInBytes float64 `stm:"rx_size_in_bytes" json:"rx_size_in_bytes"` + TxCount float64 `stm:"tx_count" json:"tx_count"` + TxSizeInBytes float64 `stm:"tx_size_in_bytes" json:"tx_size_in_bytes"` + } `stm:"transport"` + HTTP struct { + CurrentOpen float64 `stm:"current_open" json:"current_open"` + } `stm:"http"` + Breakers struct { + Request struct { + Tripped float64 `stm:"tripped"` + } `stm:"request"` + FieldData struct { + Tripped float64 `stm:"tripped"` + } `stm:"fielddata"` + InFlightRequests struct { + Tripped float64 `stm:"tripped"` + } `stm:"in_flight_requests" json:"in_flight_requests"` + ModelInference struct { + Tripped float64 `stm:"tripped"` + } `stm:"model_inference" json:"model_inference"` + Accounting struct { + Tripped float64 `stm:"tripped"` + } `stm:"accounting"` + Parent struct { + Tripped float64 `stm:"tripped"` + } `stm:"parent"` + } `stm:"breakers"` + } +) + +type esClusterHealth struct { + ClusterName string `json:"cluster_name"` + Status string + NumOfNodes float64 `stm:"number_of_nodes" json:"number_of_nodes"` + NumOfDataNodes float64 `stm:"number_of_data_nodes" json:"number_of_data_nodes"` + ActivePrimaryShards float64 `stm:"active_primary_shards" json:"active_primary_shards"` + ActiveShards float64 `stm:"active_shards" json:"active_shards"` + RelocatingShards float64 `stm:"relocating_shards" json:"relocating_shards"` + InitializingShards float64 `stm:"initializing_shards" json:"initializing_shards"` + UnassignedShards float64 `stm:"unassigned_shards" json:"unassigned_shards"` + DelayedUnassignedShards float64 `stm:"delayed_unassigned_shards" json:"delayed_unassigned_shards"` + NumOfPendingTasks float64 `stm:"number_of_pending_tasks" json:"number_of_pending_tasks"` + NumOfInFlightFetch float64 `stm:"number_of_in_flight_fetch" json:"number_of_in_flight_fetch"` + ActiveShardsPercentAsNumber float64 `stm:"active_shards_percent_as_number" json:"active_shards_percent_as_number"` +} + +type esClusterStats struct { + ClusterName string `json:"cluster_name"` + Nodes struct { + Count struct { + Total float64 `stm:"total"` + CoordinatingOnly float64 `stm:"coordinating_only" json:"coordinating_only"` + Data float64 `stm:"data"` + DataCold float64 `stm:"data_cold" json:"data_cold"` + DataContent float64 `stm:"data_content" json:"data_content"` + DataFrozen float64 `stm:"data_frozen" json:"data_frozen"` + DataHot float64 `stm:"data_hot" json:"data_hot"` + DataWarm float64 `stm:"data_warm" json:"data_warm"` + Ingest float64 `stm:"ingest"` + Master float64 `stm:"master"` + ML float64 `stm:"ml"` + RemoteClusterClient float64 `stm:"remote_cluster_client" json:"remote_cluster_client"` + Transform float64 `stm:"transform"` + VotingOnly float64 `stm:"voting_only" json:"voting_only"` + } `stm:"count"` + } `stm:"nodes"` + Indices struct { + Count float64 `stm:"count"` + Shards struct { + Total float64 `stm:"total"` + Primaries float64 `stm:"primaries"` + Replication float64 `stm:"replication"` + } `stm:"shards"` + Docs struct { + Count float64 `stm:"count"` + } `stm:"docs"` + Store struct { + SizeInBytes float64 `stm:"size_in_bytes" json:"size_in_bytes"` + } `stm:"store"` + QueryCache struct { + HitCount float64 `stm:"hit_count" json:"hit_count"` + MissCount float64 `stm:"miss_count" json:"miss_count"` + } `stm:"query_cache" json:"query_cache"` + } `stm:"indices"` +} + +type esIndexStats struct { + Index string + Health string + Rep string + DocsCount string `json:"docs.count"` + StoreSize string `json:"store.size"` +} diff --git a/src/go/collectors/go.d.plugin/modules/elasticsearch/testdata/config.json b/src/go/collectors/go.d.plugin/modules/elasticsearch/testdata/config.json new file mode 100644 index 000000000..a456d1d56 --- /dev/null +++ b/src/go/collectors/go.d.plugin/modules/elasticsearch/testdata/config.json @@ -0,0 +1,25 @@ +{ + "update_every": 123, + "url": "ok", + "body": "ok", + "method": "ok", + "headers": { + "ok": "ok" + }, + "username": "ok", + "password": "ok", + "proxy_url": "ok", + "proxy_username": "ok", + "proxy_password": "ok", + "timeout": 123.123, + "not_follow_redirects": true, + "tls_ca": "ok", + "tls_cert": "ok", + "tls_key": "ok", + "tls_skip_verify": true, + "cluster_mode": true, + "collect_node_stats": true, + "collect_cluster_health": true, + "collect_cluster_stats": true, + "collect_indices_stats": true +} diff --git a/src/go/collectors/go.d.plugin/modules/elasticsearch/testdata/config.yaml b/src/go/collectors/go.d.plugin/modules/elasticsearch/testdata/config.yaml new file mode 100644 index 000000000..af1b4a136 --- /dev/null +++ b/src/go/collectors/go.d.plugin/modules/elasticsearch/testdata/config.yaml @@ -0,0 +1,22 @@ +update_every: 123 +url: "ok" +body: "ok" +method: "ok" +headers: + ok: "ok" +username: "ok" +password: "ok" +proxy_url: "ok" +proxy_username: "ok" +proxy_password: "ok" +timeout: 123.123 +not_follow_redirects: yes +tls_ca: "ok" +tls_cert: "ok" +tls_key: "ok" +tls_skip_verify: yes +cluster_mode: yes +collect_node_stats: yes +collect_cluster_health: yes +collect_cluster_stats: yes +collect_indices_stats: yes diff --git a/src/go/collectors/go.d.plugin/modules/elasticsearch/testdata/v8.4.2/cat_indices_stats.json b/src/go/collectors/go.d.plugin/modules/elasticsearch/testdata/v8.4.2/cat_indices_stats.json new file mode 100644 index 000000000..f46794cc4 --- /dev/null +++ b/src/go/collectors/go.d.plugin/modules/elasticsearch/testdata/v8.4.2/cat_indices_stats.json @@ -0,0 +1,50 @@ +[ + { + "health": "yellow", + "status": "open", + "index": "my-index-000003", + "uuid": "Clrvbw-AQ5CB3xWI3MUXFg", + "pri": "1", + "rep": "1", + "docs.count": "1", + "docs.deleted": "1", + "store.size": "208b", + "pri.store.size": "208b" + }, + { + "health": "yellow", + "status": "open", + "index": "my-index-000002", + "uuid": "z7cy4d2PQYSSJDhi8dIjWg", + "pri": "1", + "rep": "1", + "docs.count": "1", + "docs.deleted": "1", + "store.size": "208b", + "pri.store.size": "208b" + }, + { + "health": "yellow", + "status": "open", + "index": "my-index-000001", + "uuid": "08YTiZfmQUiO67VOGZOfVg", + "pri": "1", + "rep": "1", + "docs.count": "1", + "docs.deleted": "1", + "store.size": "208b", + "pri.store.size": "208b" + }, + { + "health": "yellow", + "status": "open", + "index": ".my-system-index-000001", + "uuid": "08YTiZfmQUiO67VOGZOfVg", + "pri": "1", + "rep": "1", + "docs.count": "1", + "docs.deleted": "1", + "store.size": "208b", + "pri.store.size": "208b" + } +] diff --git a/src/go/collectors/go.d.plugin/modules/elasticsearch/testdata/v8.4.2/cluster_health.json b/src/go/collectors/go.d.plugin/modules/elasticsearch/testdata/v8.4.2/cluster_health.json new file mode 100644 index 000000000..0fdc0de49 --- /dev/null +++ b/src/go/collectors/go.d.plugin/modules/elasticsearch/testdata/v8.4.2/cluster_health.json @@ -0,0 +1,17 @@ +{ + "cluster_name": "36928dce44074ceba64d7b3d698443a7", + "status": "green", + "timed_out": false, + "number_of_nodes": 3, + "number_of_data_nodes": 2, + "active_primary_shards": 97, + "active_shards": 194, + "relocating_shards": 0, + "initializing_shards": 0, + "unassigned_shards": 0, + "delayed_unassigned_shards": 0, + "number_of_pending_tasks": 0, + "number_of_in_flight_fetch": 0, + "task_max_waiting_in_queue_millis": 0, + "active_shards_percent_as_number": 100 +} diff --git a/src/go/collectors/go.d.plugin/modules/elasticsearch/testdata/v8.4.2/cluster_stats.json b/src/go/collectors/go.d.plugin/modules/elasticsearch/testdata/v8.4.2/cluster_stats.json new file mode 100644 index 000000000..53bea1b34 --- /dev/null +++ b/src/go/collectors/go.d.plugin/modules/elasticsearch/testdata/v8.4.2/cluster_stats.json @@ -0,0 +1,377 @@ +{ + "_nodes": { + "total": 3, + "successful": 3, + "failed": 0 + }, + "cluster_name": "36928dce44074ceba64d7b3d698443a7", + "cluster_uuid": "5jO2X31FQ32kJAWoCsp3Vw", + "timestamp": 1687866240414, + "status": "green", + "indices": { + "count": 97, + "shards": { + "total": 194, + "primaries": 97, + "replication": 1, + "index": { + "shards": { + "min": 2, + "max": 2, + "avg": 2 + }, + "primaries": { + "min": 1, + "max": 1, + "avg": 1 + }, + "replication": { + "min": 1, + "max": 1, + "avg": 1 + } + } + }, + "docs": { + "count": 402750703, + "deleted": 1603 + }, + "store": { + "size_in_bytes": 380826136962, + "total_data_set_size_in_bytes": 380826136962, + "reserved_in_bytes": 0 + }, + "fielddata": { + "memory_size_in_bytes": 600, + "evictions": 0 + }, + "query_cache": { + "memory_size_in_bytes": 37465951, + "total_count": 684606952, + "hit_count": 96838726, + "miss_count": 587768226, + "cache_size": 22571, + "cache_count": 91319, + "evictions": 68748 + }, + "completion": { + "size_in_bytes": 0 + }, + "segments": { + "count": 614, + "memory_in_bytes": 0, + "terms_memory_in_bytes": 0, + "stored_fields_memory_in_bytes": 0, + "term_vectors_memory_in_bytes": 0, + "norms_memory_in_bytes": 0, + "points_memory_in_bytes": 0, + "doc_values_memory_in_bytes": 0, + "index_writer_memory_in_bytes": 368167356, + "version_map_memory_in_bytes": 54470768, + "fixed_bit_set_memory_in_bytes": 57736, + "max_unsafe_auto_id_timestamp": 1679747033889, + "file_sizes": {} + }, + "mappings": { + "total_field_count": 10467, + "total_deduplicated_field_count": 2070, + "total_deduplicated_mapping_size_in_bytes": 26441, + "field_types": [ + { + "name": "alias", + "count": 1, + "index_count": 1, + "script_count": 0 + }, + { + "name": "boolean", + "count": 77, + "index_count": 37, + "script_count": 0 + }, + { + "name": "constant_keyword", + "count": 10, + "index_count": 4, + "script_count": 0 + }, + { + "name": "date", + "count": 763, + "index_count": 86, + "script_count": 0 + }, + { + "name": "flattened", + "count": 27, + "index_count": 27, + "script_count": 0 + }, + { + "name": "float", + "count": 8, + "index_count": 4, + "script_count": 0 + }, + { + "name": "integer", + "count": 279, + "index_count": 70, + "script_count": 0 + }, + { + "name": "ip", + "count": 4, + "index_count": 4, + "script_count": 0 + }, + { + "name": "keyword", + "count": 4345, + "index_count": 86, + "script_count": 0 + }, + { + "name": "long", + "count": 1143, + "index_count": 79, + "script_count": 0 + }, + { + "name": "match_only_text", + "count": 1170, + "index_count": 69, + "script_count": 0 + }, + { + "name": "nested", + "count": 4, + "index_count": 4, + "script_count": 0 + }, + { + "name": "object", + "count": 2583, + "index_count": 85, + "script_count": 0 + }, + { + "name": "text", + "count": 49, + "index_count": 17, + "script_count": 0 + }, + { + "name": "version", + "count": 4, + "index_count": 4, + "script_count": 0 + } + ], + "runtime_field_types": [] + }, + "analysis": { + "char_filter_types": [], + "tokenizer_types": [], + "filter_types": [], + "analyzer_types": [], + "built_in_char_filters": [], + "built_in_tokenizers": [], + "built_in_filters": [], + "built_in_analyzers": [] + }, + "versions": [ + { + "version": "8.4.2", + "index_count": 97, + "primary_shard_count": 97, + "total_primary_bytes": 189671468048 + } + ] + }, + "nodes": { + "count": { + "total": 3, + "coordinating_only": 0, + "data": 0, + "data_cold": 0, + "data_content": 2, + "data_frozen": 0, + "data_hot": 2, + "data_warm": 0, + "ingest": 2, + "master": 3, + "ml": 0, + "remote_cluster_client": 2, + "transform": 2, + "voting_only": 1 + }, + "versions": [ + "8.4.2" + ], + "os": { + "available_processors": 8, + "allocated_processors": 8, + "names": [ + { + "name": "Linux", + "count": 3 + } + ], + "pretty_names": [ + { + "pretty_name": "Ubuntu 20.04.5 LTS", + "count": 3 + } + ], + "architectures": [ + { + "arch": "amd64", + "count": 3 + } + ], + "mem": { + "total_in_bytes": 33285996544, + "adjusted_total_in_bytes": 32153534464, + "free_in_bytes": 1732333568, + "used_in_bytes": 31553662976, + "free_percent": 5, + "used_percent": 95 + } + }, + "process": { + "cpu": { + "percent": 26 + }, + "open_file_descriptors": { + "min": 557, + "max": 1185, + "avg": 968 + } + }, + "jvm": { + "max_uptime_in_millis": 23671188288, + "versions": [ + { + "version": "18.0.2.1", + "vm_name": "OpenJDK 64-Bit Server VM", + "vm_version": "18.0.2.1+1-1", + "vm_vendor": "Oracle Corporation", + "bundled_jdk": true, + "using_bundled_jdk": true, + "count": 3 + } + ], + "mem": { + "heap_used_in_bytes": 8044798544, + "heap_max_in_bytes": 16009658368 + }, + "threads": 272 + }, + "fs": { + "total_in_bytes": 979252543488, + "free_in_bytes": 595738775552, + "available_in_bytes": 595738775552 + }, + "plugins": [], + "network_types": { + "transport_types": { + "security4": 3 + }, + "http_types": { + "security4": 3 + } + }, + "discovery_types": { + "multi-node": 3 + }, + "packaging_types": [ + { + "flavor": "default", + "type": "docker", + "count": 3 + } + ], + "ingest": { + "number_of_pipelines": 20, + "processor_stats": { + "conditional": { + "count": 0, + "failed": 0, + "current": 0, + "time_in_millis": 0 + }, + "date": { + "count": 0, + "failed": 0, + "current": 0, + "time_in_millis": 0 + }, + "geoip": { + "count": 0, + "failed": 0, + "current": 0, + "time_in_millis": 0 + }, + "pipeline": { + "count": 0, + "failed": 0, + "current": 0, + "time_in_millis": 0 + }, + "remove": { + "count": 0, + "failed": 0, + "current": 0, + "time_in_millis": 0 + }, + "rename": { + "count": 0, + "failed": 0, + "current": 0, + "time_in_millis": 0 + }, + "script": { + "count": 0, + "failed": 0, + "current": 0, + "time_in_millis": 0 + }, + "set": { + "count": 0, + "failed": 0, + "current": 0, + "time_in_millis": 0 + }, + "set_security_user": { + "count": 0, + "failed": 0, + "current": 0, + "time_in_millis": 0 + } + } + }, + "indexing_pressure": { + "memory": { + "current": { + "combined_coordinating_and_primary_in_bytes": 0, + "coordinating_in_bytes": 0, + "primary_in_bytes": 0, + "replica_in_bytes": 0, + "all_in_bytes": 0 + }, + "total": { + "combined_coordinating_and_primary_in_bytes": 0, + "coordinating_in_bytes": 0, + "primary_in_bytes": 0, + "replica_in_bytes": 0, + "all_in_bytes": 0, + "coordinating_rejections": 0, + "primary_rejections": 0, + "replica_rejections": 0 + }, + "limit_in_bytes": 0 + } + } + } +} diff --git a/src/go/collectors/go.d.plugin/modules/elasticsearch/testdata/v8.4.2/info.json b/src/go/collectors/go.d.plugin/modules/elasticsearch/testdata/v8.4.2/info.json new file mode 100644 index 000000000..23e3f1596 --- /dev/null +++ b/src/go/collectors/go.d.plugin/modules/elasticsearch/testdata/v8.4.2/info.json @@ -0,0 +1,17 @@ +{ + "name": "instance-0000000006", + "cluster_name": "36928dce44074ceba64d7b3d698443a7", + "cluster_uuid": "5jO2X31FQ32kJAWoCsp3Vw", + "version": { + "number": "8.4.2", + "build_flavor": "default", + "build_type": "docker", + "build_hash": "89f8c6d8429db93b816403ee75e5c270b43a940a", + "build_date": "2022-09-14T16:26:04.382547801Z", + "build_snapshot": false, + "lucene_version": "9.3.0", + "minimum_wire_compatibility_version": "7.17.0", + "minimum_index_compatibility_version": "7.0.0" + }, + "tagline": "You Know, for Search" +} diff --git a/src/go/collectors/go.d.plugin/modules/elasticsearch/testdata/v8.4.2/nodes_local_stats.json b/src/go/collectors/go.d.plugin/modules/elasticsearch/testdata/v8.4.2/nodes_local_stats.json new file mode 100644 index 000000000..77e0ad0ba --- /dev/null +++ b/src/go/collectors/go.d.plugin/modules/elasticsearch/testdata/v8.4.2/nodes_local_stats.json @@ -0,0 +1,867 @@ +{ + "_nodes": { + "total": 1, + "successful": 1, + "failed": 0 + }, + "cluster_name": "36928dce44074ceba64d7b3d698443a7", + "nodes": { + "Klg1CjgMTouentQcJlRGuA": { + "timestamp": 1687867033043, + "name": "instance-0000000006", + "transport_address": "172.25.238.204:19349", + "host": "172.25.238.204", + "ip": "172.25.238.204:19349", + "roles": [ + "data_content", + "data_hot", + "ingest", + "master", + "remote_cluster_client", + "transform" + ], + "attributes": { + "xpack.installed": "true", + "logical_availability_zone": "zone-0", + "availability_zone": "us-east-1a", + "region": "us-east-1", + "instance_configuration": "aws.es.datahot.i3", + "server_name": "instance-0000000006.36928dce44074ceba64d7b3d698443a7", + "data": "hot" + }, + "indices": { + "docs": { + "count": 403212527, + "deleted": 2287 + }, + "shard_stats": { + "total_count": 97 + }, + "store": { + "size_in_bytes": 189816312947, + "total_data_set_size_in_bytes": 189816312947, + "reserved_in_bytes": 0 + }, + "indexing": { + "index_total": 3667793202, + "index_time_in_millis": 1100149051, + "index_current": 1, + "index_failed": 149288, + "delete_total": 13333, + "delete_time_in_millis": 1883, + "delete_current": 0, + "noop_update_total": 0, + "is_throttled": false, + "throttle_time_in_millis": 0 + }, + "get": { + "total": 7502889, + "time_in_millis": 747395, + "exists_total": 7411696, + "exists_time_in_millis": 741794, + "missing_total": 91193, + "missing_time_in_millis": 5601, + "current": 0 + }, + "search": { + "open_contexts": 0, + "query_total": 166823028, + "query_time_in_millis": 51265805, + "query_current": 0, + "fetch_total": 42645288, + "fetch_time_in_millis": 21316820, + "fetch_current": 0, + "scroll_total": 13037388, + "scroll_time_in_millis": 138762688, + "scroll_current": 0, + "suggest_total": 0, + "suggest_time_in_millis": 0, + "suggest_current": 0 + }, + "merges": { + "current": 0, + "current_docs": 0, + "current_size_in_bytes": 0, + "total": 912669, + "total_time_in_millis": 1022950085, + "total_docs": 12230404828, + "total_size_in_bytes": 5503526044088, + "total_stopped_time_in_millis": 3959107, + "total_throttled_time_in_millis": 747116999, + "total_auto_throttle_in_bytes": 3674596384 + }, + "refresh": { + "total": 7721472, + "total_time_in_millis": 94304142, + "external_total": 7659770, + "external_total_time_in_millis": 100804787, + "listeners": 0 + }, + "flush": { + "total": 35134, + "periodic": 34985, + "total_time_in_millis": 22213090 + }, + "warmer": { + "current": 0, + "total": 6096195, + "total_time_in_millis": 1439617 + }, + "query_cache": { + "memory_size_in_bytes": 18034237, + "total_count": 274407233, + "hit_count": 45114414, + "miss_count": 229292819, + "cache_size": 11302, + "cache_count": 46210, + "evictions": 34908 + }, + "fielddata": { + "memory_size_in_bytes": 600, + "evictions": 0 + }, + "completion": { + "size_in_bytes": 0 + }, + "segments": { + "count": 307, + "memory_in_bytes": 0, + "terms_memory_in_bytes": 0, + "stored_fields_memory_in_bytes": 0, + "term_vectors_memory_in_bytes": 0, + "norms_memory_in_bytes": 0, + "points_memory_in_bytes": 0, + "doc_values_memory_in_bytes": 0, + "index_writer_memory_in_bytes": 240481008, + "version_map_memory_in_bytes": 44339216, + "fixed_bit_set_memory_in_bytes": 2008, + "max_unsafe_auto_id_timestamp": 1679747033889, + "file_sizes": {} + }, + "translog": { + "operations": 362831, + "size_in_bytes": 453491882, + "uncommitted_operations": 362831, + "uncommitted_size_in_bytes": 453491882, + "earliest_last_modified_age": 8 + }, + "request_cache": { + "memory_size_in_bytes": 6779720, + "evictions": 0, + "hit_count": 10885151, + "miss_count": 8798 + }, + "recovery": { + "current_as_source": 0, + "current_as_target": 0, + "throttle_time_in_millis": 5718894 + }, + "bulk": { + "total_operations": 465694640, + "total_time_in_millis": 1118684280, + "total_size_in_bytes": 3998536502390, + "avg_time_in_millis": 0, + "avg_size_in_bytes": 8526 + } + }, + "os": { + "timestamp": 1687867033054, + "cpu": { + "percent": 11, + "load_average": { + "1m": 1.24, + "5m": 2.15, + "15m": 2.39 + } + }, + "mem": { + "total_in_bytes": 16106127360, + "adjusted_total_in_bytes": 15728640000, + "free_in_bytes": 517578752, + "used_in_bytes": 15588548608, + "free_percent": 3, + "used_percent": 97 + }, + "swap": { + "total_in_bytes": 0, + "free_in_bytes": 0, + "used_in_bytes": 0 + }, + "cgroup": { + "cpuacct": { + "control_group": "/", + "usage_nanos": 2633246338856561 + }, + "cpu": { + "control_group": "/", + "cfs_period_micros": 100000, + "cfs_quota_micros": 206897, + "stat": { + "number_of_elapsed_periods": 110099433, + "number_of_times_throttled": 389045, + "time_throttled_nanos": 34502349002867 + } + }, + "memory": { + "control_group": "/", + "limit_in_bytes": "16106127360", + "usage_in_bytes": "15588548608" + } + } + }, + "process": { + "timestamp": 1687867033054, + "open_file_descriptors": 1149, + "max_file_descriptors": 1048576, + "cpu": { + "percent": 11, + "total_in_millis": 2576219400 + }, + "mem": { + "total_virtual_in_bytes": 117744459776 + } + }, + "jvm": { + "timestamp": 1687867033055, + "uptime_in_millis": 11286453256, + "mem": { + "heap_used_in_bytes": 4337402488, + "heap_used_percent": 55, + "heap_committed_in_bytes": 7864320000, + "heap_max_in_bytes": 7864320000, + "non_heap_used_in_bytes": 343633376, + "non_heap_committed_in_bytes": 350355456, + "pools": { + "young": { + "used_in_bytes": 2654994432, + "max_in_bytes": 0, + "peak_used_in_bytes": 4718592000, + "peak_max_in_bytes": 0 + }, + "old": { + "used_in_bytes": 1413394432, + "max_in_bytes": 7864320000, + "peak_used_in_bytes": 2444862976, + "peak_max_in_bytes": 7864320000 + }, + "survivor": { + "used_in_bytes": 269013624, + "max_in_bytes": 0, + "peak_used_in_bytes": 591396864, + "peak_max_in_bytes": 0 + } + } + }, + "threads": { + "count": 112, + "peak_count": 117 + }, + "gc": { + "collectors": { + "young": { + "collection_count": 78661, + "collection_time_in_millis": 6014901 + }, + "old": { + "collection_count": 0, + "collection_time_in_millis": 0 + } + } + }, + "buffer_pools": { + "mapped": { + "count": 844, + "used_in_bytes": 103411995802, + "total_capacity_in_bytes": 103411995802 + }, + "direct": { + "count": 94, + "used_in_bytes": 4654850, + "total_capacity_in_bytes": 4654848 + }, + "mapped - 'non-volatile memory'": { + "count": 0, + "used_in_bytes": 0, + "total_capacity_in_bytes": 0 + } + }, + "classes": { + "current_loaded_count": 36006, + "total_loaded_count": 37829, + "total_unloaded_count": 1823 + } + }, + "thread_pool": { + "analyze": { + "threads": 0, + "queue": 0, + "active": 0, + "rejected": 0, + "largest": 0, + "completed": 0 + }, + "auto_complete": { + "threads": 0, + "queue": 0, + "active": 0, + "rejected": 0, + "largest": 0, + "completed": 0 + }, + "azure_event_loop": { + "threads": 0, + "queue": 0, + "active": 0, + "rejected": 0, + "largest": 0, + "completed": 0 + }, + "ccr": { + "threads": 0, + "queue": 0, + "active": 0, + "rejected": 0, + "largest": 0, + "completed": 0 + }, + "cluster_coordination": { + "threads": 1, + "queue": 0, + "active": 0, + "rejected": 0, + "largest": 1, + "completed": 1130226 + }, + "fetch_shard_started": { + "threads": 0, + "queue": 0, + "active": 0, + "rejected": 0, + "largest": 0, + "completed": 0 + }, + "fetch_shard_store": { + "threads": 1, + "queue": 0, + "active": 0, + "rejected": 0, + "largest": 6, + "completed": 38 + }, + "flush": { + "threads": 2, + "queue": 0, + "active": 0, + "rejected": 0, + "largest": 2, + "completed": 89892 + }, + "force_merge": { + "threads": 1, + "queue": 0, + "active": 0, + "rejected": 0, + "largest": 1, + "completed": 143 + }, + "generic": { + "threads": 46, + "queue": 0, + "active": 0, + "rejected": 0, + "largest": 46, + "completed": 89722038 + }, + "get": { + "threads": 0, + "queue": 0, + "active": 0, + "rejected": 0, + "largest": 0, + "completed": 0 + }, + "management": { + "threads": 3, + "queue": 0, + "active": 1, + "rejected": 0, + "largest": 3, + "completed": 416796779 + }, + "ml_datafeed": { + "threads": 0, + "queue": 0, + "active": 0, + "rejected": 0, + "largest": 0, + "completed": 0 + }, + "ml_job_comms": { + "threads": 0, + "queue": 0, + "active": 0, + "rejected": 0, + "largest": 0, + "completed": 0 + }, + "ml_native_inference_comms": { + "threads": 0, + "queue": 0, + "active": 0, + "rejected": 0, + "largest": 0, + "completed": 0 + }, + "ml_utility": { + "threads": 2, + "queue": 0, + "active": 0, + "rejected": 0, + "largest": 2, + "completed": 22545252 + }, + "refresh": { + "threads": 2, + "queue": 0, + "active": 0, + "rejected": 0, + "largest": 2, + "completed": 885152069 + }, + "repository_azure": { + "threads": 0, + "queue": 0, + "active": 0, + "rejected": 0, + "largest": 0, + "completed": 0 + }, + "rollup_indexing": { + "threads": 0, + "queue": 0, + "active": 0, + "rejected": 0, + "largest": 0, + "completed": 0 + }, + "search": { + "threads": 5, + "queue": 0, + "active": 0, + "rejected": 0, + "largest": 5, + "completed": 167558865 + }, + "search_coordination": { + "threads": 2, + "queue": 0, + "active": 0, + "rejected": 0, + "largest": 2, + "completed": 14101096 + }, + "search_throttled": { + "threads": 0, + "queue": 0, + "active": 0, + "rejected": 0, + "largest": 0, + "completed": 0 + }, + "searchable_snapshots_cache_fetch_async": { + "threads": 0, + "queue": 0, + "active": 0, + "rejected": 0, + "largest": 0, + "completed": 0 + }, + "searchable_snapshots_cache_prewarming": { + "threads": 0, + "queue": 0, + "active": 0, + "rejected": 0, + "largest": 0, + "completed": 0 + }, + "security-crypto": { + "threads": 0, + "queue": 0, + "active": 0, + "rejected": 0, + "largest": 0, + "completed": 0 + }, + "security-token-key": { + "threads": 0, + "queue": 0, + "active": 0, + "rejected": 0, + "largest": 0, + "completed": 0 + }, + "snapshot": { + "threads": 1, + "queue": 0, + "active": 0, + "rejected": 0, + "largest": 2, + "completed": 806551 + }, + "snapshot_meta": { + "threads": 0, + "queue": 0, + "active": 0, + "rejected": 0, + "largest": 0, + "completed": 0 + }, + "system_critical_read": { + "threads": 2, + "queue": 0, + "active": 0, + "rejected": 0, + "largest": 2, + "completed": 2350943 + }, + "system_critical_write": { + "threads": 2, + "queue": 0, + "active": 0, + "rejected": 0, + "largest": 2, + "completed": 7637 + }, + "system_read": { + "threads": 2, + "queue": 0, + "active": 0, + "rejected": 0, + "largest": 2, + "completed": 31143771 + }, + "system_write": { + "threads": 2, + "queue": 0, + "active": 0, + "rejected": 0, + "largest": 2, + "completed": 7401359 + }, + "vector_tile_generation": { + "threads": 0, + "queue": 0, + "active": 0, + "rejected": 0, + "largest": 0, + "completed": 0 + }, + "warmer": { + "threads": 2, + "queue": 0, + "active": 0, + "rejected": 0, + "largest": 2, + "completed": 36139188 + }, + "watcher": { + "threads": 0, + "queue": 0, + "active": 0, + "rejected": 0, + "largest": 0, + "completed": 0 + }, + "write": { + "threads": 3, + "queue": 0, + "active": 2, + "rejected": 0, + "largest": 3, + "completed": 575385289 + } + }, + "fs": { + "timestamp": 1687867033056, + "total": { + "total_in_bytes": 483183820800, + "free_in_bytes": 292670836736, + "available_in_bytes": 292670836736 + }, + "data": [ + { + "path": "/app/data", + "mount": "/app (/dev/mapper/lxc-data)", + "type": "xfs", + "total_in_bytes": 483183820800, + "free_in_bytes": 292670836736, + "available_in_bytes": 292670836736 + } + ], + "io_stats": { + "devices": [ + { + "device_name": "dm-1", + "operations": 6160920260, + "read_operations": 376565165, + "write_operations": 5784355095, + "read_kilobytes": 31265075012, + "write_kilobytes": 100985041837, + "io_time_in_millis": 184335640 + } + ], + "total": { + "operations": 6160920260, + "read_operations": 376565165, + "write_operations": 5784355095, + "read_kilobytes": 31265075012, + "write_kilobytes": 100985041837, + "io_time_in_millis": 184335640 + } + } + }, + "transport": { + "server_open": 24, + "total_outbound_connections": 11, + "rx_count": 1300468666, + "rx_size_in_bytes": 1789647854011, + "tx_count": 1300468665, + "tx_size_in_bytes": 2927853534431, + "inbound_handling_time_histogram": [ + { + "lt_millis": 1, + "count": 1256244956 + }, + { + "ge_millis": 1, + "lt_millis": 2, + "count": 202091898 + }, + { + "ge_millis": 2, + "lt_millis": 4, + "count": 3242593 + }, + { + "ge_millis": 4, + "lt_millis": 8, + "count": 454964 + }, + { + "ge_millis": 8, + "lt_millis": 16, + "count": 173349 + }, + { + "ge_millis": 16, + "lt_millis": 32, + "count": 39048 + }, + { + "ge_millis": 32, + "lt_millis": 64, + "count": 14155 + }, + { + "ge_millis": 64, + "lt_millis": 128, + "count": 75267 + }, + { + "ge_millis": 128, + "lt_millis": 256, + "count": 1534 + }, + { + "ge_millis": 256, + "lt_millis": 512, + "count": 76 + }, + { + "ge_millis": 512, + "lt_millis": 1024, + "count": 3 + }, + { + "ge_millis": 1024, + "lt_millis": 2048, + "count": 0 + }, + { + "ge_millis": 2048, + "lt_millis": 4096, + "count": 0 + }, + { + "ge_millis": 4096, + "lt_millis": 8192, + "count": 0 + }, + { + "ge_millis": 8192, + "lt_millis": 16384, + "count": 0 + }, + { + "ge_millis": 16384, + "lt_millis": 32768, + "count": 0 + }, + { + "ge_millis": 32768, + "lt_millis": 65536, + "count": 0 + }, + { + "ge_millis": 65536, + "count": 0 + } + ], + "outbound_handling_time_histogram": [ + { + "lt_millis": 1, + "count": 1128511214 + }, + { + "ge_millis": 1, + "lt_millis": 2, + "count": 161858180 + }, + { + "ge_millis": 2, + "lt_millis": 4, + "count": 6819172 + }, + { + "ge_millis": 4, + "lt_millis": 8, + "count": 2563797 + }, + { + "ge_millis": 8, + "lt_millis": 16, + "count": 445824 + }, + { + "ge_millis": 16, + "lt_millis": 32, + "count": 122462 + }, + { + "ge_millis": 32, + "lt_millis": 64, + "count": 95822 + }, + { + "ge_millis": 64, + "lt_millis": 128, + "count": 49986 + }, + { + "ge_millis": 128, + "lt_millis": 256, + "count": 1931 + }, + { + "ge_millis": 256, + "lt_millis": 512, + "count": 250 + }, + { + "ge_millis": 512, + "lt_millis": 1024, + "count": 27 + }, + { + "ge_millis": 1024, + "lt_millis": 2048, + "count": 0 + }, + { + "ge_millis": 2048, + "lt_millis": 4096, + "count": 0 + }, + { + "ge_millis": 4096, + "lt_millis": 8192, + "count": 0 + }, + { + "ge_millis": 8192, + "lt_millis": 16384, + "count": 0 + }, + { + "ge_millis": 16384, + "lt_millis": 32768, + "count": 0 + }, + { + "ge_millis": 32768, + "lt_millis": 65536, + "count": 0 + }, + { + "ge_millis": 65536, + "count": 0 + } + ] + }, + "http": { + "current_open": 73, + "total_opened": 779388 + }, + "breakers": { + "fielddata": { + "limit_size_in_bytes": 3145728000, + "limit_size": "2.9gb", + "estimated_size_in_bytes": 600, + "estimated_size": "600b", + "overhead": 1.03, + "tripped": 0 + }, + "request": { + "limit_size_in_bytes": 4718592000, + "limit_size": "4.3gb", + "estimated_size_in_bytes": 16440, + "estimated_size": "16kb", + "overhead": 1, + "tripped": 0 + }, + "inflight_requests": { + "limit_size_in_bytes": 7864320000, + "limit_size": "7.3gb", + "estimated_size_in_bytes": 56628, + "estimated_size": "55.3kb", + "overhead": 2, + "tripped": 0 + }, + "model_inference": { + "limit_size_in_bytes": 3932160000, + "limit_size": "3.6gb", + "estimated_size_in_bytes": 0, + "estimated_size": "0b", + "overhead": 1, + "tripped": 0 + }, + "eql_sequence": { + "limit_size_in_bytes": 3932160000, + "limit_size": "3.6gb", + "estimated_size_in_bytes": 0, + "estimated_size": "0b", + "overhead": 1, + "tripped": 0 + }, + "parent": { + "limit_size_in_bytes": 7471104000, + "limit_size": "6.9gb", + "estimated_size_in_bytes": 4341596792, + "estimated_size": "4gb", + "overhead": 1, + "tripped": 0 + } + } + } + } +} diff --git a/src/go/collectors/go.d.plugin/modules/elasticsearch/testdata/v8.4.2/nodes_stats.json b/src/go/collectors/go.d.plugin/modules/elasticsearch/testdata/v8.4.2/nodes_stats.json new file mode 100644 index 000000000..6e6b21b91 --- /dev/null +++ b/src/go/collectors/go.d.plugin/modules/elasticsearch/testdata/v8.4.2/nodes_stats.json @@ -0,0 +1,2577 @@ +{ + "_nodes": { + "total": 3, + "successful": 3, + "failed": 0 + }, + "cluster_name": "36928dce44074ceba64d7b3d698443a7", + "nodes": { + "tk_U7GMCRkCG4FoOvusrng": { + "timestamp": 1687866153482, + "name": "instance-0000000005", + "transport_address": "172.22.146.77:19280", + "host": "172.22.146.77", + "ip": "172.22.146.77:19280", + "roles": [ + "data_content", + "data_hot", + "ingest", + "master", + "remote_cluster_client", + "transform" + ], + "attributes": { + "instance_configuration": "aws.es.datahot.i3", + "server_name": "instance-0000000005.36928dce44074ceba64d7b3d698443a7", + "data": "hot", + "xpack.installed": "true", + "logical_availability_zone": "zone-1", + "availability_zone": "us-east-1e", + "region": "us-east-1" + }, + "indices": { + "docs": { + "count": 403028528, + "deleted": 430916 + }, + "shard_stats": { + "total_count": 97 + }, + "store": { + "size_in_bytes": 190773977702, + "total_data_set_size_in_bytes": 190773977702, + "reserved_in_bytes": 0 + }, + "indexing": { + "index_total": 6550378755, + "index_time_in_millis": 1244633519, + "index_current": 0, + "index_failed": 3425, + "delete_total": 422502, + "delete_time_in_millis": 12139, + "delete_current": 0, + "noop_update_total": 0, + "is_throttled": false, + "throttle_time_in_millis": 0 + }, + "get": { + "total": 1673415, + "time_in_millis": 176085, + "exists_total": 1505245, + "exists_time_in_millis": 164637, + "missing_total": 168170, + "missing_time_in_millis": 11448, + "current": 0 + }, + "search": { + "open_contexts": 0, + "query_total": 157912598, + "query_time_in_millis": 158980385, + "query_current": 0, + "fetch_total": 25105951, + "fetch_time_in_millis": 24517851, + "fetch_current": 0, + "scroll_total": 4428540, + "scroll_time_in_millis": 153962443, + "scroll_current": 0, + "suggest_total": 0, + "suggest_time_in_millis": 0, + "suggest_current": 0 + }, + "merges": { + "current": 1, + "current_docs": 1768114, + "current_size_in_bytes": 954513675, + "total": 1494757, + "total_time_in_millis": 1621446531, + "total_docs": 21027016560, + "total_size_in_bytes": 8884898196658, + "total_stopped_time_in_millis": 4962617, + "total_throttled_time_in_millis": 1169888193, + "total_auto_throttle_in_bytes": 4651560300 + }, + "refresh": { + "total": 12359783, + "total_time_in_millis": 300152615, + "external_total": 12278845, + "external_total_time_in_millis": 311222562, + "listeners": 0 + }, + "flush": { + "total": 67895, + "periodic": 67579, + "total_time_in_millis": 81917283 + }, + "warmer": { + "current": 0, + "total": 6153265, + "total_time_in_millis": 1348469 + }, + "query_cache": { + "memory_size_in_bytes": 19433507, + "total_count": 410202459, + "hit_count": 51724734, + "miss_count": 358477725, + "cache_size": 11311, + "cache_count": 45151, + "evictions": 33840 + }, + "fielddata": { + "memory_size_in_bytes": 0, + "evictions": 0 + }, + "completion": { + "size_in_bytes": 0 + }, + "segments": { + "count": 291, + "memory_in_bytes": 0, + "terms_memory_in_bytes": 0, + "stored_fields_memory_in_bytes": 0, + "term_vectors_memory_in_bytes": 0, + "norms_memory_in_bytes": 0, + "points_memory_in_bytes": 0, + "doc_values_memory_in_bytes": 0, + "index_writer_memory_in_bytes": 57432664, + "version_map_memory_in_bytes": 568, + "fixed_bit_set_memory_in_bytes": 55672, + "max_unsafe_auto_id_timestamp": 1676581446329, + "file_sizes": {} + }, + "translog": { + "operations": 1449698, + "size_in_bytes": 1214204014, + "uncommitted_operations": 1449698, + "uncommitted_size_in_bytes": 1214204014, + "earliest_last_modified_age": 14453 + }, + "request_cache": { + "memory_size_in_bytes": 6178272, + "evictions": 0, + "hit_count": 7403041, + "miss_count": 10622 + }, + "recovery": { + "current_as_source": 0, + "current_as_target": 0, + "throttle_time_in_millis": 48470343 + }, + "bulk": { + "total_operations": 783008788, + "total_time_in_millis": 1265052645, + "total_size_in_bytes": 6949964886117, + "avg_time_in_millis": 0, + "avg_size_in_bytes": 8635 + } + }, + "os": { + "timestamp": 1687866153489, + "cpu": { + "percent": 9, + "load_average": { + "1m": 0.83, + "5m": 1.1, + "15m": 1.3 + } + }, + "mem": { + "total_in_bytes": 16106127360, + "adjusted_total_in_bytes": 15728640000, + "free_in_bytes": 1425637376, + "used_in_bytes": 14680489984, + "free_percent": 9, + "used_percent": 91 + }, + "swap": { + "total_in_bytes": 0, + "free_in_bytes": 0, + "used_in_bytes": 0 + }, + "cgroup": { + "cpuacct": { + "control_group": "/", + "usage_nanos": 4328157929052960 + }, + "cpu": { + "control_group": "/", + "cfs_period_micros": 100000, + "cfs_quota_micros": 206897, + "stat": { + "number_of_elapsed_periods": 198258313, + "number_of_times_throttled": 619367, + "time_throttled_nanos": 45229163024496 + } + }, + "memory": { + "control_group": "/", + "limit_in_bytes": "16106127360", + "usage_in_bytes": "14680489984" + } + } + }, + "process": { + "timestamp": 1687866153489, + "open_file_descriptors": 1180, + "max_file_descriptors": 1048576, + "cpu": { + "percent": 9, + "total_in_millis": 3994216500 + }, + "mem": { + "total_virtual_in_bytes": 114185707520 + } + }, + "jvm": { + "timestamp": 1687866153490, + "uptime_in_millis": 20231050756, + "mem": { + "heap_used_in_bytes": 1884124192, + "heap_used_percent": 23, + "heap_committed_in_bytes": 7864320000, + "heap_max_in_bytes": 7864320000, + "non_heap_used_in_bytes": 376433344, + "non_heap_committed_in_bytes": 385548288, + "pools": { + "young": { + "used_in_bytes": 385875968, + "max_in_bytes": 0, + "peak_used_in_bytes": 4714397696, + "peak_max_in_bytes": 0 + }, + "old": { + "used_in_bytes": 1399682080, + "max_in_bytes": 7864320000, + "peak_used_in_bytes": 7851651072, + "peak_max_in_bytes": 7864320000 + }, + "survivor": { + "used_in_bytes": 98566144, + "max_in_bytes": 0, + "peak_used_in_bytes": 591396864, + "peak_max_in_bytes": 0 + } + } + }, + "threads": { + "count": 115, + "peak_count": 126 + }, + "gc": { + "collectors": { + "young": { + "collection_count": 139959, + "collection_time_in_millis": 3581668 + }, + "old": { + "collection_count": 1, + "collection_time_in_millis": 796 + } + } + }, + "buffer_pools": { + "mapped": { + "count": 831, + "used_in_bytes": 99844219805, + "total_capacity_in_bytes": 99844219805 + }, + "direct": { + "count": 90, + "used_in_bytes": 4571713, + "total_capacity_in_bytes": 4571711 + }, + "mapped - 'non-volatile memory'": { + "count": 0, + "used_in_bytes": 0, + "total_capacity_in_bytes": 0 + } + }, + "classes": { + "current_loaded_count": 38122, + "total_loaded_count": 40402, + "total_unloaded_count": 2280 + } + }, + "thread_pool": { + "analyze": { + "threads": 0, + "queue": 0, + "active": 0, + "rejected": 0, + "largest": 0, + "completed": 0 + }, + "auto_complete": { + "threads": 1, + "queue": 0, + "active": 0, + "rejected": 0, + "largest": 1, + "completed": 1 + }, + "azure_event_loop": { + "threads": 0, + "queue": 0, + "active": 0, + "rejected": 0, + "largest": 0, + "completed": 0 + }, + "ccr": { + "threads": 0, + "queue": 0, + "active": 0, + "rejected": 0, + "largest": 0, + "completed": 0 + }, + "cluster_coordination": { + "threads": 1, + "queue": 0, + "active": 0, + "rejected": 0, + "largest": 1, + "completed": 4427981 + }, + "fetch_shard_started": { + "threads": 0, + "queue": 0, + "active": 0, + "rejected": 0, + "largest": 0, + "completed": 0 + }, + "fetch_shard_store": { + "threads": 1, + "queue": 0, + "active": 0, + "rejected": 0, + "largest": 6, + "completed": 72 + }, + "flush": { + "threads": 2, + "queue": 0, + "active": 0, + "rejected": 0, + "largest": 2, + "completed": 166429 + }, + "force_merge": { + "threads": 1, + "queue": 0, + "active": 0, + "rejected": 0, + "largest": 1, + "completed": 205 + }, + "generic": { + "threads": 40, + "queue": 0, + "active": 0, + "rejected": 0, + "largest": 40, + "completed": 171078109 + }, + "get": { + "threads": 0, + "queue": 0, + "active": 0, + "rejected": 0, + "largest": 0, + "completed": 0 + }, + "management": { + "threads": 3, + "queue": 0, + "active": 1, + "rejected": 0, + "largest": 3, + "completed": 761997145 + }, + "ml_datafeed": { + "threads": 0, + "queue": 0, + "active": 0, + "rejected": 0, + "largest": 0, + "completed": 0 + }, + "ml_job_comms": { + "threads": 0, + "queue": 0, + "active": 0, + "rejected": 0, + "largest": 0, + "completed": 0 + }, + "ml_native_inference_comms": { + "threads": 0, + "queue": 0, + "active": 0, + "rejected": 0, + "largest": 0, + "completed": 0 + }, + "ml_utility": { + "threads": 3, + "queue": 0, + "active": 0, + "rejected": 0, + "largest": 3, + "completed": 40979576 + }, + "refresh": { + "threads": 2, + "queue": 0, + "active": 0, + "rejected": 0, + "largest": 2, + "completed": 1224783637 + }, + "repository_azure": { + "threads": 0, + "queue": 0, + "active": 0, + "rejected": 0, + "largest": 0, + "completed": 0 + }, + "rollup_indexing": { + "threads": 0, + "queue": 0, + "active": 0, + "rejected": 0, + "largest": 0, + "completed": 0 + }, + "search": { + "threads": 5, + "queue": 0, + "active": 0, + "rejected": 0, + "largest": 5, + "completed": 191798560 + }, + "search_coordination": { + "threads": 2, + "queue": 0, + "active": 0, + "rejected": 0, + "largest": 2, + "completed": 18868632 + }, + "search_throttled": { + "threads": 0, + "queue": 0, + "active": 0, + "rejected": 0, + "largest": 0, + "completed": 0 + }, + "searchable_snapshots_cache_fetch_async": { + "threads": 0, + "queue": 0, + "active": 0, + "rejected": 0, + "largest": 0, + "completed": 0 + }, + "searchable_snapshots_cache_prewarming": { + "threads": 0, + "queue": 0, + "active": 0, + "rejected": 0, + "largest": 0, + "completed": 0 + }, + "security-crypto": { + "threads": 0, + "queue": 0, + "active": 0, + "rejected": 0, + "largest": 0, + "completed": 0 + }, + "security-token-key": { + "threads": 0, + "queue": 0, + "active": 0, + "rejected": 0, + "largest": 0, + "completed": 0 + }, + "snapshot": { + "threads": 1, + "queue": 0, + "active": 0, + "rejected": 0, + "largest": 2, + "completed": 1757953 + }, + "snapshot_meta": { + "threads": 1, + "queue": 0, + "active": 0, + "rejected": 0, + "largest": 9, + "completed": 700327 + }, + "system_critical_read": { + "threads": 2, + "queue": 0, + "active": 0, + "rejected": 0, + "largest": 2, + "completed": 11110320 + }, + "system_critical_write": { + "threads": 2, + "queue": 0, + "active": 0, + "rejected": 0, + "largest": 2, + "completed": 14932 + }, + "system_read": { + "threads": 2, + "queue": 0, + "active": 0, + "rejected": 0, + "largest": 2, + "completed": 39897928 + }, + "system_write": { + "threads": 2, + "queue": 0, + "active": 0, + "rejected": 0, + "largest": 2, + "completed": 13382379 + }, + "vector_tile_generation": { + "threads": 0, + "queue": 0, + "active": 0, + "rejected": 0, + "largest": 0, + "completed": 0 + }, + "warmer": { + "threads": 2, + "queue": 0, + "active": 0, + "rejected": 0, + "largest": 2, + "completed": 85786496 + }, + "watcher": { + "threads": 0, + "queue": 0, + "active": 0, + "rejected": 0, + "largest": 0, + "completed": 0 + }, + "write": { + "threads": 3, + "queue": 0, + "active": 0, + "rejected": 0, + "largest": 3, + "completed": 980512922 + } + }, + "fs": { + "timestamp": 1687866153490, + "total": { + "total_in_bytes": 483183820800, + "free_in_bytes": 290682736640, + "available_in_bytes": 290682736640 + }, + "data": [ + { + "path": "/app/data", + "mount": "/app (/dev/mapper/lxc-data)", + "type": "xfs", + "total_in_bytes": 483183820800, + "free_in_bytes": 290682736640, + "available_in_bytes": 290682736640 + } + ], + "io_stats": { + "devices": [ + { + "device_name": "dm-1", + "operations": 5478832410, + "read_operations": 89263106, + "write_operations": 5389569304, + "read_kilobytes": 9500415196, + "write_kilobytes": 67144441274, + "io_time_in_millis": 271723584 + } + ], + "total": { + "operations": 5478832410, + "read_operations": 89263106, + "write_operations": 5389569304, + "read_kilobytes": 9500415196, + "write_kilobytes": 67144441274, + "io_time_in_millis": 271723584 + } + } + }, + "transport": { + "server_open": 24, + "total_outbound_connections": 9, + "rx_count": 2167879292, + "rx_size_in_bytes": 4905919297323, + "tx_count": 2167879293, + "tx_size_in_bytes": 2964638852652, + "inbound_handling_time_histogram": [ + { + "lt_millis": 1, + "count": 2149806152 + }, + { + "ge_millis": 1, + "lt_millis": 2, + "count": 350125308 + }, + { + "ge_millis": 2, + "lt_millis": 4, + "count": 6237311 + }, + { + "ge_millis": 4, + "lt_millis": 8, + "count": 3462010 + }, + { + "ge_millis": 8, + "lt_millis": 16, + "count": 1695688 + }, + { + "ge_millis": 16, + "lt_millis": 32, + "count": 446932 + }, + { + "ge_millis": 32, + "lt_millis": 64, + "count": 34053 + }, + { + "ge_millis": 64, + "lt_millis": 128, + "count": 124821 + }, + { + "ge_millis": 128, + "lt_millis": 256, + "count": 1034 + }, + { + "ge_millis": 256, + "lt_millis": 512, + "count": 47 + }, + { + "ge_millis": 512, + "lt_millis": 1024, + "count": 7 + }, + { + "ge_millis": 1024, + "lt_millis": 2048, + "count": 1 + }, + { + "ge_millis": 2048, + "lt_millis": 4096, + "count": 2 + }, + { + "ge_millis": 4096, + "lt_millis": 8192, + "count": 0 + }, + { + "ge_millis": 8192, + "lt_millis": 16384, + "count": 0 + }, + { + "ge_millis": 16384, + "lt_millis": 32768, + "count": 0 + }, + { + "ge_millis": 32768, + "lt_millis": 65536, + "count": 0 + }, + { + "ge_millis": 65536, + "count": 0 + } + ], + "outbound_handling_time_histogram": [ + { + "lt_millis": 1, + "count": 1911876454 + }, + { + "ge_millis": 1, + "lt_millis": 2, + "count": 246835312 + }, + { + "ge_millis": 2, + "lt_millis": 4, + "count": 5928518 + }, + { + "ge_millis": 4, + "lt_millis": 8, + "count": 2342608 + }, + { + "ge_millis": 8, + "lt_millis": 16, + "count": 566388 + }, + { + "ge_millis": 16, + "lt_millis": 32, + "count": 164795 + }, + { + "ge_millis": 32, + "lt_millis": 64, + "count": 91456 + }, + { + "ge_millis": 64, + "lt_millis": 128, + "count": 68952 + }, + { + "ge_millis": 128, + "lt_millis": 256, + "count": 3952 + }, + { + "ge_millis": 256, + "lt_millis": 512, + "count": 772 + }, + { + "ge_millis": 512, + "lt_millis": 1024, + "count": 51 + }, + { + "ge_millis": 1024, + "lt_millis": 2048, + "count": 25 + }, + { + "ge_millis": 2048, + "lt_millis": 4096, + "count": 10 + }, + { + "ge_millis": 4096, + "lt_millis": 8192, + "count": 0 + }, + { + "ge_millis": 8192, + "lt_millis": 16384, + "count": 0 + }, + { + "ge_millis": 16384, + "lt_millis": 32768, + "count": 0 + }, + { + "ge_millis": 32768, + "lt_millis": 65536, + "count": 0 + }, + { + "ge_millis": 65536, + "count": 0 + } + ] + }, + "http": { + "current_open": 84, + "total_opened": 1793320 + }, + "breakers": { + "model_inference": { + "limit_size_in_bytes": 3932160000, + "limit_size": "3.6gb", + "estimated_size_in_bytes": 0, + "estimated_size": "0b", + "overhead": 1, + "tripped": 0 + }, + "eql_sequence": { + "limit_size_in_bytes": 3932160000, + "limit_size": "3.6gb", + "estimated_size_in_bytes": 0, + "estimated_size": "0b", + "overhead": 1, + "tripped": 0 + }, + "fielddata": { + "limit_size_in_bytes": 3145728000, + "limit_size": "2.9gb", + "estimated_size_in_bytes": 0, + "estimated_size": "0b", + "overhead": 1.03, + "tripped": 0 + }, + "request": { + "limit_size_in_bytes": 4718592000, + "limit_size": "4.3gb", + "estimated_size_in_bytes": 0, + "estimated_size": "0b", + "overhead": 1, + "tripped": 1 + }, + "inflight_requests": { + "limit_size_in_bytes": 7864320000, + "limit_size": "7.3gb", + "estimated_size_in_bytes": 0, + "estimated_size": "0b", + "overhead": 2, + "tripped": 0 + }, + "parent": { + "limit_size_in_bytes": 7471104000, + "limit_size": "6.9gb", + "estimated_size_in_bytes": 1884124192, + "estimated_size": "1.7gb", + "overhead": 1, + "tripped": 93 + } + } + }, + "Klg1CjgMTouentQcJlRGuA": { + "timestamp": 1687866153482, + "name": "instance-0000000006", + "transport_address": "172.25.238.204:19349", + "host": "172.25.238.204", + "ip": "172.25.238.204:19349", + "roles": [ + "data_content", + "data_hot", + "ingest", + "master", + "remote_cluster_client", + "transform" + ], + "attributes": { + "logical_availability_zone": "zone-0", + "availability_zone": "us-east-1a", + "server_name": "instance-0000000006.36928dce44074ceba64d7b3d698443a7", + "xpack.installed": "true", + "data": "hot", + "instance_configuration": "aws.es.datahot.i3", + "region": "us-east-1" + }, + "indices": { + "docs": { + "count": 402750701, + "deleted": 1501 + }, + "shard_stats": { + "total_count": 97 + }, + "store": { + "size_in_bytes": 189584860329, + "total_data_set_size_in_bytes": 189584860329, + "reserved_in_bytes": 0 + }, + "indexing": { + "index_total": 3667364815, + "index_time_in_millis": 1100012973, + "index_current": 0, + "index_failed": 149288, + "delete_total": 13333, + "delete_time_in_millis": 1883, + "delete_current": 0, + "noop_update_total": 0, + "is_throttled": false, + "throttle_time_in_millis": 0 + }, + "get": { + "total": 7502285, + "time_in_millis": 747339, + "exists_total": 7411100, + "exists_time_in_millis": 741739, + "missing_total": 91185, + "missing_time_in_millis": 5600, + "current": 0 + }, + "search": { + "open_contexts": 0, + "query_total": 166820275, + "query_time_in_millis": 51262303, + "query_current": 0, + "fetch_total": 42642621, + "fetch_time_in_millis": 21316723, + "fetch_current": 0, + "scroll_total": 13036366, + "scroll_time_in_millis": 138752334, + "scroll_current": 0, + "suggest_total": 0, + "suggest_time_in_millis": 0, + "suggest_current": 0 + }, + "merges": { + "current": 0, + "current_docs": 0, + "current_size_in_bytes": 0, + "total": 912589, + "total_time_in_millis": 1022946643, + "total_docs": 12230248422, + "total_size_in_bytes": 5503433306347, + "total_stopped_time_in_millis": 3959107, + "total_throttled_time_in_millis": 747116999, + "total_auto_throttle_in_bytes": 3674596384 + }, + "refresh": { + "total": 7720800, + "total_time_in_millis": 94297737, + "external_total": 7659102, + "external_total_time_in_millis": 100797967, + "listeners": 0 + }, + "flush": { + "total": 35130, + "periodic": 34981, + "total_time_in_millis": 22204637 + }, + "warmer": { + "current": 0, + "total": 6095530, + "total_time_in_millis": 1439528 + }, + "query_cache": { + "memory_size_in_bytes": 18032444, + "total_count": 274404002, + "hit_count": 45113976, + "miss_count": 229290026, + "cache_size": 11260, + "cache_count": 46168, + "evictions": 34908 + }, + "fielddata": { + "memory_size_in_bytes": 600, + "evictions": 0 + }, + "completion": { + "size_in_bytes": 0 + }, + "segments": { + "count": 320, + "memory_in_bytes": 0, + "terms_memory_in_bytes": 0, + "stored_fields_memory_in_bytes": 0, + "term_vectors_memory_in_bytes": 0, + "norms_memory_in_bytes": 0, + "points_memory_in_bytes": 0, + "doc_values_memory_in_bytes": 0, + "index_writer_memory_in_bytes": 262022568, + "version_map_memory_in_bytes": 49200018, + "fixed_bit_set_memory_in_bytes": 1904, + "max_unsafe_auto_id_timestamp": 1679747033889, + "file_sizes": {} + }, + "translog": { + "operations": 352376, + "size_in_bytes": 447695989, + "uncommitted_operations": 352376, + "uncommitted_size_in_bytes": 447695989, + "earliest_last_modified_age": 233 + }, + "request_cache": { + "memory_size_in_bytes": 6779128, + "evictions": 0, + "hit_count": 10884306, + "miss_count": 8796 + }, + "recovery": { + "current_as_source": 0, + "current_as_target": 0, + "throttle_time_in_millis": 5718894 + }, + "bulk": { + "total_operations": 465641149, + "total_time_in_millis": 1118546460, + "total_size_in_bytes": 3998028967189, + "avg_time_in_millis": 0, + "avg_size_in_bytes": 8613 + } + }, + "os": { + "timestamp": 1687866153492, + "cpu": { + "percent": 10, + "load_average": { + "1m": 2.38, + "5m": 2.74, + "15m": 2.45 + } + }, + "mem": { + "total_in_bytes": 16106127360, + "adjusted_total_in_bytes": 15728640000, + "free_in_bytes": 765980672, + "used_in_bytes": 15340146688, + "free_percent": 5, + "used_percent": 95 + }, + "swap": { + "total_in_bytes": 0, + "free_in_bytes": 0, + "used_in_bytes": 0 + }, + "cgroup": { + "cpuacct": { + "control_group": "/", + "usage_nanos": 2632999205547019 + }, + "cpu": { + "control_group": "/", + "cfs_period_micros": 100000, + "cfs_quota_micros": 206897, + "stat": { + "number_of_elapsed_periods": 110090960, + "number_of_times_throttled": 389008, + "time_throttled_nanos": 34498461943176 + } + }, + "memory": { + "control_group": "/", + "limit_in_bytes": "16106127360", + "usage_in_bytes": "15340146688" + } + } + }, + "process": { + "timestamp": 1687866153493, + "open_file_descriptors": 1156, + "max_file_descriptors": 1048576, + "cpu": { + "percent": 10, + "total_in_millis": 2575977020 + }, + "mem": { + "total_virtual_in_bytes": 117447507968 + } + }, + "jvm": { + "timestamp": 1687866153494, + "uptime_in_millis": 11285573694, + "mem": { + "heap_used_in_bytes": 5059735552, + "heap_used_percent": 64, + "heap_committed_in_bytes": 7864320000, + "heap_max_in_bytes": 7864320000, + "non_heap_used_in_bytes": 343633376, + "non_heap_committed_in_bytes": 350355456, + "pools": { + "young": { + "used_in_bytes": 3351248896, + "max_in_bytes": 0, + "peak_used_in_bytes": 4718592000, + "peak_max_in_bytes": 0 + }, + "old": { + "used_in_bytes": 1354067968, + "max_in_bytes": 7864320000, + "peak_used_in_bytes": 2444862976, + "peak_max_in_bytes": 7864320000 + }, + "survivor": { + "used_in_bytes": 354418688, + "max_in_bytes": 0, + "peak_used_in_bytes": 591396864, + "peak_max_in_bytes": 0 + } + } + }, + "threads": { + "count": 112, + "peak_count": 117 + }, + "gc": { + "collectors": { + "young": { + "collection_count": 78652, + "collection_time_in_millis": 6014274 + }, + "old": { + "collection_count": 0, + "collection_time_in_millis": 0 + } + } + }, + "buffer_pools": { + "mapped": { + "count": 858, + "used_in_bytes": 103114998135, + "total_capacity_in_bytes": 103114998135 + }, + "direct": { + "count": 94, + "used_in_bytes": 4654850, + "total_capacity_in_bytes": 4654848 + }, + "mapped - 'non-volatile memory'": { + "count": 0, + "used_in_bytes": 0, + "total_capacity_in_bytes": 0 + } + }, + "classes": { + "current_loaded_count": 36006, + "total_loaded_count": 37829, + "total_unloaded_count": 1823 + } + }, + "thread_pool": { + "analyze": { + "threads": 0, + "queue": 0, + "active": 0, + "rejected": 0, + "largest": 0, + "completed": 0 + }, + "auto_complete": { + "threads": 0, + "queue": 0, + "active": 0, + "rejected": 0, + "largest": 0, + "completed": 0 + }, + "azure_event_loop": { + "threads": 0, + "queue": 0, + "active": 0, + "rejected": 0, + "largest": 0, + "completed": 0 + }, + "ccr": { + "threads": 0, + "queue": 0, + "active": 0, + "rejected": 0, + "largest": 0, + "completed": 0 + }, + "cluster_coordination": { + "threads": 1, + "queue": 0, + "active": 0, + "rejected": 0, + "largest": 1, + "completed": 1130214 + }, + "fetch_shard_started": { + "threads": 0, + "queue": 0, + "active": 0, + "rejected": 0, + "largest": 0, + "completed": 0 + }, + "fetch_shard_store": { + "threads": 1, + "queue": 0, + "active": 0, + "rejected": 0, + "largest": 6, + "completed": 38 + }, + "flush": { + "threads": 2, + "queue": 0, + "active": 0, + "rejected": 0, + "largest": 2, + "completed": 89882 + }, + "force_merge": { + "threads": 1, + "queue": 0, + "active": 0, + "rejected": 0, + "largest": 1, + "completed": 143 + }, + "generic": { + "threads": 46, + "queue": 0, + "active": 0, + "rejected": 0, + "largest": 46, + "completed": 89714323 + }, + "get": { + "threads": 0, + "queue": 0, + "active": 0, + "rejected": 0, + "largest": 0, + "completed": 0 + }, + "management": { + "threads": 3, + "queue": 0, + "active": 1, + "rejected": 0, + "largest": 3, + "completed": 416760833 + }, + "ml_datafeed": { + "threads": 0, + "queue": 0, + "active": 0, + "rejected": 0, + "largest": 0, + "completed": 0 + }, + "ml_job_comms": { + "threads": 0, + "queue": 0, + "active": 0, + "rejected": 0, + "largest": 0, + "completed": 0 + }, + "ml_native_inference_comms": { + "threads": 0, + "queue": 0, + "active": 0, + "rejected": 0, + "largest": 0, + "completed": 0 + }, + "ml_utility": { + "threads": 2, + "queue": 0, + "active": 0, + "rejected": 0, + "largest": 2, + "completed": 22543494 + }, + "refresh": { + "threads": 2, + "queue": 0, + "active": 0, + "rejected": 0, + "largest": 2, + "completed": 885068032 + }, + "repository_azure": { + "threads": 0, + "queue": 0, + "active": 0, + "rejected": 0, + "largest": 0, + "completed": 0 + }, + "rollup_indexing": { + "threads": 0, + "queue": 0, + "active": 0, + "rejected": 0, + "largest": 0, + "completed": 0 + }, + "search": { + "threads": 5, + "queue": 0, + "active": 0, + "rejected": 0, + "largest": 5, + "completed": 167558078 + }, + "search_coordination": { + "threads": 2, + "queue": 0, + "active": 0, + "rejected": 0, + "largest": 2, + "completed": 14101082 + }, + "search_throttled": { + "threads": 0, + "queue": 0, + "active": 0, + "rejected": 0, + "largest": 0, + "completed": 0 + }, + "searchable_snapshots_cache_fetch_async": { + "threads": 0, + "queue": 0, + "active": 0, + "rejected": 0, + "largest": 0, + "completed": 0 + }, + "searchable_snapshots_cache_prewarming": { + "threads": 0, + "queue": 0, + "active": 0, + "rejected": 0, + "largest": 0, + "completed": 0 + }, + "security-crypto": { + "threads": 0, + "queue": 0, + "active": 0, + "rejected": 0, + "largest": 0, + "completed": 0 + }, + "security-token-key": { + "threads": 0, + "queue": 0, + "active": 0, + "rejected": 0, + "largest": 0, + "completed": 0 + }, + "snapshot": { + "threads": 1, + "queue": 0, + "active": 0, + "rejected": 0, + "largest": 2, + "completed": 806551 + }, + "snapshot_meta": { + "threads": 0, + "queue": 0, + "active": 0, + "rejected": 0, + "largest": 0, + "completed": 0 + }, + "system_critical_read": { + "threads": 2, + "queue": 0, + "active": 0, + "rejected": 0, + "largest": 2, + "completed": 2350761 + }, + "system_critical_write": { + "threads": 2, + "queue": 0, + "active": 0, + "rejected": 0, + "largest": 2, + "completed": 7635 + }, + "system_read": { + "threads": 2, + "queue": 0, + "active": 0, + "rejected": 0, + "largest": 2, + "completed": 31141408 + }, + "system_write": { + "threads": 2, + "queue": 0, + "active": 0, + "rejected": 0, + "largest": 2, + "completed": 7400801 + }, + "vector_tile_generation": { + "threads": 0, + "queue": 0, + "active": 0, + "rejected": 0, + "largest": 0, + "completed": 0 + }, + "warmer": { + "threads": 2, + "queue": 0, + "active": 0, + "rejected": 0, + "largest": 2, + "completed": 36136481 + }, + "watcher": { + "threads": 0, + "queue": 0, + "active": 0, + "rejected": 0, + "largest": 0, + "completed": 0 + }, + "write": { + "threads": 3, + "queue": 0, + "active": 0, + "rejected": 0, + "largest": 3, + "completed": 575332197 + } + }, + "fs": { + "timestamp": 1687866153494, + "total": { + "total_in_bytes": 483183820800, + "free_in_bytes": 292886683648, + "available_in_bytes": 292886683648 + }, + "data": [ + { + "path": "/app/data", + "mount": "/app (/dev/mapper/lxc-data)", + "type": "xfs", + "total_in_bytes": 483183820800, + "free_in_bytes": 292886683648, + "available_in_bytes": 292886683648 + } + ], + "io_stats": { + "devices": [ + { + "device_name": "dm-1", + "operations": 6160354146, + "read_operations": 376563348, + "write_operations": 5783790798, + "read_kilobytes": 31264865276, + "write_kilobytes": 100978561519, + "io_time_in_millis": 183984060 + } + ], + "total": { + "operations": 6160354146, + "read_operations": 376563348, + "write_operations": 5783790798, + "read_kilobytes": 31264865276, + "write_kilobytes": 100978561519, + "io_time_in_millis": 183984060 + } + } + }, + "transport": { + "server_open": 24, + "total_outbound_connections": 11, + "rx_count": 1300324276, + "rx_size_in_bytes": 1789333458217, + "tx_count": 1300324275, + "tx_size_in_bytes": 2927487680282, + "inbound_handling_time_histogram": [ + { + "lt_millis": 1, + "count": 1256115237 + }, + { + "ge_millis": 1, + "lt_millis": 2, + "count": 202073370 + }, + { + "ge_millis": 2, + "lt_millis": 4, + "count": 3242412 + }, + { + "ge_millis": 4, + "lt_millis": 8, + "count": 454921 + }, + { + "ge_millis": 8, + "lt_millis": 16, + "count": 173321 + }, + { + "ge_millis": 16, + "lt_millis": 32, + "count": 39045 + }, + { + "ge_millis": 32, + "lt_millis": 64, + "count": 14154 + }, + { + "ge_millis": 64, + "lt_millis": 128, + "count": 75261 + }, + { + "ge_millis": 128, + "lt_millis": 256, + "count": 1534 + }, + { + "ge_millis": 256, + "lt_millis": 512, + "count": 76 + }, + { + "ge_millis": 512, + "lt_millis": 1024, + "count": 3 + }, + { + "ge_millis": 1024, + "lt_millis": 2048, + "count": 0 + }, + { + "ge_millis": 2048, + "lt_millis": 4096, + "count": 0 + }, + { + "ge_millis": 4096, + "lt_millis": 8192, + "count": 0 + }, + { + "ge_millis": 8192, + "lt_millis": 16384, + "count": 0 + }, + { + "ge_millis": 16384, + "lt_millis": 32768, + "count": 0 + }, + { + "ge_millis": 32768, + "lt_millis": 65536, + "count": 0 + }, + { + "ge_millis": 65536, + "count": 0 + } + ], + "outbound_handling_time_histogram": [ + { + "lt_millis": 1, + "count": 1128384926 + }, + { + "ge_millis": 1, + "lt_millis": 2, + "count": 161841158 + }, + { + "ge_millis": 2, + "lt_millis": 4, + "count": 6818465 + }, + { + "ge_millis": 4, + "lt_millis": 8, + "count": 2563517 + }, + { + "ge_millis": 8, + "lt_millis": 16, + "count": 445765 + }, + { + "ge_millis": 16, + "lt_millis": 32, + "count": 122453 + }, + { + "ge_millis": 32, + "lt_millis": 64, + "count": 95805 + }, + { + "ge_millis": 64, + "lt_millis": 128, + "count": 49979 + }, + { + "ge_millis": 128, + "lt_millis": 256, + "count": 1930 + }, + { + "ge_millis": 256, + "lt_millis": 512, + "count": 250 + }, + { + "ge_millis": 512, + "lt_millis": 1024, + "count": 27 + }, + { + "ge_millis": 1024, + "lt_millis": 2048, + "count": 0 + }, + { + "ge_millis": 2048, + "lt_millis": 4096, + "count": 0 + }, + { + "ge_millis": 4096, + "lt_millis": 8192, + "count": 0 + }, + { + "ge_millis": 8192, + "lt_millis": 16384, + "count": 0 + }, + { + "ge_millis": 16384, + "lt_millis": 32768, + "count": 0 + }, + { + "ge_millis": 32768, + "lt_millis": 65536, + "count": 0 + }, + { + "ge_millis": 65536, + "count": 0 + } + ] + }, + "http": { + "current_open": 75, + "total_opened": 779352 + }, + "breakers": { + "fielddata": { + "limit_size_in_bytes": 3145728000, + "limit_size": "2.9gb", + "estimated_size_in_bytes": 600, + "estimated_size": "600b", + "overhead": 1.03, + "tripped": 0 + }, + "request": { + "limit_size_in_bytes": 4718592000, + "limit_size": "4.3gb", + "estimated_size_in_bytes": 0, + "estimated_size": "0b", + "overhead": 1, + "tripped": 0 + }, + "inflight_requests": { + "limit_size_in_bytes": 7864320000, + "limit_size": "7.3gb", + "estimated_size_in_bytes": 1464, + "estimated_size": "1.4kb", + "overhead": 2, + "tripped": 0 + }, + "model_inference": { + "limit_size_in_bytes": 3932160000, + "limit_size": "3.6gb", + "estimated_size_in_bytes": 0, + "estimated_size": "0b", + "overhead": 1, + "tripped": 0 + }, + "eql_sequence": { + "limit_size_in_bytes": 3932160000, + "limit_size": "3.6gb", + "estimated_size_in_bytes": 0, + "estimated_size": "0b", + "overhead": 1, + "tripped": 0 + }, + "parent": { + "limit_size_in_bytes": 7471104000, + "limit_size": "6.9gb", + "estimated_size_in_bytes": 5059735552, + "estimated_size": "4.7gb", + "overhead": 1, + "tripped": 0 + } + } + }, + "k_AifYMWQTykjUq3pgE_-w": { + "timestamp": 1687866153482, + "name": "tiebreaker-0000000002", + "transport_address": "172.25.242.111:19393", + "host": "172.25.242.111", + "ip": "172.25.242.111:19393", + "roles": [ + "master", + "voting_only" + ], + "attributes": { + "logical_availability_zone": "tiebreaker", + "availability_zone": "us-east-1b", + "server_name": "tiebreaker-0000000002.36928dce44074ceba64d7b3d698443a7", + "xpack.installed": "true", + "data": "hot", + "instance_configuration": "aws.es.master.c5d", + "region": "us-east-1" + }, + "indices": { + "docs": { + "count": 0, + "deleted": 0 + }, + "shard_stats": { + "total_count": 0 + }, + "store": { + "size_in_bytes": 0, + "total_data_set_size_in_bytes": 0, + "reserved_in_bytes": 0 + }, + "indexing": { + "index_total": 0, + "index_time_in_millis": 0, + "index_current": 0, + "index_failed": 0, + "delete_total": 0, + "delete_time_in_millis": 0, + "delete_current": 0, + "noop_update_total": 0, + "is_throttled": false, + "throttle_time_in_millis": 0 + }, + "get": { + "total": 0, + "time_in_millis": 0, + "exists_total": 0, + "exists_time_in_millis": 0, + "missing_total": 0, + "missing_time_in_millis": 0, + "current": 0 + }, + "search": { + "open_contexts": 0, + "query_total": 0, + "query_time_in_millis": 0, + "query_current": 0, + "fetch_total": 0, + "fetch_time_in_millis": 0, + "fetch_current": 0, + "scroll_total": 0, + "scroll_time_in_millis": 0, + "scroll_current": 0, + "suggest_total": 0, + "suggest_time_in_millis": 0, + "suggest_current": 0 + }, + "merges": { + "current": 0, + "current_docs": 0, + "current_size_in_bytes": 0, + "total": 0, + "total_time_in_millis": 0, + "total_docs": 0, + "total_size_in_bytes": 0, + "total_stopped_time_in_millis": 0, + "total_throttled_time_in_millis": 0, + "total_auto_throttle_in_bytes": 0 + }, + "refresh": { + "total": 0, + "total_time_in_millis": 0, + "external_total": 0, + "external_total_time_in_millis": 0, + "listeners": 0 + }, + "flush": { + "total": 0, + "periodic": 0, + "total_time_in_millis": 0 + }, + "warmer": { + "current": 0, + "total": 0, + "total_time_in_millis": 0 + }, + "query_cache": { + "memory_size_in_bytes": 0, + "total_count": 0, + "hit_count": 0, + "miss_count": 0, + "cache_size": 0, + "cache_count": 0, + "evictions": 0 + }, + "fielddata": { + "memory_size_in_bytes": 0, + "evictions": 0 + }, + "completion": { + "size_in_bytes": 0 + }, + "segments": { + "count": 0, + "memory_in_bytes": 0, + "terms_memory_in_bytes": 0, + "stored_fields_memory_in_bytes": 0, + "term_vectors_memory_in_bytes": 0, + "norms_memory_in_bytes": 0, + "points_memory_in_bytes": 0, + "doc_values_memory_in_bytes": 0, + "index_writer_memory_in_bytes": 0, + "version_map_memory_in_bytes": 0, + "fixed_bit_set_memory_in_bytes": 0, + "max_unsafe_auto_id_timestamp": -9223372036854776000, + "file_sizes": {} + }, + "translog": { + "operations": 0, + "size_in_bytes": 0, + "uncommitted_operations": 0, + "uncommitted_size_in_bytes": 0, + "earliest_last_modified_age": 0 + }, + "request_cache": { + "memory_size_in_bytes": 0, + "evictions": 0, + "hit_count": 0, + "miss_count": 0 + }, + "recovery": { + "current_as_source": 0, + "current_as_target": 0, + "throttle_time_in_millis": 0 + }, + "bulk": { + "total_operations": 0, + "total_time_in_millis": 0, + "total_size_in_bytes": 0, + "avg_time_in_millis": 0, + "avg_size_in_bytes": 0 + } + }, + "os": { + "timestamp": 1687866153483, + "cpu": { + "percent": 0, + "load_average": { + "1m": 3.18, + "5m": 2.94, + "15m": 2.54 + } + }, + "mem": { + "total_in_bytes": 1073741824, + "adjusted_total_in_bytes": 696254464, + "free_in_bytes": 101437440, + "used_in_bytes": 972304384, + "free_percent": 9, + "used_percent": 91 + }, + "swap": { + "total_in_bytes": 536870912, + "free_in_bytes": 536870912, + "used_in_bytes": 0 + }, + "cgroup": { + "cpuacct": { + "control_group": "/", + "usage_nanos": 281986757031142 + }, + "cpu": { + "control_group": "/", + "cfs_period_micros": 100000, + "cfs_quota_micros": 847058, + "stat": { + "number_of_elapsed_periods": 133754533, + "number_of_times_throttled": 226, + "time_throttled_nanos": 6732992268 + } + }, + "memory": { + "control_group": "/", + "limit_in_bytes": "1073741824", + "usage_in_bytes": "972304384" + } + } + }, + "process": { + "timestamp": 1687866153483, + "open_file_descriptors": 557, + "max_file_descriptors": 1048576, + "cpu": { + "percent": 0, + "total_in_millis": 182462990 + }, + "mem": { + "total_virtual_in_bytes": 6049042432 + } + }, + "jvm": { + "timestamp": 1687866153484, + "uptime_in_millis": 23671101768, + "mem": { + "heap_used_in_bytes": 178362704, + "heap_used_percent": 63, + "heap_committed_in_bytes": 281018368, + "heap_max_in_bytes": 281018368, + "non_heap_used_in_bytes": 221757752, + "non_heap_committed_in_bytes": 231145472, + "pools": { + "young": { + "used_in_bytes": 71303168, + "max_in_bytes": 0, + "peak_used_in_bytes": 163577856, + "peak_max_in_bytes": 0 + }, + "old": { + "used_in_bytes": 106872320, + "max_in_bytes": 281018368, + "peak_used_in_bytes": 246953424, + "peak_max_in_bytes": 281018368 + }, + "survivor": { + "used_in_bytes": 187216, + "max_in_bytes": 0, + "peak_used_in_bytes": 20971520, + "peak_max_in_bytes": 0 + } + } + }, + "threads": { + "count": 45, + "peak_count": 47 + }, + "gc": { + "collectors": { + "young": { + "collection_count": 342994, + "collection_time_in_millis": 768917 + }, + "old": { + "collection_count": 0, + "collection_time_in_millis": 0 + } + } + }, + "buffer_pools": { + "mapped": { + "count": 0, + "used_in_bytes": 0, + "total_capacity_in_bytes": 0 + }, + "direct": { + "count": 19, + "used_in_bytes": 2142216, + "total_capacity_in_bytes": 2142214 + }, + "mapped - 'non-volatile memory'": { + "count": 0, + "used_in_bytes": 0, + "total_capacity_in_bytes": 0 + } + }, + "classes": { + "current_loaded_count": 29581, + "total_loaded_count": 31244, + "total_unloaded_count": 1663 + } + }, + "thread_pool": { + "analyze": { + "threads": 0, + "queue": 0, + "active": 0, + "rejected": 0, + "largest": 0, + "completed": 0 + }, + "auto_complete": { + "threads": 0, + "queue": 0, + "active": 0, + "rejected": 0, + "largest": 0, + "completed": 0 + }, + "azure_event_loop": { + "threads": 0, + "queue": 0, + "active": 0, + "rejected": 0, + "largest": 0, + "completed": 0 + }, + "ccr": { + "threads": 0, + "queue": 0, + "active": 0, + "rejected": 0, + "largest": 0, + "completed": 0 + }, + "cluster_coordination": { + "threads": 1, + "queue": 0, + "active": 0, + "rejected": 0, + "largest": 1, + "completed": 1708790 + }, + "fetch_shard_started": { + "threads": 0, + "queue": 0, + "active": 0, + "rejected": 0, + "largest": 0, + "completed": 0 + }, + "fetch_shard_store": { + "threads": 0, + "queue": 0, + "active": 0, + "rejected": 0, + "largest": 0, + "completed": 0 + }, + "flush": { + "threads": 0, + "queue": 0, + "active": 0, + "rejected": 0, + "largest": 0, + "completed": 0 + }, + "force_merge": { + "threads": 0, + "queue": 0, + "active": 0, + "rejected": 0, + "largest": 0, + "completed": 0 + }, + "generic": { + "threads": 9, + "queue": 0, + "active": 0, + "rejected": 0, + "largest": 9, + "completed": 78631938 + }, + "get": { + "threads": 0, + "queue": 0, + "active": 0, + "rejected": 0, + "largest": 0, + "completed": 0 + }, + "management": { + "threads": 2, + "queue": 0, + "active": 1, + "rejected": 0, + "largest": 2, + "completed": 86206936 + }, + "ml_datafeed": { + "threads": 0, + "queue": 0, + "active": 0, + "rejected": 0, + "largest": 0, + "completed": 0 + }, + "ml_job_comms": { + "threads": 0, + "queue": 0, + "active": 0, + "rejected": 0, + "largest": 0, + "completed": 0 + }, + "ml_native_inference_comms": { + "threads": 0, + "queue": 0, + "active": 0, + "rejected": 0, + "largest": 0, + "completed": 0 + }, + "ml_utility": { + "threads": 2, + "queue": 0, + "active": 0, + "rejected": 0, + "largest": 2, + "completed": 47308828 + }, + "refresh": { + "threads": 0, + "queue": 0, + "active": 0, + "rejected": 0, + "largest": 0, + "completed": 0 + }, + "repository_azure": { + "threads": 0, + "queue": 0, + "active": 0, + "rejected": 0, + "largest": 0, + "completed": 0 + }, + "rollup_indexing": { + "threads": 0, + "queue": 0, + "active": 0, + "rejected": 0, + "largest": 0, + "completed": 0 + }, + "search": { + "threads": 0, + "queue": 0, + "active": 0, + "rejected": 0, + "largest": 0, + "completed": 0 + }, + "search_coordination": { + "threads": 0, + "queue": 0, + "active": 0, + "rejected": 0, + "largest": 0, + "completed": 0 + }, + "search_throttled": { + "threads": 0, + "queue": 0, + "active": 0, + "rejected": 0, + "largest": 0, + "completed": 0 + }, + "searchable_snapshots_cache_fetch_async": { + "threads": 0, + "queue": 0, + "active": 0, + "rejected": 0, + "largest": 0, + "completed": 0 + }, + "searchable_snapshots_cache_prewarming": { + "threads": 0, + "queue": 0, + "active": 0, + "rejected": 0, + "largest": 0, + "completed": 0 + }, + "security-crypto": { + "threads": 0, + "queue": 0, + "active": 0, + "rejected": 0, + "largest": 0, + "completed": 0 + }, + "security-token-key": { + "threads": 0, + "queue": 0, + "active": 0, + "rejected": 0, + "largest": 0, + "completed": 0 + }, + "snapshot": { + "threads": 0, + "queue": 0, + "active": 0, + "rejected": 0, + "largest": 0, + "completed": 0 + }, + "snapshot_meta": { + "threads": 0, + "queue": 0, + "active": 0, + "rejected": 0, + "largest": 0, + "completed": 0 + }, + "system_critical_read": { + "threads": 1, + "queue": 0, + "active": 0, + "rejected": 0, + "largest": 1, + "completed": 1 + }, + "system_critical_write": { + "threads": 0, + "queue": 0, + "active": 0, + "rejected": 0, + "largest": 0, + "completed": 0 + }, + "system_read": { + "threads": 0, + "queue": 0, + "active": 0, + "rejected": 0, + "largest": 0, + "completed": 0 + }, + "system_write": { + "threads": 0, + "queue": 0, + "active": 0, + "rejected": 0, + "largest": 0, + "completed": 0 + }, + "vector_tile_generation": { + "threads": 0, + "queue": 0, + "active": 0, + "rejected": 0, + "largest": 0, + "completed": 0 + }, + "warmer": { + "threads": 0, + "queue": 0, + "active": 0, + "rejected": 0, + "largest": 0, + "completed": 0 + }, + "watcher": { + "threads": 0, + "queue": 0, + "active": 0, + "rejected": 0, + "largest": 0, + "completed": 0 + }, + "write": { + "threads": 2, + "queue": 0, + "active": 0, + "rejected": 0, + "largest": 2, + "completed": 2 + } + }, + "fs": { + "timestamp": 1687866153484, + "total": { + "total_in_bytes": 12884901888, + "free_in_bytes": 12789022720, + "available_in_bytes": 12789022720 + }, + "data": [ + { + "path": "/app/data", + "mount": "/app (/dev/mapper/lxc-data)", + "type": "xfs", + "total_in_bytes": 12884901888, + "free_in_bytes": 12789022720, + "available_in_bytes": 12789022720 + } + ], + "io_stats": { + "devices": [ + { + "device_name": "dm-1", + "operations": 1025442756, + "read_operations": 12887271, + "write_operations": 1012555485, + "read_kilobytes": 666215440, + "write_kilobytes": 20200424566, + "io_time_in_millis": 547217376 + } + ], + "total": { + "operations": 1025442756, + "read_operations": 12887271, + "write_operations": 1012555485, + "read_kilobytes": 666215440, + "write_kilobytes": 20200424566, + "io_time_in_millis": 547217376 + } + } + }, + "transport": { + "server_open": 26, + "total_outbound_connections": 20, + "rx_count": 107632996, + "rx_size_in_bytes": 180620082152, + "tx_count": 107633007, + "tx_size_in_bytes": 420999501235, + "inbound_handling_time_histogram": [ + { + "lt_millis": 1, + "count": 146874447 + }, + { + "ge_millis": 1, + "lt_millis": 2, + "count": 16292686 + }, + { + "ge_millis": 2, + "lt_millis": 4, + "count": 50826 + }, + { + "ge_millis": 4, + "lt_millis": 8, + "count": 1965 + }, + { + "ge_millis": 8, + "lt_millis": 16, + "count": 187 + }, + { + "ge_millis": 16, + "lt_millis": 32, + "count": 84 + }, + { + "ge_millis": 32, + "lt_millis": 64, + "count": 2 + }, + { + "ge_millis": 64, + "lt_millis": 128, + "count": 65800 + }, + { + "ge_millis": 128, + "lt_millis": 256, + "count": 14 + }, + { + "ge_millis": 256, + "lt_millis": 512, + "count": 0 + }, + { + "ge_millis": 512, + "lt_millis": 1024, + "count": 0 + }, + { + "ge_millis": 1024, + "lt_millis": 2048, + "count": 0 + }, + { + "ge_millis": 2048, + "lt_millis": 4096, + "count": 0 + }, + { + "ge_millis": 4096, + "lt_millis": 8192, + "count": 0 + }, + { + "ge_millis": 8192, + "lt_millis": 16384, + "count": 0 + }, + { + "ge_millis": 16384, + "lt_millis": 32768, + "count": 0 + }, + { + "ge_millis": 32768, + "lt_millis": 65536, + "count": 0 + }, + { + "ge_millis": 65536, + "count": 0 + } + ], + "outbound_handling_time_histogram": [ + { + "lt_millis": 1, + "count": 97208157 + }, + { + "ge_millis": 1, + "lt_millis": 2, + "count": 10385725 + }, + { + "ge_millis": 2, + "lt_millis": 4, + "count": 28647 + }, + { + "ge_millis": 4, + "lt_millis": 8, + "count": 6334 + }, + { + "ge_millis": 8, + "lt_millis": 16, + "count": 1042 + }, + { + "ge_millis": 16, + "lt_millis": 32, + "count": 818 + }, + { + "ge_millis": 32, + "lt_millis": 64, + "count": 1556 + }, + { + "ge_millis": 64, + "lt_millis": 128, + "count": 725 + }, + { + "ge_millis": 128, + "lt_millis": 256, + "count": 3 + }, + { + "ge_millis": 256, + "lt_millis": 512, + "count": 0 + }, + { + "ge_millis": 512, + "lt_millis": 1024, + "count": 0 + }, + { + "ge_millis": 1024, + "lt_millis": 2048, + "count": 0 + }, + { + "ge_millis": 2048, + "lt_millis": 4096, + "count": 0 + }, + { + "ge_millis": 4096, + "lt_millis": 8192, + "count": 0 + }, + { + "ge_millis": 8192, + "lt_millis": 16384, + "count": 0 + }, + { + "ge_millis": 16384, + "lt_millis": 32768, + "count": 0 + }, + { + "ge_millis": 32768, + "lt_millis": 65536, + "count": 0 + }, + { + "ge_millis": 65536, + "count": 0 + } + ] + }, + "http": { + "current_open": 14, + "total_opened": 13364 + }, + "breakers": { + "model_inference": { + "limit_size_in_bytes": 140509184, + "limit_size": "134mb", + "estimated_size_in_bytes": 0, + "estimated_size": "0b", + "overhead": 1, + "tripped": 0 + }, + "eql_sequence": { + "limit_size_in_bytes": 140509184, + "limit_size": "134mb", + "estimated_size_in_bytes": 0, + "estimated_size": "0b", + "overhead": 1, + "tripped": 0 + }, + "fielddata": { + "limit_size_in_bytes": 112407347, + "limit_size": "107.1mb", + "estimated_size_in_bytes": 0, + "estimated_size": "0b", + "overhead": 1.03, + "tripped": 0 + }, + "request": { + "limit_size_in_bytes": 168611020, + "limit_size": "160.7mb", + "estimated_size_in_bytes": 0, + "estimated_size": "0b", + "overhead": 1, + "tripped": 0 + }, + "inflight_requests": { + "limit_size_in_bytes": 281018368, + "limit_size": "268mb", + "estimated_size_in_bytes": 1464, + "estimated_size": "1.4kb", + "overhead": 2, + "tripped": 0 + }, + "parent": { + "limit_size_in_bytes": 266967449, + "limit_size": "254.5mb", + "estimated_size_in_bytes": 178362704, + "estimated_size": "170mb", + "overhead": 1, + "tripped": 0 + } + } + } + } +} |