1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
|
plugin_name: go.d.plugin
modules:
- meta:
id: collector-go.d.plugin-clickhouse
plugin_name: go.d.plugin
module_name: clickhouse
monitored_instance:
name: ClickHouse
link: https://clickhouse.com/
icon_filename: clickhouse.svg
categories:
- data-collection.database-servers
keywords:
- database
related_resources:
integrations:
list: []
info_provided_to_referring_integrations:
description: ""
most_popular: false
overview:
data_collection:
metrics_description: |
This collector retrieves performance data from ClickHouse for connections, queries, resources, replication, IO, and data operations (inserts, selects, merges) using HTTP requests and ClickHouse system tables. It monitors your ClickHouse server's health and activity.
method_description: |
It sends HTTP requests to the ClickHouse [HTTP interface](https://clickhouse.com/docs/en/interfaces/http), executing SELECT queries to retrieve data from various system tables.
Specifically, it collects metrics from the following tables:
- system.metrics
- system.async_metrics
- system.events
- system.disks
- system.parts
- system.processes
supported_platforms:
include: []
exclude: []
multi_instance: true
additional_permissions:
description: ""
default_behavior:
auto_detection:
description: |
By default, it detects ClickHouse instances running on localhost that are listening on port 8123.
On startup, it tries to collect metrics from:
- http://127.0.0.1:8123
limits:
description: ""
performance_impact:
description: ""
setup:
prerequisites:
list: []
configuration:
file:
name: go.d/clickhouse.conf
options:
description: |
The following options can be defined globally: update_every, autodetection_retry.
folding:
title: Config options
enabled: true
list:
- name: update_every
description: Data collection frequency.
default_value: 1
required: false
- name: autodetection_retry
description: Recheck interval in seconds. Zero means no recheck will be scheduled.
default_value: 0
required: false
- name: url
description: Server URL.
default_value: http://127.0.0.1:8123
required: true
- name: timeout
description: HTTP request timeout.
default_value: 1
required: false
- name: username
description: Username for basic HTTP authentication.
default_value: ""
required: false
- name: password
description: Password for basic HTTP authentication.
default_value: ""
required: false
- name: proxy_url
description: Proxy URL.
default_value: ""
required: false
- name: proxy_username
description: Username for proxy basic HTTP authentication.
default_value: ""
required: false
- name: proxy_password
description: Password for proxy basic HTTP authentication.
default_value: ""
required: false
- name: method
description: HTTP request method.
default_value: "GET"
required: false
- name: body
description: HTTP request body.
default_value: ""
required: false
- name: headers
description: HTTP request headers.
default_value: ""
required: false
- name: not_follow_redirects
description: Redirect handling policy. Controls whether the client follows redirects.
default_value: no
required: false
- name: tls_skip_verify
description: Server certificate chain and hostname validation policy. Controls whether the client performs this check.
default_value: no
required: false
- name: tls_ca
description: Certification authority that the client uses when verifying the server's certificates.
default_value: ""
required: false
- name: tls_cert
description: Client TLS certificate.
default_value: ""
required: false
- name: tls_key
description: Client TLS key.
default_value: ""
required: false
examples:
folding:
title: Config
enabled: true
list:
- name: Basic
folding:
enabled: false
description: A basic example configuration.
config: |
jobs:
- name: local
url: http://127.0.0.1:8123
- name: HTTP authentication
description: Basic HTTP authentication.
config: |
jobs:
- name: local
url: http://127.0.0.1:8123
username: username
password: password
- name: HTTPS with self-signed certificate
description: ClickHouse with enabled HTTPS and self-signed certificate.
config: |
jobs:
- name: local
url: https://127.0.0.1:8123
tls_skip_verify: yes
- name: Multi-instance
description: |
> **Note**: When you define multiple jobs, their names must be unique.
Collecting metrics from local and remote instances.
config: |
jobs:
- name: local
url: http://127.0.0.1:8123
- name: remote
url: http://192.0.2.1:8123
troubleshooting:
problems:
list: []
alerts:
- name: clickhouse_restarted
metric: clickhouse.uptime
info: ClickHouse has recently been restarted
link: https://github.com/netdata/netdata/blob/master/src/health/health.d/clickhouse.conf
- name: clickhouse_queries_preempted
metric: clickhouse.queries_preempted
info: ClickHouse has queries that are stopped and waiting due to priority setting
link: https://github.com/netdata/netdata/blob/master/src/health/health.d/clickhouse.conf
- name: clickhouse_long_running_query
metric: clickhouse.longest_running_query_time
info: ClickHouse has a long-running query exceeding the threshold
link: https://github.com/netdata/netdata/blob/master/src/health/health.d/clickhouse.conf
- name: clickhouse_rejected_inserts
metric: clickhouse.rejected_inserts
info: ClickHouse has INSERT queries that are rejected due to high number of active data parts for partition in a MergeTree
link: https://github.com/netdata/netdata/blob/master/src/health/health.d/clickhouse.conf
- name: clickhouse_delayed_inserts
metric: clickhouse.delayed_inserts
info: ClickHouse has INSERT queries that are throttled due to high number of active data parts for partition in a MergeTree
link: https://github.com/netdata/netdata/blob/master/src/health/health.d/clickhouse.conf
- name: clickhouse_replication_lag
metric: clickhouse.replicas_max_absolute_delay
info: ClickHouse is experiencing replication lag greater than 5 minutes
link: https://github.com/netdata/netdata/blob/master/src/health/health.d/clickhouse.conf
- name: clickhouse_replicated_readonly_tables
metric: clickhouse.replicated_readonly_tables
info: ClickHouse has replicated tables in readonly state due to ZooKeeper session loss/startup without ZooKeeper configured
link: https://github.com/netdata/netdata/blob/master/src/health/health.d/clickhouse.conf
- name: clickhouse_max_part_count_for_partition
metric: clickhouse.max_part_count_for_partition
info: ClickHouse high number of parts per partition
link: https://github.com/netdata/netdata/blob/master/src/health/health.d/clickhouse.conf
- name: clickhouse_distributed_connections_failures
metric: clickhouse.distributed_connections_fail_exhausted_retries
info: ClickHouse has failed distributed connections after exhausting all retry attempts
link: https://github.com/netdata/netdata/blob/master/src/health/health.d/clickhouse.conf
- name: clickhouse_distributed_files_to_insert
metric: clickhouse.distributed_files_to_insert
info: ClickHouse high number of pending files to process for asynchronous insertion into Distributed tables
link: https://github.com/netdata/netdata/blob/master/src/health/health.d/clickhouse.conf
metrics:
folding:
title: Metrics
enabled: false
description: ""
availability: []
scopes:
- name: global
description: These metrics refer to the entire monitored application.
labels: []
metrics:
- name: clickhouse.connections
description: Connections
unit: connections
chart_type: line
dimensions:
- name: tcp
- name: http
- name: mysql
- name: postgresql
- name: interserver
- name: clickhouse.slow_reads
description: Slow reads from a file
unit: reads/s
chart_type: line
dimensions:
- name: slow
- name: clickhouse.read_backoff
description: Read backoff events
unit: events/s
chart_type: line
dimensions:
- name: read_backoff
- name: clickhouse.memory_usage
description: Memory usage
unit: bytes
chart_type: area
dimensions:
- name: used
- name: clickhouse.running_queries
description: Running queries
unit: queries
chart_type: line
dimensions:
- name: running
- name: clickhouse.queries_preempted
description: Queries waiting due to priority
unit: queries
chart_type: line
dimensions:
- name: preempted
- name: clickhouse.queries
description: Queries
unit: queries/s
chart_type: stacked
dimensions:
- name: successful
- name: failed
- name: clickhouse.select_queries
description: Select queries
unit: selects/s
chart_type: stacked
dimensions:
- name: successful
- name: failed
- name: clickhouse.insert_queries
description: Insert queries
unit: inserts/s
chart_type: stacked
dimensions:
- name: successful
- name: failed
- name: clickhouse.queries_memory_limit_exceeded
description: Memory limit exceeded for query
unit: queries/s
chart_type: line
dimensions:
- name: mem_limit_exceeded
- name: clickhouse.longest_running_query_time
description: Longest running query time
unit: seconds
chart_type: line
dimensions:
- name: longest_query_time
- name: clickhouse.queries_latency
description: Queries latency
unit: microseconds
chart_type: line
dimensions:
- name: queries_time
- name: clickhouse.select_queries_latency
description: Select queries latency
unit: microseconds
chart_type: line
dimensions:
- name: selects_time
- name: clickhouse.insert_queries_latency
description: Insert queries latency
unit: microseconds
chart_type: line
dimensions:
- name: inserts_time
- name: clickhouse.io
description: Read and written data
unit: bytes/s
chart_type: area
dimensions:
- name: reads
- name: writes
- name: clickhouse.iops
description: Read and write operations
unit: ops/s
chart_type: line
dimensions:
- name: reads
- name: writes
- name: clickhouse.io_errors
description: Read and write errors
unit: errors/s
chart_type: line
dimensions:
- name: read
- name: write
- name: clickhouse.io_seeks
description: lseek function calls
unit: ops/s
chart_type: line
dimensions:
- name: lseek
- name: clickhouse.io_file_opens
description: File opens
unit: ops/s
chart_type: line
dimensions:
- name: file_open
- name: clickhouse.replicated_parts_current_activity
description: Replicated parts current activity
unit: parts
chart_type: line
dimensions:
- name: fetch
- name: send
- name: check
- name: clickhouse.replicas_max_absolute_dela
description: Replicas max absolute delay
unit: seconds
chart_type: line
dimensions:
- name: replication_delay
- name: clickhouse.replicated_readonly_tables
description: Replicated tables in readonly state
unit: tables
chart_type: line
dimensions:
- name: read_only
- name: clickhouse.replicated_data_loss
description: Replicated data loss
unit: events
chart_type: line
dimensions:
- name: data_loss
- name: clickhouse.replicated_part_fetches
description: Replicated part fetches
unit: fetches/s
chart_type: line
dimensions:
- name: successful
- name: failed
- name: clickhouse.inserted_rows
description: Inserted rows
unit: rows/s
chart_type: line
dimensions:
- name: inserted
- name: clickhouse.inserted_bytes
description: Inserted data
unit: bytes/s
chart_type: area
dimensions:
- name: inserted
- name: clickhouse.rejected_inserts
description: Rejected inserts
unit: inserts/s
chart_type: line
dimensions:
- name: rejected
- name: clickhouse.delayed_inserts
description: Delayed inserts
unit: inserts/s
chart_type: line
dimensions:
- name: delayed
- name: clickhouse.delayed_inserts_throttle_time
description: Delayed inserts throttle time
unit: milliseconds
chart_type: line
dimensions:
- name: delayed_inserts_throttle_time
- name: clickhouse.selected_bytes
description: Selected data
unit: bytes/s
chart_type: area
dimensions:
- name: selected
- name: clickhouse.selected_rows
description: Selected rows
unit: rows/s
chart_type: line
dimensions:
- name: selected
- name: clickhouse.selected_parts
description: Selected parts
unit: parts/s
chart_type: line
dimensions:
- name: selected
- name: clickhouse.selected_ranges
description: Selected ranges
unit: ranges/s
chart_type: line
dimensions:
- name: selected
- name: clickhouse.selected_marks
description: Selected marks
unit: marks/s
chart_type: line
dimensions:
- name: selected
- name: clickhouse.merges
description: Merge operations
unit: ops/s
chart_type: line
dimensions:
- name: merge
- name: clickhouse.merges_latency
description: Time spent for background merges
unit: milliseconds
chart_type: line
dimensions:
- name: merges_time
- name: clickhouse.merged_uncompressed_bytes
description: Uncompressed data read for background merges
unit: bytes/s
chart_type: area
dimensions:
- name: merged_uncompressed
- name: clickhouse.merged_rows
description: Merged rows
unit: rows/s
chart_type: line
dimensions:
- name: merged
- name: clickhouse.merge_tree_data_writer_inserted_rows
description: Rows INSERTed to MergeTree tables
unit: rows/s
chart_type: line
dimensions:
- name: inserted
- name: clickhouse.merge_tree_data_writer_uncompressed_bytes
description: Data INSERTed to MergeTree tables
unit: bytes/s
chart_type: area
dimensions:
- name: inserted
- name: clickhouse.merge_tree_data_writer_compressed_bytes
description: Data written to disk for data INSERTed to MergeTree tables
unit: bytes/s
chart_type: area
dimensions:
- name: written
- name: clickhouse.uncompressed_cache_requests
description: Uncompressed cache requests
unit: requests/s
chart_type: line
dimensions:
- name: hits
- name: misses
- name: clickhouse.mark_cache_requests
description: Mark cache requests
unit: requests/s
chart_type: line
dimensions:
- name: hits
- name: misses
- name: clickhouse.max_part_count_for_partition
description: Max part count for partition
unit: parts
chart_type: line
dimensions:
- name: max_parts_partition
- name: clickhouse.parts_count
description: Parts
unit: parts
chart_type: line
dimensions:
- name: temporary
- name: pre_active
- name: active
- name: deleting
- name: delete_on_destroy
- name: outdated
- name: wide
- name: compact
- name: distributed_connections
description: Active distributed connection
unit: connections
chart_type: line
dimensions:
- name: active
- name: distributed_connections_attempts
description: Distributed connection attempts
unit: attempts/s
chart_type: line
dimensions:
- name: connection
- name: distributed_connections_fail_retries
description: Distributed connection fails with retry
unit: fails/s
chart_type: line
dimensions:
- name: connection_retry
- name: distributed_connections_fail_exhausted_retries
description: Distributed connection fails after all retries finished
unit: fails/s
chart_type: line
dimensions:
- name: connection_retry_exhausted
- name: distributed_files_to_insert
description: Pending files to process for asynchronous insertion into Distributed tables
unit: files
chart_type: line
dimensions:
- name: pending_insertions
- name: distributed_rejected_inserts
description: Rejected INSERTs to a Distributed table
unit: inserts/s
chart_type: line
dimensions:
- name: rejected
- name: distributed_delayed_inserts
description: Delayed INSERTs to a Distributed table
unit: inserts/s
chart_type: line
dimensions:
- name: delayed
- name: distributed_delayed_inserts_latency
description: Time spent while the INSERT of a block to a Distributed table was throttled
unit: milliseconds
chart_type: line
dimensions:
- name: delayed_time
- name: distributed_sync_insertion_timeout_exceeded
description: Distributed table sync insertions timeouts
unit: timeouts/s
chart_type: line
dimensions:
- name: sync_insertion
- name: distributed_async_insertions_failures
description: Distributed table async insertion failures
unit: failures/s
chart_type: line
dimensions:
- name: async_insertions
- name: clickhouse.uptime
description: Uptime
unit: seconds
chart_type: line
dimensions:
- name: uptime
- name: disk
description: These metrics refer to the Disk.
labels:
- name: disk_name
description: Name of the disk as defined in the [server configuration](https://clickhouse.com/docs/en/engines/table-engines/mergetree-family/mergetree#table_engine-mergetree-multiple-volumes_configure).
metrics:
- name: clickhouse.disk_space_usage
description: Disk space usage
unit: bytes
chart_type: stacked
dimensions:
- name: free
- name: used
- name: table
description: These metrics refer to the Database Table.
labels:
- name: database
description: Name of the database.
- name: table
description: Name of the table.
metrics:
- name: clickhouse.database_table_size
description: Table size
unit: bytes
chart_type: area
dimensions:
- name: size
- name: clickhouse.database_table_parts
description: Table parts
unit: parts
chart_type: line
dimensions:
- name: parts
- name: clickhouse.database_table_rows
description: Table rows
unit: rows
chart_type: line
dimensions:
- name: rows
|