summaryrefslogtreecommitdiffstats
path: root/python.d/ceph.chart.py
diff options
context:
space:
mode:
authorDaniel Baumann <daniel.baumann@progress-linux.org>2018-11-07 12:19:29 +0000
committerDaniel Baumann <daniel.baumann@progress-linux.org>2018-11-07 12:20:17 +0000
commita64a253794ac64cb40befee54db53bde17dd0d49 (patch)
treec1024acc5f6e508814b944d99f112259bb28b1be /python.d/ceph.chart.py
parentNew upstream version 1.10.0+dfsg (diff)
downloadnetdata-a64a253794ac64cb40befee54db53bde17dd0d49.tar.xz
netdata-a64a253794ac64cb40befee54db53bde17dd0d49.zip
New upstream version 1.11.0+dfsgupstream/1.11.0+dfsg
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'python.d/ceph.chart.py')
-rw-r--r--python.d/ceph.chart.py313
1 files changed, 0 insertions, 313 deletions
diff --git a/python.d/ceph.chart.py b/python.d/ceph.chart.py
deleted file mode 100644
index fb78397d0..000000000
--- a/python.d/ceph.chart.py
+++ /dev/null
@@ -1,313 +0,0 @@
-# -*- coding: utf-8 -*-
-# Description: ceph netdata python.d module
-# Author: Luis Eduardo (lets00)
-
-try:
- import rados
- CEPH = True
-except ImportError:
- CEPH = False
-
-import json
-from bases.FrameworkServices.SimpleService import SimpleService
-
-# default module values (can be overridden per job in `config`)
-update_every = 10
-priority = 60000
-retries = 60
-
-ORDER = ['general_usage', 'general_objects', 'general_bytes', 'general_operations',
- 'general_latency', 'pool_usage', 'pool_objects', 'pool_read_bytes',
- 'pool_write_bytes', 'pool_read_operations', 'pool_write_operations', 'osd_usage',
- 'osd_apply_latency', 'osd_commit_latency']
-
-CHARTS = {
- 'general_usage': {
- 'options': [None, 'Ceph General Space', 'KB', 'general', 'ceph.general_usage', 'stacked'],
- 'lines': [
- ['general_available', 'avail', 'absolute', 1, 1024],
- ['general_usage', 'used', 'absolute', 1, 1024]
- ]
- },
- 'general_objects': {
- 'options': [None, 'Ceph General Objects', 'objects', 'general', 'ceph.general_objects', 'area'],
- 'lines': [
- ['general_objects', 'cluster', 'absolute']
- ]
- },
- 'general_bytes': {
- 'options': [None, 'Ceph General Read/Write Data/s', 'KB', 'general', 'ceph.general_bytes',
- 'area'],
- 'lines': [
- ['general_read_bytes', 'read', 'absolute', 1, 1024],
- ['general_write_bytes', 'write', 'absolute', -1, 1024]
- ]
- },
- 'general_operations': {
- 'options': [None, 'Ceph General Read/Write Operations/s', 'operations', 'general', 'ceph.general_operations',
- 'area'],
- 'lines': [
- ['general_read_operations', 'read', 'absolute', 1],
- ['general_write_operations', 'write', 'absolute', -1]
- ]
- },
- 'general_latency': {
- 'options': [None, 'Ceph General Apply/Commit latency', 'milliseconds', 'general', 'ceph.general_latency',
- 'area'],
- 'lines': [
- ['general_apply_latency', 'apply', 'absolute'],
- ['general_commit_latency', 'commit', 'absolute']
- ]
- },
- 'pool_usage': {
- 'options': [None, 'Ceph Pools', 'KB', 'pool', 'ceph.pool_usage', 'line'],
- 'lines': []
- },
- 'pool_objects': {
- 'options': [None, 'Ceph Pools', 'objects', 'pool', 'ceph.pool_objects', 'line'],
- 'lines': []
- },
- 'pool_read_bytes': {
- 'options': [None, 'Ceph Read Pool Data/s', 'KB', 'pool', 'ceph.pool_read_bytes', 'area'],
- 'lines': []
- },
- 'pool_write_bytes': {
- 'options': [None, 'Ceph Write Pool Data/s', 'KB', 'pool', 'ceph.pool_write_bytes', 'area'],
- 'lines': []
- },
- 'pool_read_operations': {
- 'options': [None, 'Ceph Read Pool Operations/s', 'operations', 'pool', 'ceph.pool_read_operations', 'area'],
- 'lines': []
- },
- 'pool_write_operations': {
- 'options': [None, 'Ceph Write Pool Operations/s', 'operations', 'pool', 'ceph.pool_write_operations', 'area'],
- 'lines': []
- },
- 'osd_usage': {
- 'options': [None, 'Ceph OSDs', 'KB', 'osd', 'ceph.osd_usage', 'line'],
- 'lines': []
- },
- 'osd_apply_latency': {
- 'options': [None, 'Ceph OSDs apply latency', 'milliseconds', 'osd', 'ceph.apply_latency', 'line'],
- 'lines': []
- },
- 'osd_commit_latency': {
- 'options': [None, 'Ceph OSDs commit latency', 'milliseconds', 'osd', 'ceph.commit_latency', 'line'],
- 'lines': []
- }
-
-}
-
-
-class Service(SimpleService):
- def __init__(self, configuration=None, name=None):
- SimpleService.__init__(self, configuration=configuration, name=name)
- self.order = ORDER
- self.definitions = CHARTS
- self.config_file = self.configuration.get('config_file')
- self.keyring_file = self.configuration.get('keyring_file')
-
- def check(self):
- """
- Checks module
- :return:
- """
- if not CEPH:
- self.error('rados module is needed to use ceph.chart.py')
- return False
- if not (self.config_file and self.keyring_file):
- self.error('config_file and/or keyring_file is not defined')
- return False
- try:
- self.cluster = rados.Rados(conffile=self.config_file,
- conf=dict(keyring=self.keyring_file))
- self.cluster.connect()
- except rados.Error as error:
- self.error(error)
- return False
- self.create_definitions()
- return True
-
- def create_definitions(self):
- """
- Create dynamically charts options
- :return: None
- """
- # Pool lines
- for pool in sorted(self._get_df()['pools']):
- self.definitions['pool_usage']['lines'].append([pool['name'],
- pool['name'],
- 'absolute'])
- self.definitions['pool_objects']['lines'].append(["obj_{0}".format(pool['name']),
- pool['name'],
- 'absolute'])
- self.definitions['pool_read_bytes']['lines'].append(['read_{0}'.format(pool['name']),
- pool['name'],
- 'absolute', 1, 1024])
- self.definitions['pool_write_bytes']['lines'].append(['write_{0}'.format(pool['name']),
- pool['name'],
- 'absolute', 1, 1024])
- self.definitions['pool_read_operations']['lines'].append(['read_operations_{0}'.format(pool['name']),
- pool['name'],
- 'absolute'])
- self.definitions['pool_write_operations']['lines'].append(['write_operations_{0}'.format(pool['name']),
- pool['name'],
- 'absolute'])
-
- # OSD lines
- for osd in sorted(self._get_osd_df()['nodes']):
- self.definitions['osd_usage']['lines'].append([osd['name'],
- osd['name'],
- 'absolute'])
- self.definitions['osd_apply_latency']['lines'].append(['apply_latency_{0}'.format(osd['name']),
- osd['name'],
- 'absolute'])
- self.definitions['osd_commit_latency']['lines'].append(['commit_latency_{0}'.format(osd['name']),
- osd['name'],
- 'absolute'])
-
- def get_data(self):
- """
- Catch all ceph data
- :return: dict
- """
- try:
- data = {}
- df = self._get_df()
- osd_df = self._get_osd_df()
- osd_perf = self._get_osd_perf()
- pool_stats = self._get_osd_pool_stats()
- data.update(self._get_general(osd_perf, pool_stats))
- for pool in df['pools']:
- data.update(self._get_pool_usage(pool))
- data.update(self._get_pool_objects(pool))
- for pool_io in pool_stats:
- data.update(self._get_pool_rw(pool_io))
- for osd in osd_df['nodes']:
- data.update(self._get_osd_usage(osd))
- for osd_apply_commit in osd_perf['osd_perf_infos']:
- data.update(self._get_osd_latency(osd_apply_commit))
- return data
- except (ValueError, AttributeError) as error:
- self.error(error)
- return None
-
- def _get_general(self, osd_perf, pool_stats):
- """
- Get ceph's general usage
- :return: dict
- """
- status = self.cluster.get_cluster_stats()
- read_bytes_sec = 0
- write_bytes_sec = 0
- read_op_per_sec = 0
- write_op_per_sec = 0
- apply_latency = 0
- commit_latency = 0
-
- for pool_rw_io_b in pool_stats:
- read_bytes_sec += pool_rw_io_b['client_io_rate'].get('read_bytes_sec', 0)
- write_bytes_sec += pool_rw_io_b['client_io_rate'].get('write_bytes_sec', 0)
- read_op_per_sec += pool_rw_io_b['client_io_rate'].get('read_op_per_sec', 0)
- write_op_per_sec += pool_rw_io_b['client_io_rate'].get('write_op_per_sec', 0)
- for perf in osd_perf['osd_perf_infos']:
- apply_latency += perf['perf_stats']['apply_latency_ms']
- commit_latency += perf['perf_stats']['commit_latency_ms']
-
- return {'general_usage': int(status['kb_used']),
- 'general_available': int(status['kb_avail']),
- 'general_objects': int(status['num_objects']),
- 'general_read_bytes': read_bytes_sec,
- 'general_write_bytes': write_bytes_sec,
- 'general_read_operations': read_op_per_sec,
- 'general_write_operations': write_op_per_sec,
- 'general_apply_latency': apply_latency,
- 'general_commit_latency': commit_latency
- }
-
- @staticmethod
- def _get_pool_usage(pool):
- """
- Process raw data into pool usage dict information
- :return: A pool dict with pool name's key and usage bytes' value
- """
- return {pool['name']: pool['stats']['kb_used']}
-
- @staticmethod
- def _get_pool_objects(pool):
- """
- Process raw data into pool usage dict information
- :return: A pool dict with pool name's key and object numbers
- """
- return {'obj_{0}'.format(pool['name']): pool['stats']['objects']}
-
- @staticmethod
- def _get_pool_rw(pool):
- """
- Get read/write kb and operations in a pool
- :return: A pool dict with both read/write bytes and operations.
- """
- return {'read_{0}'.format(pool['pool_name']): int(pool['client_io_rate'].get('read_bytes_sec', 0)),
- 'write_{0}'.format(pool['pool_name']): int(pool['client_io_rate'].get('write_bytes_sec', 0)),
- 'read_operations_{0}'.format(pool['pool_name']): int(pool['client_io_rate'].get('read_op_per_sec', 0)),
- 'write_operations_{0}'.format(pool['pool_name']): int(pool['client_io_rate'].get('write_op_per_sec', 0))
- }
-
- @staticmethod
- def _get_osd_usage(osd):
- """
- Process raw data into osd dict information to get osd usage
- :return: A osd dict with osd name's key and usage bytes' value
- """
- return {osd['name']: float(osd['kb_used'])}
-
- @staticmethod
- def _get_osd_latency(osd):
- """
- Get ceph osd apply and commit latency
- :return: A osd dict with osd name's key with both apply and commit latency values
- """
- return {'apply_latency_osd.{0}'.format(osd['id']): osd['perf_stats']['apply_latency_ms'],
- 'commit_latency_osd.{0}'.format(osd['id']): osd['perf_stats']['commit_latency_ms']}
-
- def _get_df(self):
- """
- Get ceph df output
- :return: ceph df --format json
- """
- return json.loads(self.cluster.mon_command(json.dumps({
- 'prefix': 'df',
- 'format': 'json'
- }), '')[1])
-
- def _get_osd_df(self):
- """
- Get ceph osd df output
- :return: ceph osd df --format json
- """
- return json.loads(self.cluster.mon_command(json.dumps({
- 'prefix': 'osd df',
- 'format': 'json'
- }), '')[1])
-
- def _get_osd_perf(self):
- """
- Get ceph osd performance
- :return: ceph osd perf --format json
- """
- return json.loads(self.cluster.mon_command(json.dumps({
- 'prefix': 'osd perf',
- 'format': 'json'
- }), '')[1])
-
- def _get_osd_pool_stats(self):
- """
- Get ceph osd pool status.
- This command is used to get information about both
- read/write operation and bytes per second on each pool
- :return: ceph osd pool stats --format json
- """
- return json.loads(self.cluster.mon_command(json.dumps({
- 'prefix': 'osd pool stats',
- 'format': 'json'
- }), '')[1])