path: root/src/tools/crushdiff

#!/usr/bin/python3
#
# A tool to test the effect (number of pgs, objects, bytes moved) of a
# crushmap change. This is a wrapper around osdmaptool, hardly relying
# on its --test-map-pgs-dump option to get the list of changed pgs.
# Additionally it uses pg stats to calculate the numbers of objects
# and bytes moved.
#
# Typical usage:
#
# # Get current crushmap
# $ crushdiff export cm.txt
# # Edit the map
# $ $EDITOR cm.txt
# # Check the result
# $ crushdiff compare cm.txt
# # Install the updated map
# $ crushdiff import cm.txt
#
# By default, crushdiff will use the cluster current osdmap and pg
# stats, which requires access to the cluster. But one can use the
# --osdmap and --pg-dump options to test against previously obtained
# data.
#

import argparse
import re
import json
import os
import sys
import tempfile

#
# Global
#

parser = argparse.ArgumentParser(prog='crushdiff',
                                 description='Tool for updating crush map')
parser.add_argument(
    'command',
    metavar='compare|export|import',
    help='command',
    default=None,
)
parser.add_argument(
    '-c', '--compiled',
    action='store_true',
    help='use compiled crush map',
    default=False,
)
parser.add_argument(
    'crushmap',
    metavar='crushmap',
    help='crushmap json file',
    default=None,
)
parser.add_argument(
    '-m', '--osdmap',
    metavar='osdmap',
    help='',
    default=None,
)
parser.add_argument(
    '-p', '--pg-dump',
    metavar='pg-dump',
    help='`ceph pg dump` json output',
    default=None,
)
parser.add_argument(
    '-v', '--verbose',
    action='store_true',
    help='be verbose',
    default=False,
)

#
# Functions
#

def get_human_readable(bytes, precision=2):
    suffixes = ['', 'Ki', 'Mi', 'Gi', 'Ti']
    suffix_index = 0
    while bytes > 1024 and suffix_index < 4:
        # increment the index of the suffix
        suffix_index += 1
        # apply the division
        bytes = bytes / 1024.0
    return '%.*f%s' % (precision, bytes, suffixes[suffix_index])

def run_cmd(cmd, verbose=False):
    if verbose:
        print(cmd, file=sys.stderr, flush=True)
    os.system(cmd)

def get_osdmap(file):
    with open(file, "r") as f:
        return json.load(f)

def get_pools(osdmap):
    return {p['pool']: p for p in osdmap['pools']}

def get_erasure_code_profiles(osdmap):
    return osdmap['erasure_code_profiles']

def get_pgmap(pg_dump_file):
    with open(pg_dump_file, "r") as f:
        dump = json.load(f)
        return dump.get('pg_map', dump)

def get_pg_stats(pgmap):
    return {pg['pgid']: pg for pg in pgmap['pg_stats']}

def parse_test_map_pgs_dump(file):
    # Format:
    # pool 1 pg_num 16
    # 1.0	[1,0,2]	1
    # 1.1	[2,0,1]	2
    # ...
    # pool 2 pg_num 32
    # 2.0	[2,1,0]	2
    # 2.1	[2,1,0]	2
    # ...
    # #osd	count	first	primary	c wt	wt
    # osd.1	208	123	123	0.098587	1

    pgs = {}

    with open(file, "r") as f:
        pool = None
        for l in f.readlines():
            m = re.match('^pool (\d+) pg_num (\d+)', l)
            if m:
                pool = m.group(1)
                continue
            if not pool:
                continue
            m = re.match('^#osd', l)
            if m:
                break
            m = re.match('^(\d+\.[0-9a-f]+)\s+\[([\d,]+)\]', l)
            if not m:
                continue
            pgid = m.group(1)
            osds = [int(x) for x in m.group(2).split(',')]
            pgs[pgid] = osds

    return pgs

def do_compare(new_crushmap_in, osdmap=None, pg_dump=None, compiled=False,
               verbose=False):
    with tempfile.TemporaryDirectory() as tmpdirname:
        if compiled:
            new_crushmap_file = new_crushmap_in
        else:
            new_crushmap_file = os.path.join(tmpdirname, 'crushmap')
            run_cmd('crushtool -c {} -o {}'.format(new_crushmap_in,
                                                   new_crushmap_file), verbose)

        osdmap_file = os.path.join(tmpdirname, 'osdmap')
        if osdmap:
            run_cmd('cp {} {}'.format(osdmap, osdmap_file), verbose)
        else:
            run_cmd('ceph osd getmap -o {}'.format(osdmap_file), verbose)

        if not pg_dump:
            pg_dump = os.path.join(tmpdirname, 'pg_dump.json')
            run_cmd('ceph pg dump --format json > {}'.format(pg_dump), verbose)

        old_test_map_pgs_dump = os.path.join(tmpdirname, 'pgs.old.txt')
        run_cmd('osdmaptool {} --test-map-pgs-dump > {}'.format(
            osdmap_file, old_test_map_pgs_dump), verbose)
        if verbose:
            run_cmd('cat {} >&2'.format(old_test_map_pgs_dump), True)

        new_test_map_pgs_dump = os.path.join(tmpdirname, 'pgs.new.txt')
        run_cmd(
            'osdmaptool {} --import-crush {} --test-map-pgs-dump > {}'.format(
                osdmap_file, new_crushmap_file, new_test_map_pgs_dump), verbose)
        if verbose:
            run_cmd('cat {} >&2'.format(new_test_map_pgs_dump), True)

        osdmap_file_json = os.path.join(tmpdirname, 'osdmap.json')
        run_cmd('osdmaptool {} --dump json > {}'.format(
            osdmap_file, osdmap_file_json), verbose)
        osdmap = get_osdmap(osdmap_file_json)
        pools = get_pools(osdmap)
        ec_profiles = get_erasure_code_profiles(osdmap)

        pgmap = get_pgmap(pg_dump)
        pg_stats = get_pg_stats(pgmap)

        old_pgs = parse_test_map_pgs_dump(old_test_map_pgs_dump)
        new_pgs = parse_test_map_pgs_dump(new_test_map_pgs_dump)

    diff_pg_count = 0
    total_object_count = 0
    diff_object_count = 0
    for pgid in old_pgs:
        objects = pg_stats[pgid]['stat_sum']['num_objects']
        total_object_count += objects

        if old_pgs[pgid] == new_pgs[pgid]:
            continue

        pool_id = int(pgid.split('.')[0])

        if len(new_pgs[pgid]) < pools[pool_id]['size']:
            print("WARNING: {} will be undersized ({})".format(
                pgid, new_pgs[pgid]), file=sys.stderr, flush=True)

        if not pools[pool_id]['erasure_code_profile'] and \
           sorted(old_pgs[pgid]) == sorted(new_pgs[pgid]):
            continue

        if verbose:
            print("{}\t{} -> {}".format(pgid, old_pgs[pgid], new_pgs[pgid]),
                  file=sys.stderr, flush=True)
        diff_pg_count += 1
        diff_object_count += objects

    print("{}/{} ({:.2f}%) pgs affected".format(
        diff_pg_count, len(old_pgs),
        100 * diff_pg_count / len(old_pgs) if len(old_pgs) else 0),
        flush=True)
    print("{}/{} ({:.2f}%) objects affected".format(
        diff_object_count, total_object_count,
        100 * diff_object_count / total_object_count \
        if total_object_count else 0), flush=True)

    total_pg_shard_count = 0
    diff_pg_shard_count = 0
    total_object_shard_count = 0
    diff_object_shard_count = 0
    total_bytes = 0
    diff_bytes = 0
    for pgid in old_pgs:
        pool_id = int(pgid.split('.')[0])
        ec_profile = pools[pool_id]['erasure_code_profile']
        if ec_profile:
            k = int(ec_profiles[ec_profile]['k'])
            m = int(ec_profiles[ec_profile]['m'])
        else:
            k = 1
            m = pools[pool_id]['size'] - 1

        bytes = pg_stats[pgid]['stat_sum']['num_bytes'] + \
            pg_stats[pgid]['stat_sum']['num_omap_bytes']
        objects = pg_stats[pgid]['stat_sum']['num_objects']

        total_pg_shard_count += len(old_pgs[pgid])
        total_object_shard_count += objects * (k + m)
        total_bytes += bytes * (k + m) / k

        if old_pgs[pgid] == new_pgs[pgid]:
            continue

        old_count = diff_pg_shard_count

        if ec_profile:
            for i in range(len(old_pgs[pgid])):
                if old_pgs[pgid][i] != new_pgs[pgid][i]:
                    diff_pg_shard_count += 1
                    diff_object_shard_count += objects
                    diff_bytes += bytes / k
        else:
            for osd in old_pgs[pgid]:
                if osd not in new_pgs[pgid]:
                    diff_pg_shard_count += 1
                    diff_object_shard_count += objects
                    diff_bytes += bytes / k

        if old_count == diff_pg_shard_count:
            continue

        if verbose:
            print("{}\t{} -> {}".format(pgid, old_pgs[pgid], new_pgs[pgid]),
                  file=sys.stderr, flush=True)

    print("{}/{} ({:.2f}%) pg shards to move".format(
        diff_pg_shard_count, total_pg_shard_count,
        100 * diff_pg_shard_count / total_pg_shard_count \
        if total_pg_shard_count else 0), flush=True)
    print("{}/{} ({:.2f}%) pg object shards to move".format(
        diff_object_shard_count, total_object_shard_count,
        100 * diff_object_shard_count / total_object_shard_count \
        if total_object_shard_count else 0), flush=True)
    print("{}/{} ({:.2f}%) bytes to move".format(
        get_human_readable(int(diff_bytes)),
        get_human_readable(int(total_bytes)),
        100 * diff_bytes / total_bytes if total_bytes else 0),
        flush=True)

def do_export(crushmap_out, osdmap_file=None, compiled=False, verbose=False):
    with tempfile.TemporaryDirectory() as tmpdirname:
        if not osdmap_file:
            osdmap_file = os.path.join(tmpdirname, 'osdmap')
            run_cmd('ceph osd getmap -o {}'.format(osdmap_file), verbose)

        crushmap_file = crushmap_out if compiled else \
            os.path.join(tmpdirname, 'crushmap')
        run_cmd('osdmaptool {} --export-crush {}'.format(
            osdmap_file, crushmap_file), verbose)
        if not compiled:
            run_cmd('crushtool -d {} -o {}'.format(crushmap_file, crushmap_out),
                    verbose)

def do_import(crushmap_in, osdmap=None, compiled=False, verbose=False):
    with tempfile.TemporaryDirectory() as tmpdirname:
        if compiled:
            crushmap_file = crushmap_in
        else:
            crushmap_file = os.path.join(tmpdirname, 'crushmap')
            run_cmd('crushtool -c {} -o {}'.format(crushmap_in,
                                                   crushmap_file), verbose)
        if osdmap:
            run_cmd('osdmaptool {} --import-crush {}'.format(
                osdmap, crushmap_file), verbose)
        else:
            run_cmd('ceph osd setcrushmap -i {}'.format(crushmap_file), verbose)

def main():
    args = parser.parse_args()

    if args.command == 'compare':
        do_compare(args.crushmap, args.osdmap, args.pg_dump, args.compiled,
                   args.verbose)
    elif args.command == 'export':
        do_export(args.crushmap, args.osdmap, args.compiled, args.verbose)
    elif args.command == 'import':
        do_import(args.crushmap, args.osdmap, args.compiled, args.verbose)

#
# main
#

main()