summaryrefslogtreecommitdiffstats
path: root/qa/tasks/rgw_multisite.py
blob: f5a6f5a261519e184426d9fcb844a1bb9ae29371 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
"""
rgw multisite configuration routines
"""
import argparse
import logging
import random
import string
from copy import deepcopy
from tasks.util.rgw import rgwadmin, wait_for_radosgw
from tasks.util.rados import create_ec_pool, create_replicated_pool
from tasks.rgw_multi import multisite
from tasks.rgw_multi.zone_rados import RadosZone as RadosZone

from teuthology.orchestra import run
from teuthology import misc
from teuthology.exceptions import ConfigError
from teuthology.task import Task

log = logging.getLogger(__name__)

class RGWMultisite(Task):
    """
    Performs rgw multisite configuration to match the given realm definition.

        - rgw-multisite:
            realm:
              name: test-realm
              is_default: true

    List one or more zonegroup definitions. These are provided as json
    input to `radosgw-admin zonegroup set`, with the exception of these keys:

    * 'is_master' is passed on the command line as --master
    * 'is_default' is passed on the command line as --default
    * 'endpoints' given as client names are replaced with actual endpoints

            zonegroups:
              - name: test-zonegroup
                api_name: test-api
                is_master: true
                is_default: true
                endpoints: [c1.client.0]

    List each of the zones to be created in this zonegroup.

                zones:
                  - name: test-zone1
                    is_master: true
                    is_default: true
                    endpoints: [c1.client.0]
                  - name: test-zone2
                    is_default: true
                    endpoints: [c2.client.0]

    A complete example:

        tasks:
        - install:
        - ceph: {cluster: c1}
        - ceph: {cluster: c2}
        - rgw:
            c1.client.0:
            c2.client.0:
        - rgw-multisite:
            realm:
              name: test-realm
              is_default: true
            zonegroups:
              - name: test-zonegroup
                is_master: true
                is_default: true
                zones:
                  - name: test-zone1
                    is_master: true
                    is_default: true
                    endpoints: [c1.client.0]
                  - name: test-zone2
                    is_default: true
                    endpoints: [c2.client.0]

    """
    def __init__(self, ctx, config):
        super(RGWMultisite, self).__init__(ctx, config)

    def setup(self):
        super(RGWMultisite, self).setup()

        overrides = self.ctx.config.get('overrides', {})
        misc.deep_merge(self.config, overrides.get('rgw-multisite', {}))

        if not self.ctx.rgw:
            raise ConfigError('rgw-multisite must run after the rgw task')
        role_endpoints = self.ctx.rgw.role_endpoints

        # construct Clusters and Gateways for each client in the rgw task
        clusters, gateways = extract_clusters_and_gateways(self.ctx,
                                                           role_endpoints)

        # get the master zone and zonegroup configuration
        mz, mzg = extract_master_zone_zonegroup(self.config['zonegroups'])
        cluster1 = cluster_for_zone(clusters, mz)

        # create the realm and period on the master zone's cluster
        log.info('creating realm..')
        realm = create_realm(cluster1, self.config['realm'])
        period = realm.current_period

        creds = gen_credentials()

        # create the master zonegroup and its master zone
        log.info('creating master zonegroup..')
        master_zonegroup = create_zonegroup(cluster1, gateways, period,
                                            deepcopy(mzg))
        period.master_zonegroup = master_zonegroup

        log.info('creating master zone..')
        master_zone = create_zone(self.ctx, cluster1, gateways, creds,
                                  master_zonegroup, deepcopy(mz))
        master_zonegroup.master_zone = master_zone

        period.update(master_zone, commit=True)
        restart_zone_gateways(master_zone) # restart with --rgw-zone

        # create the admin user on the master zone
        log.info('creating admin user..')
        user_args = ['--display-name', 'Realm Admin', '--system']
        user_args += creds.credential_args()
        admin_user = multisite.User('realm-admin')
        admin_user.create(master_zone, user_args)

        # process 'zonegroups'
        for zg_config in self.config['zonegroups']:
            zones_config = zg_config.pop('zones')

            zonegroup = None
            for zone_config in zones_config:
                # get the cluster for this zone
                cluster = cluster_for_zone(clusters, zone_config)

                if cluster != cluster1: # already created on master cluster
                    log.info('pulling realm configuration to %s', cluster.name)
                    realm.pull(cluster, master_zone.gateways[0], creds)

                # use the first zone's cluster to create the zonegroup
                if not zonegroup:
                    if zg_config['name'] == master_zonegroup.name:
                        zonegroup = master_zonegroup
                    else:
                        log.info('creating zonegroup..')
                        zonegroup = create_zonegroup(cluster, gateways,
                                                     period, zg_config)

                if zone_config['name'] == master_zone.name:
                    # master zone was already created
                    zone = master_zone
                else:
                    # create the zone and commit the period
                    log.info('creating zone..')
                    zone = create_zone(self.ctx, cluster, gateways, creds,
                                       zonegroup, zone_config)
                    period.update(zone, commit=True)

                    restart_zone_gateways(zone) # restart with --rgw-zone

        # attach configuration to the ctx for other tasks
        self.ctx.rgw_multisite = argparse.Namespace()
        self.ctx.rgw_multisite.clusters = clusters
        self.ctx.rgw_multisite.gateways = gateways
        self.ctx.rgw_multisite.realm = realm
        self.ctx.rgw_multisite.admin_user = admin_user

        log.info('rgw multisite configuration completed')

    def end(self):
        del self.ctx.rgw_multisite

class Cluster(multisite.Cluster):
    """ Issues 'radosgw-admin' commands with the rgwadmin() helper """
    def __init__(self, ctx, name, client):
        super(Cluster, self).__init__()
        self.ctx = ctx
        self.name = name
        self.client = client

    def admin(self, args = None, **kwargs):
        """ radosgw-admin command """
        args = args or []
        args += ['--cluster', self.name]
        args += ['--debug-rgw', str(kwargs.pop('debug_rgw', 0))]
        args += ['--debug-ms', str(kwargs.pop('debug_ms', 0))]
        if kwargs.pop('read_only', False):
            args += ['--rgw-cache-enabled', 'false']
        kwargs['decode'] = False
        check_retcode = kwargs.pop('check_retcode', True)
        r, s = rgwadmin(self.ctx, self.client, args, **kwargs)
        if check_retcode:
            assert r == 0
        return s, r

class Gateway(multisite.Gateway):
    """ Controls a radosgw instance using its daemon """
    def __init__(self, role, remote, daemon, *args, **kwargs):
        super(Gateway, self).__init__(*args, **kwargs)
        self.role = role
        self.remote = remote
        self.daemon = daemon

    def set_zone(self, zone):
        """ set the zone and add its args to the daemon's command line """
        assert self.zone is None, 'zone can only be set once'
        self.zone = zone
        # daemon.restart_with_args() would be perfect for this, except that
        # radosgw args likely include a pipe and redirect. zone arguments at
        # the end won't actually apply to radosgw
        args = self.daemon.command_kwargs.get('args', [])
        try:
            # insert zone args before the first |
            pipe = args.index(run.Raw('|'))
            args = args[0:pipe] + zone.zone_args() + args[pipe:]
        except ValueError:
            args += zone.zone_args()
        self.daemon.command_kwargs['args'] = args

    def start(self, args = None):
        """ (re)start the daemon """
        self.daemon.restart()
        # wait until startup completes
        wait_for_radosgw(self.endpoint(), self.remote)

    def stop(self):
        """ stop the daemon """
        self.daemon.stop()

def extract_clusters_and_gateways(ctx, role_endpoints):
    """ create cluster and gateway instances for all of the radosgw roles """
    clusters = {}
    gateways = {}
    for role, endpoint in role_endpoints.items():
        cluster_name, daemon_type, client_id = misc.split_role(role)
        # find or create the cluster by name
        cluster = clusters.get(cluster_name)
        if not cluster:
            clusters[cluster_name] = cluster = Cluster(ctx, cluster_name, role)
        # create a gateway for this daemon
        client_with_id = daemon_type + '.' + client_id # match format from rgw.py
        daemon = ctx.daemons.get_daemon('rgw', client_with_id, cluster_name)
        if not daemon:
            raise ConfigError('no daemon for role=%s cluster=%s type=rgw id=%s' % \
                              (role, cluster_name, client_id))
        (remote,) = ctx.cluster.only(role).remotes.keys()
        gateways[role] = Gateway(role, remote, daemon, endpoint.hostname,
                endpoint.port, cluster)
    return clusters, gateways

def create_realm(cluster, config):
    """ create a realm from configuration and initialize its first period """
    realm = multisite.Realm(config['name'])
    args = []
    if config.get('is_default', False):
        args += ['--default']
    realm.create(cluster, args)
    realm.current_period = multisite.Period(realm)
    return realm

def extract_user_credentials(config):
    """ extract keys from configuration """
    return multisite.Credentials(config['access_key'], config['secret_key'])

def extract_master_zone(zonegroup_config):
    """ find and return the master zone definition """
    master = None
    for zone in zonegroup_config['zones']:
        if not zone.get('is_master', False):
            continue
        if master:
            raise ConfigError('zones %s and %s cannot both set \'is_master\'' % \
                              (master['name'], zone['name']))
        master = zone
        # continue the loop so we can detect duplicates
    if not master:
        raise ConfigError('one zone must set \'is_master\' in zonegroup %s' % \
                          zonegroup_config['name'])
    return master

def extract_master_zone_zonegroup(zonegroups_config):
    """ find and return the master zone and zonegroup definitions """
    master_zone, master_zonegroup = (None, None)
    for zonegroup in zonegroups_config:
        # verify that all zonegroups have a master zone set, even if they
        # aren't in the master zonegroup
        zone = extract_master_zone(zonegroup)
        if not zonegroup.get('is_master', False):
            continue
        if master_zonegroup:
            raise ConfigError('zonegroups %s and %s cannot both set \'is_master\'' % \
                              (master_zonegroup['name'], zonegroup['name']))
        master_zonegroup = zonegroup
        master_zone = zone
        # continue the loop so we can detect duplicates
    if not master_zonegroup:
        raise ConfigError('one zonegroup must set \'is_master\'')
    return master_zone, master_zonegroup

def extract_zone_cluster_name(zone_config):
    """ return the cluster (must be common to all zone endpoints) """
    cluster_name = None
    endpoints = zone_config.get('endpoints')
    if not endpoints:
        raise ConfigError('zone %s missing \'endpoints\' list' % \
                          zone_config['name'])
    for role in endpoints:
        name, _, _ = misc.split_role(role)
        if not cluster_name:
            cluster_name = name
        elif cluster_name != name:
            raise ConfigError('all zone %s endpoints must be in the same cluster' % \
                              zone_config['name'])
    return cluster_name

def cluster_for_zone(clusters, zone_config):
    """ return the cluster entry for the given zone """
    name = extract_zone_cluster_name(zone_config)
    try:
        return clusters[name]
    except KeyError:
        raise ConfigError('no cluster %s found' % name)

def gen_access_key():
    return ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(16))

def gen_secret():
    return ''.join(random.choice(string.ascii_uppercase + string.ascii_lowercase + string.digits) for _ in range(32))

def gen_credentials():
    return multisite.Credentials(gen_access_key(), gen_secret())

def extract_gateway_endpoints(gateways, endpoints_config):
    """ return a list of gateway endpoints associated with the given roles """
    endpoints = []
    for role in endpoints_config:
        try:
            # replace role names with their gateway's endpoint
            endpoints.append(gateways[role].endpoint())
        except KeyError:
            raise ConfigError('no radosgw endpoint found for role %s' % role)
    return endpoints

def is_default_arg(config):
    return ['--default'] if config.pop('is_default', False) else []

def is_master_arg(config):
    return ['--master'] if config.pop('is_master', False) else []

def create_zonegroup(cluster, gateways, period, config):
    """ pass the zonegroup configuration to `zonegroup set` """
    config.pop('zones', None) # remove 'zones' from input to `zonegroup set`
    endpoints = config.get('endpoints')
    if endpoints:
        # replace client names with their gateway endpoints
        config['endpoints'] = extract_gateway_endpoints(gateways, endpoints)
    zonegroup = multisite.ZoneGroup(config['name'], period)
    # `zonegroup set` needs --default on command line, and 'is_master' in json
    args = is_default_arg(config)
    zonegroup.set(cluster, config, args)
    period.zonegroups.append(zonegroup)
    return zonegroup

def create_zone(ctx, cluster, gateways, creds, zonegroup, config):
    """ create a zone with the given configuration """
    zone = multisite.Zone(config['name'], zonegroup, cluster)
    zone = RadosZone(config['name'], zonegroup, cluster)

    # collect Gateways for the zone's endpoints
    endpoints = config.get('endpoints')
    if not endpoints:
        raise ConfigError('no \'endpoints\' for zone %s' % config['name'])
    zone.gateways = [gateways[role] for role in endpoints]
    for gateway in zone.gateways:
        gateway.set_zone(zone)

    # format the gateway endpoints
    endpoints = [g.endpoint() for g in zone.gateways]

    args = is_default_arg(config)
    args += is_master_arg(config)
    args += creds.credential_args()
    if len(endpoints):
        args += ['--endpoints', ','.join(endpoints)]
    zone.create(cluster, args)
    zonegroup.zones.append(zone)

    create_zone_pools(ctx, zone)
    if ctx.rgw.compression_type:
        configure_zone_compression(zone, ctx.rgw.compression_type)

    zonegroup.zones_by_type.setdefault(zone.tier_type(), []).append(zone)

    if zone.is_read_only():
        zonegroup.ro_zones.append(zone)
    else:
        zonegroup.rw_zones.append(zone)

    return zone

def create_zone_pools(ctx, zone):
    """ Create the data_pool for each placement type """
    gateway = zone.gateways[0]
    cluster = zone.cluster
    for pool_config in zone.data.get('placement_pools', []):
        pool_name = pool_config['val']['storage_classes']['STANDARD']['data_pool']
        if ctx.rgw.ec_data_pool:
            create_ec_pool(gateway.remote, pool_name, zone.name, 64,
                           ctx.rgw.erasure_code_profile, cluster.name, 'rgw')
        else:
            create_replicated_pool(gateway.remote, pool_name, 64, cluster.name, 'rgw')

def configure_zone_compression(zone, compression):
    """ Set compression type in the zone's default-placement """
    zone.json_command(zone.cluster, 'placement', ['modify',
                          '--placement-id', 'default-placement',
                          '--compression', compression
                      ])

def restart_zone_gateways(zone):
    zone.stop()
    zone.start()

task = RGWMultisite