Adding upstream version 18.2.2.upstream/18.2.2

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

1 files changed, 402 insertions, 0 deletions

diff --git a/qa/tasks/s3tests_java.py b/qa/tasks/s3tests_java.py
new file mode 100644
index 000000000..dbe03921c
--- /dev/null
+++ b/qa/tasks/s3tests_java.py
@@ -0,0 +1,402 @@
+"""
+Task for running RGW S3 tests with the AWS Java SDK
+"""
+from io import BytesIO
+import logging
+
+import base64
+import os
+import random
+import string
+import yaml
+import getpass
+
+from teuthology import misc as teuthology
+from teuthology.task import Task
+from teuthology.orchestra import run
+
+log = logging.getLogger(__name__)
+
+"""
+    Task for running RGW S3 tests with the AWS Java SDK
+    
+    Tests run only on clients specified in the s3tests-java config section. 
+    If no client is given a default 'client.0' is chosen.
+    If such does not match the rgw client the task will fail.
+        
+        tasks:
+        - ceph:
+        - rgw: [client.0]
+        - s3tests-java:
+            client.0:
+
+    Extra arguments can be passed by adding options to the corresponding client
+    section under the s3tests-java task (e.g. to run a certain test, 
+    specify a different repository and branch for the test suite, 
+    run in info/debug mode (for the java suite) or forward the gradle output to a log file):
+
+        tasks:
+        - ceph:
+        - rgw: [client.0]
+        - s3tests-java:
+            client.0:
+                force-branch: wip
+                force-repo: 'https://github.com/adamyanova/java_s3tests.git'
+                log-fwd: '../s3tests-java.log'
+                log-level: info
+                extra-args: ['--tests', 'ObjectTest.testEncryptionKeySSECInvalidMd5']
+
+    To run a specific test, provide its name to the extra-args section e.g.:
+        - s3tests-java:
+            client.0:
+                extra-args: ['--tests', 'ObjectTest.testEncryptionKeySSECInvalidMd5']
+    
+"""
+
+
+class S3tests_java(Task):
+    """
+    Download and install S3 tests in Java
+    This will require openjdk and gradle
+    """
+
+    def __init__(self, ctx, config):
+        super(S3tests_java, self).__init__(ctx, config)
+        self.log = log
+        log.debug('S3 Tests Java: __INIT__ ')
+        assert hasattr(ctx, 'rgw'), 'S3tests_java must run after the rgw task'
+        clients = ['client.{id}'.format(id=id_)
+                   for id_ in teuthology.all_roles_of_type(self.ctx.cluster, 'client')]
+        self.all_clients = []
+        for client in clients:
+            if client in self.config:
+                self.all_clients.extend([client])
+        if self.all_clients is None:
+            self.all_clients = 'client.0'
+        self.users = {'s3main': 'tester',
+                      's3alt': 'johndoe', 'tenanted': 'testx$tenanteduser'}
+
+    def setup(self):
+        super(S3tests_java, self).setup()
+        log.debug('S3 Tests Java: SETUP')
+        for client in self.all_clients:
+            self.download_test_suite(client)
+            self.install_required_packages(client)
+
+    def begin(self):
+        super(S3tests_java, self).begin()
+        log.debug('S3 Tests Java: BEGIN')
+        for (host, roles) in self.ctx.cluster.remotes.items():
+            log.debug(
+                'S3 Tests Java: Cluster config is: {cfg}'.format(cfg=roles))
+            log.debug('S3 Tests Java: Host is: {host}'.format(host=host))
+        self.create_users()
+        self.run_tests()
+
+    def end(self):
+        super(S3tests_java, self).end()
+        log.debug('S3 Tests Java: END')
+        for client in self.all_clients:
+            self.remove_tests(client)
+            self.delete_users(client)
+
+    def download_test_suite(self, client):
+        log.info("S3 Tests Java: Downloading test suite...")
+        testdir = teuthology.get_testdir(self.ctx)
+        branch = 'master'
+        repo = 'https://github.com/ceph/java_s3tests.git'
+        if client in self.config and self.config[client] is not None:
+            if 'force-branch' in self.config[client] and self.config[client]['force-branch'] is not None:
+                branch = self.config[client]['force-branch']
+            if 'force-repo' in self.config[client] and self.config[client]['force-repo'] is not None:
+                repo = self.config[client]['force-repo']
+        self.ctx.cluster.only(client).run(
+            args=[
+                'git', 'clone',
+                '-b', branch,
+                repo,
+                '{tdir}/s3-tests-java'.format(tdir=testdir),
+            ],
+            stdout=BytesIO()
+        )
+        if client in self.config and self.config[client] is not None:
+            if 'sha1' in self.config[client] and self.config[client]['sha1'] is not None:
+                self.ctx.cluster.only(client).run(
+                    args=[
+                        'cd', '{tdir}/s3-tests-java'.format(tdir=testdir),
+                        run.Raw('&&'),
+                        'git', 'reset', '--hard', self.config[client]['sha1'],
+                    ],
+                )
+
+            if 'log-level' in self.config[client]:
+                if self.config[client]['log-level'] == 'info':
+                    self.ctx.cluster.only(client).run(
+                        args=[
+                            'sed', '-i', '\'s/log4j.rootLogger=WARN/log4j.rootLogger=INFO/g\'',
+                            '{tdir}/s3-tests-java/src/main/resources/log4j.properties'.format(
+                                tdir=testdir)
+                        ]
+                    )
+                if self.config[client]['log-level'] == 'debug':
+                    self.ctx.cluster.only(client).run(
+                        args=[
+                            'sed', '-i', '\'s/log4j.rootLogger=WARN/log4j.rootLogger=DEBUG/g\'',
+                            '{tdir}/s3-tests-java/src/main/resources/log4j.properties'.format(
+                                tdir=testdir)
+                        ]
+                    )
+
+    def install_required_packages(self, client):
+        """
+        Run bootstrap script to install openjdk and gradle.
+        Add certificates to java keystore
+        """
+        log.info("S3 Tests Java: Installing required packages...")
+        testdir = teuthology.get_testdir(self.ctx)
+        self.ctx.cluster.only(client).run(
+            args=['{tdir}/s3-tests-java/bootstrap.sh'.format(tdir=testdir)],
+            stdout=BytesIO()
+        )
+
+        endpoint = self.ctx.rgw.role_endpoints[client]
+        if endpoint.cert:
+            path = 'lib/security/cacerts'
+            self.ctx.cluster.only(client).run(
+                args=['sudo',
+                      'keytool',
+                      '-import', '-alias', '{alias}'.format(
+                          alias=endpoint.hostname),
+                      '-keystore',
+                      run.Raw(
+                          '$(readlink -e $(dirname $(readlink -e $(which keytool)))/../{path})'.format(path=path)),
+                      '-file', endpoint.cert.certificate,
+                      '-storepass', 'changeit',
+                      ],
+                stdout=BytesIO()
+            )
+
+    def create_users(self):
+        """
+        Create a main and an alternative s3 user.
+        Configuration is read from a skelethon config file
+        s3tests.teuth.config.yaml in the java-s3tests repository
+        and missing information is added from the task.
+        Existing values are NOT overriden unless they are empty!
+        """
+        log.info("S3 Tests Java: Creating S3 users...")
+        testdir = teuthology.get_testdir(self.ctx)
+        for client in self.all_clients:
+            endpoint = self.ctx.rgw.role_endpoints.get(client)
+            local_user = getpass.getuser()
+            remote_user = teuthology.get_test_user()
+            os.system("scp {remote}@{host}:{tdir}/s3-tests-java/s3tests.teuth.config.yaml /home/{local}/".format(
+                host=endpoint.hostname, tdir=testdir, remote=remote_user, local=local_user))
+            s3tests_conf = teuthology.config_file(
+                '/home/{local}/s3tests.teuth.config.yaml'.format(local=local_user))
+            log.debug("S3 Tests Java: s3tests_conf is {s3cfg}".format(
+                s3cfg=s3tests_conf))
+            for section, user in list(self.users.items()):
+                if section in s3tests_conf:
+                    s3_user_id = '{user}.{client}'.format(
+                        user=user, client=client)
+                    log.debug(
+                        'S3 Tests Java: Creating user {s3_user_id}'.format(s3_user_id=s3_user_id))
+                    self._config_user(s3tests_conf=s3tests_conf,
+                                      section=section, user=s3_user_id, client=client)
+                    cluster_name, daemon_type, client_id = teuthology.split_role(
+                        client)
+                    client_with_id = daemon_type + '.' + client_id
+                    args = [
+                        'adjust-ulimits',
+                        'ceph-coverage',
+                        '{tdir}/archive/coverage'.format(tdir=testdir),
+                        'radosgw-admin',
+                        '-n', client_with_id,
+                        'user', 'create',
+                        '--uid', s3tests_conf[section]['user_id'],
+                        '--display-name', s3tests_conf[section]['display_name'],
+                        '--access-key', s3tests_conf[section]['access_key'],
+                        '--secret', s3tests_conf[section]['access_secret'],
+                        '--email', s3tests_conf[section]['email'],
+                        '--cluster', cluster_name,
+                    ]
+                    log.info('{args}'.format(args=args))
+                    self.ctx.cluster.only(client).run(
+                        args=args,
+                        stdout=BytesIO()
+                    )
+                else:
+                    self.users.pop(section)
+            self._write_cfg_file(s3tests_conf, client)
+            os.system(
+                "rm -rf /home/{local}/s3tests.teuth.config.yaml".format(local=local_user))
+
+    def _config_user(self, s3tests_conf, section, user, client):
+        """
+        Generate missing users data for this section by stashing away keys, ids, and
+        email addresses.
+        """
+        access_key = ''.join(random.choice(string.ascii_uppercase)
+                             for i in range(20))
+        access_secret = base64.b64encode(os.urandom(40)).decode('ascii')
+        endpoint = self.ctx.rgw.role_endpoints.get(client)
+
+        self._set_cfg_entry(
+            s3tests_conf[section], 'user_id', '{user}'.format(user=user))
+        self._set_cfg_entry(
+            s3tests_conf[section], 'email', '{user}_test@test.test'.format(user=user))
+        self._set_cfg_entry(
+            s3tests_conf[section], 'display_name', 'Ms. {user}'.format(user=user))
+        self._set_cfg_entry(
+            s3tests_conf[section], 'access_key', '{ak}'.format(ak=access_key))
+        self._set_cfg_entry(
+            s3tests_conf[section], 'access_secret', '{asc}'.format(asc=access_secret))
+        self._set_cfg_entry(
+            s3tests_conf[section], 'region', 'us-east-1')
+        self._set_cfg_entry(
+            s3tests_conf[section], 'endpoint', '{ip}:{port}'.format(
+                ip=endpoint.hostname, port=endpoint.port))
+        self._set_cfg_entry(
+            s3tests_conf[section], 'host', endpoint.hostname)
+        self._set_cfg_entry(
+            s3tests_conf[section], 'port', endpoint.port)
+        self._set_cfg_entry(
+            s3tests_conf[section], 'is_secure', True if endpoint.cert else False)
+
+        log.debug("S3 Tests Java: s3tests_conf[{sect}] is {s3cfg}".format(
+            sect=section, s3cfg=s3tests_conf[section]))
+        log.debug('S3 Tests Java: Setion, User = {sect}, {user}'.format(
+            sect=section, user=user))
+
+    def _write_cfg_file(self, cfg_dict, client):
+        """
+        Write s3 tests java config file on the remote node.
+        """
+        testdir = teuthology.get_testdir(self.ctx)
+        (remote,) = self.ctx.cluster.only(client).remotes.keys()
+        data = yaml.safe_dump(cfg_dict, default_flow_style=False)
+        path = testdir + '/archive/s3-tests-java.' + client + '.conf'
+        remote.write_file(path, data)
+
+    def _set_cfg_entry(self, cfg_dict, key, value):
+        if not (key in cfg_dict):
+            cfg_dict.setdefault(key, value)
+        elif cfg_dict[key] is None:
+            cfg_dict[key] = value
+
+    def run_tests(self):
+        log.info("S3 Tests Java: Running tests...")
+        testdir = teuthology.get_testdir(self.ctx)
+        for client in self.all_clients:
+            self.ctx.cluster.only(client).run(
+                args=['cp',
+                      '{tdir}/archive/s3-tests-java.{client}.conf'.format(
+                          tdir=testdir, client=client),
+                      '{tdir}/s3-tests-java/config.properties'.format(
+                          tdir=testdir)
+                      ],
+                stdout=BytesIO()
+            )
+            args = ['cd',
+                    '{tdir}/s3-tests-java'.format(tdir=testdir),
+                    run.Raw('&&'),
+                    '/opt/gradle/gradle/bin/gradle', 'clean', 'test',
+                    '--rerun-tasks', '--no-build-cache',
+                    ]
+            extra_args = []
+            suppress_groups = False
+            self.log_fwd = False
+            self.log_name = ''
+            if client in self.config and self.config[client] is not None:
+                if 'extra-args' in self.config[client]:
+                    extra_args.extend(self.config[client]['extra-args'])
+                    suppress_groups = True
+                if 'log-level' in self.config[client] and self.config[client]['log-level'] == 'debug':
+                    extra_args += ['--debug']
+                if 'log-fwd' in self.config[client]:
+                    self.log_fwd = True
+                    self.log_name = '{tdir}/s3tests_log.txt'.format(
+                        tdir=testdir)
+                    if self.config[client]['log-fwd'] is not None:
+                        self.log_name = self.config[client]['log-fwd']
+                    extra_args += [run.Raw('>>'),
+                                   self.log_name]
+
+            if not suppress_groups:
+                test_groups = ['AWS4Test', 'BucketTest', 'ObjectTest']
+            else:
+                test_groups = ['All']
+
+            for gr in test_groups:
+                for i in range(2):
+                    self.ctx.cluster.only(client).run(
+                        args=['radosgw-admin', 'gc',
+                              'process', '--include-all'],
+                        stdout=BytesIO()
+                    )
+
+                if gr != 'All':
+                    self.ctx.cluster.only(client).run(
+                        args=args + ['--tests'] + [gr] + extra_args,
+                        stdout=BytesIO()
+                    )
+                else:
+                    self.ctx.cluster.only(client).run(
+                        args=args + extra_args,
+                        stdout=BytesIO()
+                    )
+
+                for i in range(2):
+                    self.ctx.cluster.only(client).run(
+                        args=['radosgw-admin', 'gc',
+                              'process', '--include-all'],
+                        stdout=BytesIO()
+                    )
+
+    def remove_tests(self, client):
+        log.info('S3 Tests Java: Cleaning up s3-tests-java...')
+        testdir = teuthology.get_testdir(self.ctx)
+
+        if self.log_fwd:
+            self.ctx.cluster.only(client).run(
+                args=['cd',
+                      '{tdir}/s3-tests-java'.format(tdir=testdir),
+                      run.Raw('&&'),
+                      'cat', self.log_name,
+                      run.Raw('&&'),
+                      'rm', self.log_name],
+                stdout=BytesIO()
+            )
+
+        self.ctx.cluster.only(client).run(
+            args=[
+                'rm',
+                '-rf',
+                '{tdir}/s3-tests-java'.format(tdir=testdir),
+            ],
+            stdout=BytesIO()
+        )
+
+    def delete_users(self, client):
+        log.info("S3 Tests Java: Deleting S3 users...")
+        testdir = teuthology.get_testdir(self.ctx)
+        for section, user in self.users.items():
+            s3_user_id = '{user}.{client}'.format(user=user, client=client)
+            self.ctx.cluster.only(client).run(
+                args=[
+                    'adjust-ulimits',
+                    'ceph-coverage',
+                    '{tdir}/archive/coverage'.format(tdir=testdir),
+                    'radosgw-admin',
+                    '-n', client,
+                    'user', 'rm',
+                    '--uid', s3_user_id,
+                    '--purge-data',
+                    '--cluster', 'ceph',
+                ],
+                stdout=BytesIO()
+            )
+
+
+task = S3tests_java