Adding upstream version 18.2.3.upstream/18.2.3

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

4 files changed, 343 insertions, 10 deletions

diff --git a/src/test/rgw/bucket_notification/test_bn.py b/src/test/rgw/bucket_notification/test_bn.py
index 87a2acb76..ee89d326d 100644
--- a/src/test/rgw/bucket_notification/test_bn.py
+++ b/src/test/rgw/bucket_notification/test_bn.py
@@ -2346,6 +2346,71 @@ def test_http_post_object_upload():
     conn1.delete_bucket(Bucket=bucket_name)
 
 
+@attr('mpu_test')
+def test_ps_s3_multipart_on_master_http():
+    """ test http multipart object upload on master"""
+    conn = connection()
+    zonegroup = 'default'
+
+    # create random port for the http server
+    host = get_ip()
+    port = random.randint(10000, 20000)
+    # start an http server in a separate thread
+    http_server = StreamingHTTPServer(host, port, num_workers=10)
+
+    # create bucket
+    bucket_name = gen_bucket_name()
+    bucket = conn.create_bucket(bucket_name)
+    topic_name = bucket_name + TOPIC_SUFFIX
+
+    # create s3 topic
+    endpoint_address = 'http://'+host+':'+str(port)
+    endpoint_args = 'push-endpoint='+endpoint_address
+    opaque_data = 'http://1.2.3.4:8888'
+    topic_conf = PSTopicS3(conn, topic_name, zonegroup, endpoint_args=endpoint_args, opaque_data=opaque_data)
+    topic_arn = topic_conf.set_config()
+    # create s3 notification
+    notification_name = bucket_name + NOTIFICATION_SUFFIX
+    topic_conf_list = [{'Id': notification_name,
+                        'TopicArn': topic_arn,
+                        'Events': []
+                        }]
+    s3_notification_conf = PSNotificationS3(conn, bucket_name, topic_conf_list)
+    response, status = s3_notification_conf.set_config()
+    assert_equal(status/100, 2)
+
+    # create objects in the bucket
+    client_threads = []
+    content = str(os.urandom(20*1024*1024))
+    key = bucket.new_key('obj')
+    thr = threading.Thread(target = set_contents_from_string, args=(key, content,))
+    thr.start()
+    client_threads.append(thr)
+    [thr.join() for thr in client_threads]
+
+    print('wait for 5sec for the messages...')
+    time.sleep(5)
+
+    # check http receiver
+    keys = list(bucket.list())
+    print('total number of objects: ' + str(len(keys)))
+    events = http_server.get_and_reset_events()
+    for event in events:
+        assert_equal(event['Records'][0]['opaqueData'], opaque_data)
+        assert_equal(event['Records'][0]['s3']['object']['eTag'] != '', True)
+        print(event['Records'][0]['s3']['object'])
+
+    # cleanup
+    for key in keys:
+        key.delete()
+    [thr.join() for thr in client_threads]
+    topic_conf.del_config()
+    s3_notification_conf.del_config(notification=notification_name)
+    # delete the bucket
+    conn.delete_bucket(bucket_name)
+    http_server.close()
+
+
 @attr('amqp_test')
 def test_ps_s3_multipart_on_master():
     """ test multipart object upload on master"""
diff --git a/src/test/rgw/rgw_multi/tests.py b/src/test/rgw/rgw_multi/tests.py
index 156fac12e..fee0c9a3a 100644
--- a/src/test/rgw/rgw_multi/tests.py
+++ b/src/test/rgw/rgw_multi/tests.py
@@ -664,6 +664,39 @@ def test_object_delete():
             zone_bucket_checkpoint(target_conn.zone, source_conn.zone, bucket.name)
             check_bucket_eq(source_conn, target_conn, bucket)
 
+def test_multi_object_delete():
+    zonegroup = realm.master_zonegroup()
+    zonegroup_conns = ZonegroupConns(zonegroup)
+    buckets, zone_bucket = create_bucket_per_zone(zonegroup_conns)
+
+    objnames = [f'obj{i}' for i in range(1,50)]
+    content = 'asdasd'
+
+    # don't wait for meta sync just yet
+    for zone, bucket in zone_bucket:
+        create_objects(zone, bucket, objnames, content)
+
+    zonegroup_meta_checkpoint(zonegroup)
+
+    # check objects exist
+    for source_conn, bucket in zone_bucket:
+        for target_conn in zonegroup_conns.zones:
+            if source_conn.zone == target_conn.zone:
+                continue
+
+            zone_bucket_checkpoint(target_conn.zone, source_conn.zone, bucket.name)
+            check_bucket_eq(source_conn, target_conn, bucket)
+
+    # check object removal
+    for source_conn, bucket in zone_bucket:
+        bucket.delete_keys(objnames)
+        for target_conn in zonegroup_conns.zones:
+            if source_conn.zone == target_conn.zone:
+                continue
+
+            zone_bucket_checkpoint(target_conn.zone, source_conn.zone, bucket.name)
+            check_bucket_eq(source_conn, target_conn, bucket)
+
 def get_latest_object_version(key):
     for k in key.bucket.list_versions(key.name):
         if k.is_latest:
diff --git a/src/test/rgw/test_rgw_dmclock_scheduler.cc b/src/test/rgw/test_rgw_dmclock_scheduler.cc
index 92800767c..28ae78180 100644
--- a/src/test/rgw/test_rgw_dmclock_scheduler.cc
+++ b/src/test/rgw/test_rgw_dmclock_scheduler.cc
@@ -105,7 +105,7 @@ TEST(Queue, RateLimit)
   EXPECT_EQ(1u, counters(client_id::admin)->get(queue_counters::l_qlen));
   EXPECT_EQ(1u, counters(client_id::auth)->get(queue_counters::l_qlen));
 
-  context.run_for(std::chrono::milliseconds(1));
+  context.run_for(std::chrono::milliseconds(50));
   EXPECT_TRUE(context.stopped());
 
   ASSERT_TRUE(ec1);
@@ -163,7 +163,7 @@ TEST(Queue, AsyncRequest)
   EXPECT_EQ(1u, counters(client_id::admin)->get(queue_counters::l_qlen));
   EXPECT_EQ(1u, counters(client_id::auth)->get(queue_counters::l_qlen));
 
-  context.run_for(std::chrono::milliseconds(1));
+  context.run_for(std::chrono::milliseconds(50));
   EXPECT_TRUE(context.stopped());
 
   ASSERT_TRUE(ec1);
@@ -217,7 +217,7 @@ TEST(Queue, Cancel)
   EXPECT_FALSE(ec1);
   EXPECT_FALSE(ec2);
 
-  context.run_for(std::chrono::milliseconds(1));
+  context.run_for(std::chrono::milliseconds(50));
   EXPECT_TRUE(context.stopped());
 
   ASSERT_TRUE(ec1);
@@ -265,7 +265,7 @@ TEST(Queue, CancelClient)
   EXPECT_FALSE(ec1);
   EXPECT_FALSE(ec2);
 
-  context.run_for(std::chrono::milliseconds(1));
+  context.run_for(std::chrono::milliseconds(50));
   EXPECT_TRUE(context.stopped());
 
   ASSERT_TRUE(ec1);
@@ -315,7 +315,7 @@ TEST(Queue, CancelOnDestructor)
   EXPECT_FALSE(ec1);
   EXPECT_FALSE(ec2);
 
-  context.run_for(std::chrono::milliseconds(1));
+  context.run_for(std::chrono::milliseconds(50));
   EXPECT_TRUE(context.stopped());
 
   ASSERT_TRUE(ec1);
@@ -369,20 +369,20 @@ TEST(Queue, CrossExecutorRequest)
   EXPECT_EQ(1u, counters(client_id::admin)->get(queue_counters::l_qlen));
   EXPECT_EQ(1u, counters(client_id::auth)->get(queue_counters::l_qlen));
 
-  callback_context.run_for(std::chrono::milliseconds(1));
+  callback_context.poll();
   // maintains work on callback executor while in queue
   EXPECT_FALSE(callback_context.stopped());
 
   EXPECT_FALSE(ec1);
   EXPECT_FALSE(ec2);
 
-  queue_context.run_for(std::chrono::milliseconds(1));
+  queue_context.run_for(std::chrono::milliseconds(50));
   EXPECT_TRUE(queue_context.stopped());
 
   EXPECT_FALSE(ec1); // no callbacks until callback executor runs
   EXPECT_FALSE(ec2);
 
-  callback_context.run_for(std::chrono::milliseconds(1));
+  callback_context.run_for(std::chrono::milliseconds(50));
   EXPECT_TRUE(callback_context.stopped());
 
   ASSERT_TRUE(ec1);
@@ -421,7 +421,7 @@ TEST(Queue, SpawnAsyncRequest)
     EXPECT_EQ(PhaseType::priority, p2);
   });
 
-  context.run_for(std::chrono::milliseconds(1));
+  context.run_for(std::chrono::milliseconds(50));
   EXPECT_TRUE(context.stopped());
 }
 
diff --git a/src/test/rgw/test_rgw_lc.cc b/src/test/rgw/test_rgw_lc.cc
index 83a4cac67..d10b482cb 100644
--- a/src/test/rgw/test_rgw_lc.cc
+++ b/src/test/rgw/test_rgw_lc.cc
@@ -5,7 +5,6 @@
 #include "rgw_lc.h"
 #include "rgw_lc_s3.h"
 #include <gtest/gtest.h>
-//#include <spawn/spawn.hpp>
 #include <string>
 #include <vector>
 #include <stdexcept>
@@ -107,3 +106,239 @@ TEST(TestLCFilterInvalidAnd, XMLDoc3)
   /* check our flags */
   ASSERT_EQ(filter.get_flags(), uint32_t(LCFlagType::none));
 }
+
+struct LCWorkTimeTests : ::testing::Test
+{
+   CephContext* cct;
+   std::unique_ptr<RGWLC::LCWorker> worker;
+
+   // expects input in the form of "%m/%d/%y %H:%M:%S"; e.g., "01/15/23 23:59:01"
+   utime_t get_utime_by_date_time_string(const std::string& date_time_str)
+   {
+      struct tm tm{};
+      struct timespec ts = {0};
+
+      strptime(date_time_str.c_str(), "%m/%d/%y %H:%M:%S", &tm);
+      ts.tv_sec = mktime(&tm);
+
+      return utime_t(ts);
+   }
+
+   // expects a map from input value (date & time string) to expected result (boolean)
+   void run_should_work_test(const auto& test_values_to_expectations_map) {
+      for (const auto& [date_time_str, expected_value] : test_values_to_expectations_map) {
+         auto ut = get_utime_by_date_time_string(date_time_str);
+         auto should_work = worker->should_work(ut);
+
+         ASSERT_EQ(should_work, expected_value)
+            << "input time: " << ut
+            << " expected: " << expected_value
+            << " should_work: " << should_work
+            << " work-time-window: " << cct->_conf->rgw_lifecycle_work_time << std::endl;
+      }
+   }
+
+   // expects a map from input value (a tuple of date & time strings) to expected result (seconds)
+   void run_schedule_next_start_time_test(const auto& test_values_to_expectations_map) {
+      for (const auto& [date_time_str_tuple, expected_value] : test_values_to_expectations_map) {
+         auto work_started_at = get_utime_by_date_time_string(std::get<0>(date_time_str_tuple));
+         auto work_completed_at = get_utime_by_date_time_string(std::get<1>(date_time_str_tuple));
+         auto wait_secs_till_next_start = worker->schedule_next_start_time(work_started_at, work_completed_at);
+
+         ASSERT_EQ(wait_secs_till_next_start, expected_value)
+            << "work_started_at: " << work_started_at
+            << " work_completed_at: " << work_completed_at
+            << " expected: " << expected_value
+            << " wait_secs_till_next_start: " << wait_secs_till_next_start
+            << " work-time-window: " << cct->_conf->rgw_lifecycle_work_time << std::endl;
+      }
+   }
+
+protected:
+
+   void SetUp() override {
+      cct = (new CephContext(CEPH_ENTITY_TYPE_ANY))->get();
+
+      cct->_conf->set_value("rgw_lc_max_wp_worker", 0, 0); // no need to create a real workpool
+      worker = std::make_unique<RGWLC::LCWorker>(nullptr, cct, nullptr, 0);
+   }
+
+   void TearDown() override {
+      worker.reset();
+      cct->put();
+   }
+};
+
+TEST_F(LCWorkTimeTests, ShouldWorkDefaultWorkTime)
+{
+   std::unordered_map<std::string, bool> test_values_to_expectations = {
+      {"01/01/23 00:00:00", true},
+      {"01/01/24 00:00:00", true}, // date is not relevant, but only the time-window
+      {"01/01/23 00:00:01", true},
+      {"01/01/23 03:00:00", true},
+      {"01/01/23 05:59:59", true},
+      {"01/01/23 06:00:00", true},
+      {"01/01/23 06:00:59", true}, // seconds don't matter, but only hours and minutes
+      {"01/01/23 06:01:00", false},
+      {"01/01/23 23:59:59", false},
+      {"01/02/23 23:59:59", false},
+      {"01/01/23 12:00:00", false},
+      {"01/01/23 14:00:00", false}
+   };
+
+   run_should_work_test(test_values_to_expectations);
+}
+
+TEST_F(LCWorkTimeTests, ShouldWorkCustomWorkTimeEndTimeInTheSameDay)
+{
+   cct->_conf->rgw_lifecycle_work_time = "14:00-16:00";
+
+   std::unordered_map<std::string, bool> test_values_to_expectations = {
+      {"01/01/23 00:00:00", false},
+      {"01/01/23 12:00:00", false},
+      {"01/01/24 13:59:59", false},
+      {"01/01/23 14:00:00", true},
+      {"01/01/23 16:00:00", true},
+      {"01/01/23 16:00:59", true},
+      {"01/01/23 16:01:00", false},
+      {"01/01/23 17:00:00", false},
+      {"01/01/23 23:59:59", false},
+   };
+
+   run_should_work_test(test_values_to_expectations);
+}
+
+TEST_F(LCWorkTimeTests, ShouldWorkCustomWorkTimeEndTimeInTheSameDay24Hours)
+{
+   cct->_conf->rgw_lifecycle_work_time = "00:00-23:59";
+
+   std::unordered_map<std::string, bool> test_values_to_expectations = {
+      {"01/01/23 23:59:00", true},
+      {"01/01/23 23:59:59", true},
+      {"01/01/23 00:00:00", true},
+      {"01/01/23 00:00:01", true},
+      {"01/01/23 00:01:00", true},
+      {"01/01/23 01:00:00", true},
+      {"01/01/23 12:00:00", true},
+      {"01/01/23 17:00:00", true},
+      {"01/01/23 23:00:00", true}
+   };
+
+   run_should_work_test(test_values_to_expectations);
+}
+
+
+TEST_F(LCWorkTimeTests, ShouldWorkCustomWorkTimeEndTimeInTheNextDay)
+{
+   cct->_conf->rgw_lifecycle_work_time = "14:00-01:00";
+
+   std::unordered_map<std::string, bool> test_values_to_expectations = {
+      {"01/01/23 13:59:00", false},
+      {"01/01/23 13:59:59", false},
+      {"01/01/24 14:00:00", true}, // used-to-fail
+      {"01/01/24 17:00:00", true}, // used-to-fail
+      {"01/01/24 23:59:59", true}, // used-to-fail
+      {"01/01/23 00:00:00", true}, // used-to-fail
+      {"01/01/23 00:59:59", true}, // used-to-fail
+      {"01/01/23 01:00:00", true}, // used-to-fail
+      {"01/01/23 01:00:59", true}, // used-to-fail
+      {"01/01/23 01:01:00", false},
+      {"01/01/23 05:00:00", false},
+      {"01/01/23 12:00:00", false},
+      {"01/01/23 13:00:00", false}
+   };
+
+   run_should_work_test(test_values_to_expectations);
+}
+
+TEST_F(LCWorkTimeTests, ShouldWorkCustomWorkTimeEndTimeInTheNextDay24Hours)
+{
+   cct->_conf->rgw_lifecycle_work_time = "14:00-13:59";
+
+   // all of the below cases used-to-fail
+   std::unordered_map<std::string, bool> test_values_to_expectations = {
+      {"01/01/23 00:00:00", true},
+      {"01/01/23 00:00:01", true},
+      {"01/01/23 00:01:00", true},
+      {"01/01/24 01:00:00", true},
+      {"01/01/24 12:00:00", true},
+      {"01/01/24 13:00:00", true},
+      {"01/01/24 13:59:00", true},
+      {"01/01/24 13:59:59", true},
+      {"01/01/23 14:00:00", true},
+      {"01/01/23 14:00:01", true},
+      {"01/01/23 14:01:00", true},
+      {"01/01/23 16:00:00", true},
+      {"01/01/23 23:59:00", true},
+      {"01/01/23 23:59:59", true},
+   };
+
+   run_should_work_test(test_values_to_expectations);
+}
+
+TEST_F(LCWorkTimeTests, ShouldWorkCustomWorkTimeEndTimeInTheNextDayIrregularMins)
+{
+   cct->_conf->rgw_lifecycle_work_time = "22:15-03:33";
+
+   std::unordered_map<std::string, bool> test_values_to_expectations = {
+      {"01/01/23 22:14:59", false},
+      {"01/01/23 22:15:00", true}, // used-to-fail
+      {"01/01/24 00:00:00", true}, // used-to-fail
+      {"01/01/24 01:00:00", true}, // used-to-fail
+      {"01/01/24 02:00:00", true}, // used-to-fail
+      {"01/01/23 03:33:00", true}, // used-to-fail
+      {"01/01/23 03:33:59", true}, // used-to-fail
+      {"01/01/23 03:34:00", false},
+      {"01/01/23 04:00:00", false},
+      {"01/01/23 12:00:00", false},
+      {"01/01/23 22:00:00", false},
+   };
+
+   run_should_work_test(test_values_to_expectations);
+}
+
+TEST_F(LCWorkTimeTests, ShouldWorkCustomWorkTimeStartEndSameHour)
+{
+   cct->_conf->rgw_lifecycle_work_time = "22:15-22:45";
+
+   std::unordered_map<std::string, bool> test_values_to_expectations = {
+      {"01/01/23 22:14:59", false},
+      {"01/01/23 22:15:00", true},
+      {"01/01/24 22:44:59", true},
+      {"01/01/24 22:45:59", true},
+      {"01/01/24 22:46:00", false},
+      {"01/01/23 23:00:00", false},
+      {"01/01/23 00:00:00", false},
+      {"01/01/23 12:00:00", false},
+      {"01/01/23 21:00:00", false},
+   };
+
+   run_should_work_test(test_values_to_expectations);
+}
+
+TEST_F(LCWorkTimeTests, ScheduleNextStartTime)
+{
+   cct->_conf->rgw_lifecycle_work_time = "22:15-03:33";
+
+   // items of the map: [ (work_started_time, work_completed_time), expected_value (seconds) ]
+   //
+   // expected_value is the difference between configured start time (i.e, 22:15:00) and
+   // the second item of the tuple (i.e., work_completed_time).
+   //
+   // Note that "seconds" of work completion time is taken into account but date is not relevant.
+   // e.g., the first testcase: 75713 == 01:13:07 - 22:15:00 (https://tinyurl.com/ydm86752)
+   std::map<std::tuple<std::string, std::string>, int> test_values_to_expectations = {
+      {{"01/01/23 22:15:05", "01/01/23 01:13:07"}, 75713},
+      {{"01/01/23 22:15:05", "01/02/23 01:13:07"}, 75713},
+      {{"01/01/23 22:15:05", "01/01/23 22:17:07"}, 86273},
+      {{"01/01/23 22:15:05", "01/02/23 22:17:07"}, 86273},
+      {{"01/01/23 22:15:05", "01/01/23 22:14:00"}, 60},
+      {{"01/01/23 22:15:05", "01/02/23 22:14:00"}, 60},
+      {{"01/01/23 22:15:05", "01/01/23 22:15:00"}, 24 * 60 * 60},
+      {{"01/01/23 22:15:05", "01/02/23 22:15:00"}, 24 * 60 * 60},
+      {{"01/01/23 22:15:05", "01/01/23 22:15:01"}, 24 * 60 * 60 - 1},
+      {{"01/01/23 22:15:05", "01/02/23 22:15:01"}, 24 * 60 * 60 - 1},
+   };
+
+   run_schedule_next_start_time_test(test_values_to_expectations);
+}