summaryrefslogtreecommitdiffstats
path: root/src/test/rgw
diff options
context:
space:
mode:
Diffstat (limited to 'src/test/rgw')
-rw-r--r--src/test/rgw/bucket_notification/test_bn.py65
-rw-r--r--src/test/rgw/rgw_multi/tests.py33
-rw-r--r--src/test/rgw/test_rgw_dmclock_scheduler.cc18
-rw-r--r--src/test/rgw/test_rgw_lc.cc237
4 files changed, 343 insertions, 10 deletions
diff --git a/src/test/rgw/bucket_notification/test_bn.py b/src/test/rgw/bucket_notification/test_bn.py
index 87a2acb76..ee89d326d 100644
--- a/src/test/rgw/bucket_notification/test_bn.py
+++ b/src/test/rgw/bucket_notification/test_bn.py
@@ -2346,6 +2346,71 @@ def test_http_post_object_upload():
conn1.delete_bucket(Bucket=bucket_name)
+@attr('mpu_test')
+def test_ps_s3_multipart_on_master_http():
+ """ test http multipart object upload on master"""
+ conn = connection()
+ zonegroup = 'default'
+
+ # create random port for the http server
+ host = get_ip()
+ port = random.randint(10000, 20000)
+ # start an http server in a separate thread
+ http_server = StreamingHTTPServer(host, port, num_workers=10)
+
+ # create bucket
+ bucket_name = gen_bucket_name()
+ bucket = conn.create_bucket(bucket_name)
+ topic_name = bucket_name + TOPIC_SUFFIX
+
+ # create s3 topic
+ endpoint_address = 'http://'+host+':'+str(port)
+ endpoint_args = 'push-endpoint='+endpoint_address
+ opaque_data = 'http://1.2.3.4:8888'
+ topic_conf = PSTopicS3(conn, topic_name, zonegroup, endpoint_args=endpoint_args, opaque_data=opaque_data)
+ topic_arn = topic_conf.set_config()
+ # create s3 notification
+ notification_name = bucket_name + NOTIFICATION_SUFFIX
+ topic_conf_list = [{'Id': notification_name,
+ 'TopicArn': topic_arn,
+ 'Events': []
+ }]
+ s3_notification_conf = PSNotificationS3(conn, bucket_name, topic_conf_list)
+ response, status = s3_notification_conf.set_config()
+ assert_equal(status/100, 2)
+
+ # create objects in the bucket
+ client_threads = []
+ content = str(os.urandom(20*1024*1024))
+ key = bucket.new_key('obj')
+ thr = threading.Thread(target = set_contents_from_string, args=(key, content,))
+ thr.start()
+ client_threads.append(thr)
+ [thr.join() for thr in client_threads]
+
+ print('wait for 5sec for the messages...')
+ time.sleep(5)
+
+ # check http receiver
+ keys = list(bucket.list())
+ print('total number of objects: ' + str(len(keys)))
+ events = http_server.get_and_reset_events()
+ for event in events:
+ assert_equal(event['Records'][0]['opaqueData'], opaque_data)
+ assert_equal(event['Records'][0]['s3']['object']['eTag'] != '', True)
+ print(event['Records'][0]['s3']['object'])
+
+ # cleanup
+ for key in keys:
+ key.delete()
+ [thr.join() for thr in client_threads]
+ topic_conf.del_config()
+ s3_notification_conf.del_config(notification=notification_name)
+ # delete the bucket
+ conn.delete_bucket(bucket_name)
+ http_server.close()
+
+
@attr('amqp_test')
def test_ps_s3_multipart_on_master():
""" test multipart object upload on master"""
diff --git a/src/test/rgw/rgw_multi/tests.py b/src/test/rgw/rgw_multi/tests.py
index 156fac12e..fee0c9a3a 100644
--- a/src/test/rgw/rgw_multi/tests.py
+++ b/src/test/rgw/rgw_multi/tests.py
@@ -664,6 +664,39 @@ def test_object_delete():
zone_bucket_checkpoint(target_conn.zone, source_conn.zone, bucket.name)
check_bucket_eq(source_conn, target_conn, bucket)
+def test_multi_object_delete():
+ zonegroup = realm.master_zonegroup()
+ zonegroup_conns = ZonegroupConns(zonegroup)
+ buckets, zone_bucket = create_bucket_per_zone(zonegroup_conns)
+
+ objnames = [f'obj{i}' for i in range(1,50)]
+ content = 'asdasd'
+
+ # don't wait for meta sync just yet
+ for zone, bucket in zone_bucket:
+ create_objects(zone, bucket, objnames, content)
+
+ zonegroup_meta_checkpoint(zonegroup)
+
+ # check objects exist
+ for source_conn, bucket in zone_bucket:
+ for target_conn in zonegroup_conns.zones:
+ if source_conn.zone == target_conn.zone:
+ continue
+
+ zone_bucket_checkpoint(target_conn.zone, source_conn.zone, bucket.name)
+ check_bucket_eq(source_conn, target_conn, bucket)
+
+ # check object removal
+ for source_conn, bucket in zone_bucket:
+ bucket.delete_keys(objnames)
+ for target_conn in zonegroup_conns.zones:
+ if source_conn.zone == target_conn.zone:
+ continue
+
+ zone_bucket_checkpoint(target_conn.zone, source_conn.zone, bucket.name)
+ check_bucket_eq(source_conn, target_conn, bucket)
+
def get_latest_object_version(key):
for k in key.bucket.list_versions(key.name):
if k.is_latest:
diff --git a/src/test/rgw/test_rgw_dmclock_scheduler.cc b/src/test/rgw/test_rgw_dmclock_scheduler.cc
index 92800767c..28ae78180 100644
--- a/src/test/rgw/test_rgw_dmclock_scheduler.cc
+++ b/src/test/rgw/test_rgw_dmclock_scheduler.cc
@@ -105,7 +105,7 @@ TEST(Queue, RateLimit)
EXPECT_EQ(1u, counters(client_id::admin)->get(queue_counters::l_qlen));
EXPECT_EQ(1u, counters(client_id::auth)->get(queue_counters::l_qlen));
- context.run_for(std::chrono::milliseconds(1));
+ context.run_for(std::chrono::milliseconds(50));
EXPECT_TRUE(context.stopped());
ASSERT_TRUE(ec1);
@@ -163,7 +163,7 @@ TEST(Queue, AsyncRequest)
EXPECT_EQ(1u, counters(client_id::admin)->get(queue_counters::l_qlen));
EXPECT_EQ(1u, counters(client_id::auth)->get(queue_counters::l_qlen));
- context.run_for(std::chrono::milliseconds(1));
+ context.run_for(std::chrono::milliseconds(50));
EXPECT_TRUE(context.stopped());
ASSERT_TRUE(ec1);
@@ -217,7 +217,7 @@ TEST(Queue, Cancel)
EXPECT_FALSE(ec1);
EXPECT_FALSE(ec2);
- context.run_for(std::chrono::milliseconds(1));
+ context.run_for(std::chrono::milliseconds(50));
EXPECT_TRUE(context.stopped());
ASSERT_TRUE(ec1);
@@ -265,7 +265,7 @@ TEST(Queue, CancelClient)
EXPECT_FALSE(ec1);
EXPECT_FALSE(ec2);
- context.run_for(std::chrono::milliseconds(1));
+ context.run_for(std::chrono::milliseconds(50));
EXPECT_TRUE(context.stopped());
ASSERT_TRUE(ec1);
@@ -315,7 +315,7 @@ TEST(Queue, CancelOnDestructor)
EXPECT_FALSE(ec1);
EXPECT_FALSE(ec2);
- context.run_for(std::chrono::milliseconds(1));
+ context.run_for(std::chrono::milliseconds(50));
EXPECT_TRUE(context.stopped());
ASSERT_TRUE(ec1);
@@ -369,20 +369,20 @@ TEST(Queue, CrossExecutorRequest)
EXPECT_EQ(1u, counters(client_id::admin)->get(queue_counters::l_qlen));
EXPECT_EQ(1u, counters(client_id::auth)->get(queue_counters::l_qlen));
- callback_context.run_for(std::chrono::milliseconds(1));
+ callback_context.poll();
// maintains work on callback executor while in queue
EXPECT_FALSE(callback_context.stopped());
EXPECT_FALSE(ec1);
EXPECT_FALSE(ec2);
- queue_context.run_for(std::chrono::milliseconds(1));
+ queue_context.run_for(std::chrono::milliseconds(50));
EXPECT_TRUE(queue_context.stopped());
EXPECT_FALSE(ec1); // no callbacks until callback executor runs
EXPECT_FALSE(ec2);
- callback_context.run_for(std::chrono::milliseconds(1));
+ callback_context.run_for(std::chrono::milliseconds(50));
EXPECT_TRUE(callback_context.stopped());
ASSERT_TRUE(ec1);
@@ -421,7 +421,7 @@ TEST(Queue, SpawnAsyncRequest)
EXPECT_EQ(PhaseType::priority, p2);
});
- context.run_for(std::chrono::milliseconds(1));
+ context.run_for(std::chrono::milliseconds(50));
EXPECT_TRUE(context.stopped());
}
diff --git a/src/test/rgw/test_rgw_lc.cc b/src/test/rgw/test_rgw_lc.cc
index 83a4cac67..d10b482cb 100644
--- a/src/test/rgw/test_rgw_lc.cc
+++ b/src/test/rgw/test_rgw_lc.cc
@@ -5,7 +5,6 @@
#include "rgw_lc.h"
#include "rgw_lc_s3.h"
#include <gtest/gtest.h>
-//#include <spawn/spawn.hpp>
#include <string>
#include <vector>
#include <stdexcept>
@@ -107,3 +106,239 @@ TEST(TestLCFilterInvalidAnd, XMLDoc3)
/* check our flags */
ASSERT_EQ(filter.get_flags(), uint32_t(LCFlagType::none));
}
+
+struct LCWorkTimeTests : ::testing::Test
+{
+ CephContext* cct;
+ std::unique_ptr<RGWLC::LCWorker> worker;
+
+ // expects input in the form of "%m/%d/%y %H:%M:%S"; e.g., "01/15/23 23:59:01"
+ utime_t get_utime_by_date_time_string(const std::string& date_time_str)
+ {
+ struct tm tm{};
+ struct timespec ts = {0};
+
+ strptime(date_time_str.c_str(), "%m/%d/%y %H:%M:%S", &tm);
+ ts.tv_sec = mktime(&tm);
+
+ return utime_t(ts);
+ }
+
+ // expects a map from input value (date & time string) to expected result (boolean)
+ void run_should_work_test(const auto& test_values_to_expectations_map) {
+ for (const auto& [date_time_str, expected_value] : test_values_to_expectations_map) {
+ auto ut = get_utime_by_date_time_string(date_time_str);
+ auto should_work = worker->should_work(ut);
+
+ ASSERT_EQ(should_work, expected_value)
+ << "input time: " << ut
+ << " expected: " << expected_value
+ << " should_work: " << should_work
+ << " work-time-window: " << cct->_conf->rgw_lifecycle_work_time << std::endl;
+ }
+ }
+
+ // expects a map from input value (a tuple of date & time strings) to expected result (seconds)
+ void run_schedule_next_start_time_test(const auto& test_values_to_expectations_map) {
+ for (const auto& [date_time_str_tuple, expected_value] : test_values_to_expectations_map) {
+ auto work_started_at = get_utime_by_date_time_string(std::get<0>(date_time_str_tuple));
+ auto work_completed_at = get_utime_by_date_time_string(std::get<1>(date_time_str_tuple));
+ auto wait_secs_till_next_start = worker->schedule_next_start_time(work_started_at, work_completed_at);
+
+ ASSERT_EQ(wait_secs_till_next_start, expected_value)
+ << "work_started_at: " << work_started_at
+ << " work_completed_at: " << work_completed_at
+ << " expected: " << expected_value
+ << " wait_secs_till_next_start: " << wait_secs_till_next_start
+ << " work-time-window: " << cct->_conf->rgw_lifecycle_work_time << std::endl;
+ }
+ }
+
+protected:
+
+ void SetUp() override {
+ cct = (new CephContext(CEPH_ENTITY_TYPE_ANY))->get();
+
+ cct->_conf->set_value("rgw_lc_max_wp_worker", 0, 0); // no need to create a real workpool
+ worker = std::make_unique<RGWLC::LCWorker>(nullptr, cct, nullptr, 0);
+ }
+
+ void TearDown() override {
+ worker.reset();
+ cct->put();
+ }
+};
+
+TEST_F(LCWorkTimeTests, ShouldWorkDefaultWorkTime)
+{
+ std::unordered_map<std::string, bool> test_values_to_expectations = {
+ {"01/01/23 00:00:00", true},
+ {"01/01/24 00:00:00", true}, // date is not relevant, but only the time-window
+ {"01/01/23 00:00:01", true},
+ {"01/01/23 03:00:00", true},
+ {"01/01/23 05:59:59", true},
+ {"01/01/23 06:00:00", true},
+ {"01/01/23 06:00:59", true}, // seconds don't matter, but only hours and minutes
+ {"01/01/23 06:01:00", false},
+ {"01/01/23 23:59:59", false},
+ {"01/02/23 23:59:59", false},
+ {"01/01/23 12:00:00", false},
+ {"01/01/23 14:00:00", false}
+ };
+
+ run_should_work_test(test_values_to_expectations);
+}
+
+TEST_F(LCWorkTimeTests, ShouldWorkCustomWorkTimeEndTimeInTheSameDay)
+{
+ cct->_conf->rgw_lifecycle_work_time = "14:00-16:00";
+
+ std::unordered_map<std::string, bool> test_values_to_expectations = {
+ {"01/01/23 00:00:00", false},
+ {"01/01/23 12:00:00", false},
+ {"01/01/24 13:59:59", false},
+ {"01/01/23 14:00:00", true},
+ {"01/01/23 16:00:00", true},
+ {"01/01/23 16:00:59", true},
+ {"01/01/23 16:01:00", false},
+ {"01/01/23 17:00:00", false},
+ {"01/01/23 23:59:59", false},
+ };
+
+ run_should_work_test(test_values_to_expectations);
+}
+
+TEST_F(LCWorkTimeTests, ShouldWorkCustomWorkTimeEndTimeInTheSameDay24Hours)
+{
+ cct->_conf->rgw_lifecycle_work_time = "00:00-23:59";
+
+ std::unordered_map<std::string, bool> test_values_to_expectations = {
+ {"01/01/23 23:59:00", true},
+ {"01/01/23 23:59:59", true},
+ {"01/01/23 00:00:00", true},
+ {"01/01/23 00:00:01", true},
+ {"01/01/23 00:01:00", true},
+ {"01/01/23 01:00:00", true},
+ {"01/01/23 12:00:00", true},
+ {"01/01/23 17:00:00", true},
+ {"01/01/23 23:00:00", true}
+ };
+
+ run_should_work_test(test_values_to_expectations);
+}
+
+
+TEST_F(LCWorkTimeTests, ShouldWorkCustomWorkTimeEndTimeInTheNextDay)
+{
+ cct->_conf->rgw_lifecycle_work_time = "14:00-01:00";
+
+ std::unordered_map<std::string, bool> test_values_to_expectations = {
+ {"01/01/23 13:59:00", false},
+ {"01/01/23 13:59:59", false},
+ {"01/01/24 14:00:00", true}, // used-to-fail
+ {"01/01/24 17:00:00", true}, // used-to-fail
+ {"01/01/24 23:59:59", true}, // used-to-fail
+ {"01/01/23 00:00:00", true}, // used-to-fail
+ {"01/01/23 00:59:59", true}, // used-to-fail
+ {"01/01/23 01:00:00", true}, // used-to-fail
+ {"01/01/23 01:00:59", true}, // used-to-fail
+ {"01/01/23 01:01:00", false},
+ {"01/01/23 05:00:00", false},
+ {"01/01/23 12:00:00", false},
+ {"01/01/23 13:00:00", false}
+ };
+
+ run_should_work_test(test_values_to_expectations);
+}
+
+TEST_F(LCWorkTimeTests, ShouldWorkCustomWorkTimeEndTimeInTheNextDay24Hours)
+{
+ cct->_conf->rgw_lifecycle_work_time = "14:00-13:59";
+
+ // all of the below cases used-to-fail
+ std::unordered_map<std::string, bool> test_values_to_expectations = {
+ {"01/01/23 00:00:00", true},
+ {"01/01/23 00:00:01", true},
+ {"01/01/23 00:01:00", true},
+ {"01/01/24 01:00:00", true},
+ {"01/01/24 12:00:00", true},
+ {"01/01/24 13:00:00", true},
+ {"01/01/24 13:59:00", true},
+ {"01/01/24 13:59:59", true},
+ {"01/01/23 14:00:00", true},
+ {"01/01/23 14:00:01", true},
+ {"01/01/23 14:01:00", true},
+ {"01/01/23 16:00:00", true},
+ {"01/01/23 23:59:00", true},
+ {"01/01/23 23:59:59", true},
+ };
+
+ run_should_work_test(test_values_to_expectations);
+}
+
+TEST_F(LCWorkTimeTests, ShouldWorkCustomWorkTimeEndTimeInTheNextDayIrregularMins)
+{
+ cct->_conf->rgw_lifecycle_work_time = "22:15-03:33";
+
+ std::unordered_map<std::string, bool> test_values_to_expectations = {
+ {"01/01/23 22:14:59", false},
+ {"01/01/23 22:15:00", true}, // used-to-fail
+ {"01/01/24 00:00:00", true}, // used-to-fail
+ {"01/01/24 01:00:00", true}, // used-to-fail
+ {"01/01/24 02:00:00", true}, // used-to-fail
+ {"01/01/23 03:33:00", true}, // used-to-fail
+ {"01/01/23 03:33:59", true}, // used-to-fail
+ {"01/01/23 03:34:00", false},
+ {"01/01/23 04:00:00", false},
+ {"01/01/23 12:00:00", false},
+ {"01/01/23 22:00:00", false},
+ };
+
+ run_should_work_test(test_values_to_expectations);
+}
+
+TEST_F(LCWorkTimeTests, ShouldWorkCustomWorkTimeStartEndSameHour)
+{
+ cct->_conf->rgw_lifecycle_work_time = "22:15-22:45";
+
+ std::unordered_map<std::string, bool> test_values_to_expectations = {
+ {"01/01/23 22:14:59", false},
+ {"01/01/23 22:15:00", true},
+ {"01/01/24 22:44:59", true},
+ {"01/01/24 22:45:59", true},
+ {"01/01/24 22:46:00", false},
+ {"01/01/23 23:00:00", false},
+ {"01/01/23 00:00:00", false},
+ {"01/01/23 12:00:00", false},
+ {"01/01/23 21:00:00", false},
+ };
+
+ run_should_work_test(test_values_to_expectations);
+}
+
+TEST_F(LCWorkTimeTests, ScheduleNextStartTime)
+{
+ cct->_conf->rgw_lifecycle_work_time = "22:15-03:33";
+
+ // items of the map: [ (work_started_time, work_completed_time), expected_value (seconds) ]
+ //
+ // expected_value is the difference between configured start time (i.e, 22:15:00) and
+ // the second item of the tuple (i.e., work_completed_time).
+ //
+ // Note that "seconds" of work completion time is taken into account but date is not relevant.
+ // e.g., the first testcase: 75713 == 01:13:07 - 22:15:00 (https://tinyurl.com/ydm86752)
+ std::map<std::tuple<std::string, std::string>, int> test_values_to_expectations = {
+ {{"01/01/23 22:15:05", "01/01/23 01:13:07"}, 75713},
+ {{"01/01/23 22:15:05", "01/02/23 01:13:07"}, 75713},
+ {{"01/01/23 22:15:05", "01/01/23 22:17:07"}, 86273},
+ {{"01/01/23 22:15:05", "01/02/23 22:17:07"}, 86273},
+ {{"01/01/23 22:15:05", "01/01/23 22:14:00"}, 60},
+ {{"01/01/23 22:15:05", "01/02/23 22:14:00"}, 60},
+ {{"01/01/23 22:15:05", "01/01/23 22:15:00"}, 24 * 60 * 60},
+ {{"01/01/23 22:15:05", "01/02/23 22:15:00"}, 24 * 60 * 60},
+ {{"01/01/23 22:15:05", "01/01/23 22:15:01"}, 24 * 60 * 60 - 1},
+ {{"01/01/23 22:15:05", "01/02/23 22:15:01"}, 24 * 60 * 60 - 1},
+ };
+
+ run_schedule_next_start_time_test(test_values_to_expectations);
+}
generated by cgit v1.2.3 (git 2.39.1) at 2024-07-01 21:35:57 +0000