diff options
author | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-04-14 13:40:54 +0000 |
---|---|---|
committer | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-04-14 13:40:54 +0000 |
commit | 317c0644ccf108aa23ef3fd8358bd66c2840bfc0 (patch) | |
tree | c417b3d25c86b775989cb5ac042f37611b626c8a /tests/unit/moduleapi | |
parent | Initial commit. (diff) | |
download | redis-317c0644ccf108aa23ef3fd8358bd66c2840bfc0.tar.xz redis-317c0644ccf108aa23ef3fd8358bd66c2840bfc0.zip |
Adding upstream version 5:7.2.4.upstream/5%7.2.4
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'tests/unit/moduleapi')
41 files changed, 6998 insertions, 0 deletions
diff --git a/tests/unit/moduleapi/aclcheck.tcl b/tests/unit/moduleapi/aclcheck.tcl new file mode 100644 index 0000000..ae3f671 --- /dev/null +++ b/tests/unit/moduleapi/aclcheck.tcl @@ -0,0 +1,137 @@ +set testmodule [file normalize tests/modules/aclcheck.so] + +start_server {tags {"modules acl"}} { + r module load $testmodule + + test {test module check acl for command perm} { + # by default all commands allowed + assert_equal [r aclcheck.rm_call.check.cmd set x 5] OK + # block SET command for user + r acl setuser default -set + catch {r aclcheck.rm_call.check.cmd set x 5} e + assert_match {*DENIED CMD*} $e + + # verify that new log entry added + set entry [lindex [r ACL LOG] 0] + assert {[dict get $entry username] eq {default}} + assert {[dict get $entry context] eq {module}} + assert {[dict get $entry object] eq {set}} + assert {[dict get $entry reason] eq {command}} + } + + test {test module check acl for key perm} { + # give permission for SET and block all keys but x(READ+WRITE), y(WRITE), z(READ) + r acl setuser default +set resetkeys ~x %W~y %R~z + + assert_equal [r aclcheck.set.check.key "*" x 5] OK + catch {r aclcheck.set.check.key "*" v 5} e + assert_match "*DENIED KEY*" $e + + assert_equal [r aclcheck.set.check.key "~" x 5] OK + assert_equal [r aclcheck.set.check.key "~" y 5] OK + assert_equal [r aclcheck.set.check.key "~" z 5] OK + catch {r aclcheck.set.check.key "~" v 5} e + assert_match "*DENIED KEY*" $e + + assert_equal [r aclcheck.set.check.key "W" y 5] OK + catch {r aclcheck.set.check.key "W" v 5} e + assert_match "*DENIED KEY*" $e + + assert_equal [r aclcheck.set.check.key "R" z 5] OK + catch {r aclcheck.set.check.key "R" v 5} e + assert_match "*DENIED KEY*" $e + } + + test {test module check acl for module user} { + # the module user has access to all keys + assert_equal [r aclcheck.rm_call.check.cmd.module.user set y 5] OK + } + + test {test module check acl for channel perm} { + # block all channels but ch1 + r acl setuser default resetchannels &ch1 + assert_equal [r aclcheck.publish.check.channel ch1 msg] 0 + catch {r aclcheck.publish.check.channel ch2 msg} e + set e + } {*DENIED CHANNEL*} + + test {test module check acl in rm_call} { + # rm call check for key permission (x: READ + WRITE) + assert_equal [r aclcheck.rm_call set x 5] OK + assert_equal [r aclcheck.rm_call set x 6 get] 5 + + # rm call check for key permission (y: only WRITE) + assert_equal [r aclcheck.rm_call set y 5] OK + assert_error {*NOPERM*} {r aclcheck.rm_call set y 5 get} + assert_error {*NOPERM*No permissions to access a key*} {r aclcheck.rm_call_with_errors set y 5 get} + + # rm call check for key permission (z: only READ) + assert_error {*NOPERM*} {r aclcheck.rm_call set z 5} + catch {r aclcheck.rm_call_with_errors set z 5} e + assert_match {*NOPERM*No permissions to access a key*} $e + assert_error {*NOPERM*} {r aclcheck.rm_call set z 6 get} + assert_error {*NOPERM*No permissions to access a key*} {r aclcheck.rm_call_with_errors set z 6 get} + + # verify that new log entry added + set entry [lindex [r ACL LOG] 0] + assert {[dict get $entry username] eq {default}} + assert {[dict get $entry context] eq {module}} + assert {[dict get $entry object] eq {z}} + assert {[dict get $entry reason] eq {key}} + + # rm call check for command permission + r acl setuser default -set + assert_error {*NOPERM*} {r aclcheck.rm_call set x 5} + assert_error {*NOPERM*has no permissions to run the 'set' command*} {r aclcheck.rm_call_with_errors set x 5} + + # verify that new log entry added + set entry [lindex [r ACL LOG] 0] + assert {[dict get $entry username] eq {default}} + assert {[dict get $entry context] eq {module}} + assert {[dict get $entry object] eq {set}} + assert {[dict get $entry reason] eq {command}} + } + + test {test blocking of Commands outside of OnLoad} { + assert_equal [r block.commands.outside.onload] OK + } + + test {test users to have access to module commands having acl categories} { + r acl SETUSER j1 on >password -@all +@WRITE + r acl SETUSER j2 on >password -@all +@READ + assert_equal [r acl DRYRUN j1 aclcheck.module.command.aclcategories.write] OK + assert_equal [r acl DRYRUN j2 aclcheck.module.command.aclcategories.write.function.read.category] OK + assert_equal [r acl DRYRUN j2 aclcheck.module.command.aclcategories.read.only.category] OK + } + + test {test existing users to have access to module commands loaded on runtime} { + assert_equal [r module unload aclcheck] OK + r acl SETUSER j3 on >password -@all +@WRITE + assert_equal [r module load $testmodule] OK + assert_equal [r acl DRYRUN j3 aclcheck.module.command.aclcategories.write] OK + } + + test {test existing users without permissions, do not have access to module commands loaded on runtime.} { + assert_equal [r module unload aclcheck] OK + r acl SETUSER j4 on >password -@all +@READ + r acl SETUSER j5 on >password -@all +@WRITE + assert_equal [r module load $testmodule] OK + catch {r acl DRYRUN j4 aclcheck.module.command.aclcategories.write} e + assert_equal {User j4 has no permissions to run the 'aclcheck.module.command.aclcategories.write' command} $e + catch {r acl DRYRUN j5 aclcheck.module.command.aclcategories.write.function.read.category} e + assert_equal {User j5 has no permissions to run the 'aclcheck.module.command.aclcategories.write.function.read.category' command} $e + } + + test {test users without permissions, do not have access to module commands.} { + r acl SETUSER j6 on >password -@all +@READ + catch {r acl DRYRUN j6 aclcheck.module.command.aclcategories.write} e + assert_equal {User j6 has no permissions to run the 'aclcheck.module.command.aclcategories.write' command} $e + r acl SETUSER j7 on >password -@all +@WRITE + catch {r acl DRYRUN j7 aclcheck.module.command.aclcategories.write.function.read.category} e + assert_equal {User j7 has no permissions to run the 'aclcheck.module.command.aclcategories.write.function.read.category' command} $e + } + + test "Unload the module - aclcheck" { + assert_equal {OK} [r module unload aclcheck] + } +} diff --git a/tests/unit/moduleapi/async_rm_call.tcl b/tests/unit/moduleapi/async_rm_call.tcl new file mode 100644 index 0000000..1bf12de --- /dev/null +++ b/tests/unit/moduleapi/async_rm_call.tcl @@ -0,0 +1,437 @@ +set testmodule [file normalize tests/modules/blockedclient.so] +set testmodule2 [file normalize tests/modules/postnotifications.so] +set testmodule3 [file normalize tests/modules/blockonkeys.so] + +start_server {tags {"modules"}} { + r module load $testmodule + + test {Locked GIL acquisition from async RM_Call} { + assert_equal {OK} [r do_rm_call_async acquire_gil] + } + + test "Blpop on async RM_Call fire and forget" { + assert_equal {Blocked} [r do_rm_call_fire_and_forget blpop l 0] + r lpush l a + assert_equal {0} [r llen l] + } + + test "Blpop on threaded async RM_Call" { + set rd [redis_deferring_client] + + $rd do_rm_call_async_on_thread blpop l 0 + wait_for_blocked_clients_count 1 + r lpush l a + assert_equal [$rd read] {l a} + wait_for_blocked_clients_count 0 + $rd close + } + + foreach cmd {do_rm_call_async do_rm_call_async_script_mode } { + + test "Blpop on async RM_Call using $cmd" { + set rd [redis_deferring_client] + + $rd $cmd blpop l 0 + wait_for_blocked_clients_count 1 + r lpush l a + assert_equal [$rd read] {l a} + wait_for_blocked_clients_count 0 + $rd close + } + + test "Brpop on async RM_Call using $cmd" { + set rd [redis_deferring_client] + + $rd $cmd brpop l 0 + wait_for_blocked_clients_count 1 + r lpush l a + assert_equal [$rd read] {l a} + wait_for_blocked_clients_count 0 + $rd close + } + + test "Brpoplpush on async RM_Call using $cmd" { + set rd [redis_deferring_client] + + $rd $cmd brpoplpush l1 l2 0 + wait_for_blocked_clients_count 1 + r lpush l1 a + assert_equal [$rd read] {a} + wait_for_blocked_clients_count 0 + $rd close + r lpop l2 + } {a} + + test "Blmove on async RM_Call using $cmd" { + set rd [redis_deferring_client] + + $rd $cmd blmove l1 l2 LEFT LEFT 0 + wait_for_blocked_clients_count 1 + r lpush l1 a + assert_equal [$rd read] {a} + wait_for_blocked_clients_count 0 + $rd close + r lpop l2 + } {a} + + test "Bzpopmin on async RM_Call using $cmd" { + set rd [redis_deferring_client] + + $rd $cmd bzpopmin s 0 + wait_for_blocked_clients_count 1 + r zadd s 10 foo + assert_equal [$rd read] {s foo 10} + wait_for_blocked_clients_count 0 + $rd close + } + + test "Bzpopmax on async RM_Call using $cmd" { + set rd [redis_deferring_client] + + $rd $cmd bzpopmax s 0 + wait_for_blocked_clients_count 1 + r zadd s 10 foo + assert_equal [$rd read] {s foo 10} + wait_for_blocked_clients_count 0 + $rd close + } + } + + test {Nested async RM_Call} { + set rd [redis_deferring_client] + + $rd do_rm_call_async do_rm_call_async do_rm_call_async do_rm_call_async blpop l 0 + wait_for_blocked_clients_count 1 + r lpush l a + assert_equal [$rd read] {l a} + wait_for_blocked_clients_count 0 + $rd close + } + + test {Test multiple async RM_Call waiting on the same event} { + set rd1 [redis_deferring_client] + set rd2 [redis_deferring_client] + + $rd1 do_rm_call_async do_rm_call_async do_rm_call_async do_rm_call_async blpop l 0 + $rd2 do_rm_call_async do_rm_call_async do_rm_call_async do_rm_call_async blpop l 0 + wait_for_blocked_clients_count 2 + r lpush l element element + assert_equal [$rd1 read] {l element} + assert_equal [$rd2 read] {l element} + wait_for_blocked_clients_count 0 + $rd1 close + $rd2 close + } + + test {async RM_Call calls RM_Call} { + assert_equal {PONG} [r do_rm_call_async do_rm_call ping] + } + + test {async RM_Call calls background RM_Call calls RM_Call} { + assert_equal {PONG} [r do_rm_call_async do_bg_rm_call do_rm_call ping] + } + + test {async RM_Call calls background RM_Call calls RM_Call calls async RM_Call} { + assert_equal {PONG} [r do_rm_call_async do_bg_rm_call do_rm_call do_rm_call_async ping] + } + + test {async RM_Call inside async RM_Call callback} { + set rd [redis_deferring_client] + $rd wait_and_do_rm_call blpop l 0 + wait_for_blocked_clients_count 1 + + start_server {} { + test "Connect a replica to the master instance" { + r slaveof [srv -1 host] [srv -1 port] + wait_for_condition 50 100 { + [s role] eq {slave} && + [string match {*master_link_status:up*} [r info replication]] + } else { + fail "Can't turn the instance into a replica" + } + } + + assert_equal {1} [r -1 lpush l a] + assert_equal [$rd read] {l a} + } + + wait_for_blocked_clients_count 0 + $rd close + } + + test {Become replica while having async RM_Call running} { + r flushall + set rd [redis_deferring_client] + $rd do_rm_call_async blpop l 0 + wait_for_blocked_clients_count 1 + + #become a replica of a not existing redis + r replicaof localhost 30000 + + catch {[$rd read]} e + assert_match {UNBLOCKED force unblock from blocking operation*} $e + wait_for_blocked_clients_count 0 + + r replicaof no one + + r lpush l 1 + # make sure the async rm_call was aborted + assert_equal [r llen l] {1} + $rd close + } + + test {Pipeline with blocking RM_Call} { + r flushall + set rd [redis_deferring_client] + set buf "" + append buf "do_rm_call_async blpop l 0\r\n" + append buf "ping\r\n" + $rd write $buf + $rd flush + wait_for_blocked_clients_count 1 + + # release the blocked client + r lpush l 1 + + assert_equal [$rd read] {l 1} + assert_equal [$rd read] {PONG} + + wait_for_blocked_clients_count 0 + $rd close + } + + test {blocking RM_Call abort} { + r flushall + set rd [redis_deferring_client] + + $rd client id + set client_id [$rd read] + + $rd do_rm_call_async blpop l 0 + wait_for_blocked_clients_count 1 + + r client kill ID $client_id + assert_error {*error reading reply*} {$rd read} + + wait_for_blocked_clients_count 0 + + r lpush l 1 + # make sure the async rm_call was aborted + assert_equal [r llen l] {1} + $rd close + } +} + +start_server {tags {"modules"}} { + r module load $testmodule + + test {Test basic replication stream on unblock handler} { + r flushall + set repl [attach_to_replication_stream] + + set rd [redis_deferring_client] + + $rd do_rm_call_async blpop l 0 + wait_for_blocked_clients_count 1 + r lpush l a + assert_equal [$rd read] {l a} + + assert_replication_stream $repl { + {select *} + {lpush l a} + {lpop l} + } + close_replication_stream $repl + + wait_for_blocked_clients_count 0 + $rd close + } + + test {Test unblock handler are executed as a unit} { + r flushall + set repl [attach_to_replication_stream] + + set rd [redis_deferring_client] + + $rd blpop_and_set_multiple_keys l x 1 y 2 + wait_for_blocked_clients_count 1 + r lpush l a + assert_equal [$rd read] {OK} + + assert_replication_stream $repl { + {select *} + {lpush l a} + {multi} + {lpop l} + {set x 1} + {set y 2} + {exec} + } + close_replication_stream $repl + + wait_for_blocked_clients_count 0 + $rd close + } + + test {Test no propagation of blocking command} { + r flushall + set repl [attach_to_replication_stream] + + set rd [redis_deferring_client] + + $rd do_rm_call_async_no_replicate blpop l 0 + wait_for_blocked_clients_count 1 + r lpush l a + assert_equal [$rd read] {l a} + + # make sure the lpop are not replicated + r set x 1 + + assert_replication_stream $repl { + {select *} + {lpush l a} + {set x 1} + } + close_replication_stream $repl + + wait_for_blocked_clients_count 0 + $rd close + } +} + +start_server {tags {"modules"}} { + r module load $testmodule + r module load $testmodule2 + + test {Test unblock handler are executed as a unit with key space notifications} { + r flushall + set repl [attach_to_replication_stream] + + set rd [redis_deferring_client] + + $rd blpop_and_set_multiple_keys l string_foo 1 string_bar 2 + wait_for_blocked_clients_count 1 + r lpush l a + assert_equal [$rd read] {OK} + + # Explanation of the first multi exec block: + # {lpop l} - pop the value by our blocking command 'blpop_and_set_multiple_keys' + # {set string_foo 1} - the action of our blocking command 'blpop_and_set_multiple_keys' + # {set string_bar 2} - the action of our blocking command 'blpop_and_set_multiple_keys' + # {incr string_changed{string_foo}} - post notification job that was registered when 'string_foo' changed + # {incr string_changed{string_bar}} - post notification job that was registered when 'string_bar' changed + # {incr string_total} - post notification job that was registered when string_changed{string_foo} changed + # {incr string_total} - post notification job that was registered when string_changed{string_bar} changed + assert_replication_stream $repl { + {select *} + {lpush l a} + {multi} + {lpop l} + {set string_foo 1} + {set string_bar 2} + {incr string_changed{string_foo}} + {incr string_changed{string_bar}} + {incr string_total} + {incr string_total} + {exec} + } + close_replication_stream $repl + + wait_for_blocked_clients_count 0 + $rd close + } + + test {Test unblock handler are executed as a unit with lazy expire} { + r flushall + r DEBUG SET-ACTIVE-EXPIRE 0 + set repl [attach_to_replication_stream] + + set rd [redis_deferring_client] + + $rd blpop_and_set_multiple_keys l string_foo 1 string_bar 2 + wait_for_blocked_clients_count 1 + r lpush l a + assert_equal [$rd read] {OK} + + # set expiration on string_foo + r pexpire string_foo 1 + after 10 + + # now the key should have been expired + $rd blpop_and_set_multiple_keys l string_foo 1 string_bar 2 + wait_for_blocked_clients_count 1 + r lpush l a + assert_equal [$rd read] {OK} + + # Explanation of the first multi exec block: + # {lpop l} - pop the value by our blocking command 'blpop_and_set_multiple_keys' + # {set string_foo 1} - the action of our blocking command 'blpop_and_set_multiple_keys' + # {set string_bar 2} - the action of our blocking command 'blpop_and_set_multiple_keys' + # {incr string_changed{string_foo}} - post notification job that was registered when 'string_foo' changed + # {incr string_changed{string_bar}} - post notification job that was registered when 'string_bar' changed + # {incr string_total} - post notification job that was registered when string_changed{string_foo} changed + # {incr string_total} - post notification job that was registered when string_changed{string_bar} changed + # + # Explanation of the second multi exec block: + # {lpop l} - pop the value by our blocking command 'blpop_and_set_multiple_keys' + # {del string_foo} - lazy expiration of string_foo when 'blpop_and_set_multiple_keys' tries to write to it. + # {set string_foo 1} - the action of our blocking command 'blpop_and_set_multiple_keys' + # {set string_bar 2} - the action of our blocking command 'blpop_and_set_multiple_keys' + # {incr expired} - the post notification job, registered after string_foo got expired + # {incr string_changed{string_foo}} - post notification job triggered when we set string_foo + # {incr string_changed{string_bar}} - post notification job triggered when we set string_bar + # {incr string_total} - post notification job triggered when we incr 'string_changed{string_foo}' + # {incr string_total} - post notification job triggered when we incr 'string_changed{string_bar}' + assert_replication_stream $repl { + {select *} + {lpush l a} + {multi} + {lpop l} + {set string_foo 1} + {set string_bar 2} + {incr string_changed{string_foo}} + {incr string_changed{string_bar}} + {incr string_total} + {incr string_total} + {exec} + {pexpireat string_foo *} + {lpush l a} + {multi} + {lpop l} + {del string_foo} + {set string_foo 1} + {set string_bar 2} + {incr expired} + {incr string_changed{string_foo}} + {incr string_changed{string_bar}} + {incr string_total} + {incr string_total} + {exec} + } + close_replication_stream $repl + r DEBUG SET-ACTIVE-EXPIRE 1 + + wait_for_blocked_clients_count 0 + $rd close + } +} + +start_server {tags {"modules"}} { + r module load $testmodule + r module load $testmodule3 + + test {Test unblock handler on module blocked on keys} { + set rd [redis_deferring_client] + + r fsl.push l 1 + $rd do_rm_call_async FSL.BPOPGT l 3 0 + wait_for_blocked_clients_count 1 + r fsl.push l 2 + r fsl.push l 3 + r fsl.push l 4 + assert_equal [$rd read] {4} + + wait_for_blocked_clients_count 0 + $rd close + } +} diff --git a/tests/unit/moduleapi/auth.tcl b/tests/unit/moduleapi/auth.tcl new file mode 100644 index 0000000..c7c2def --- /dev/null +++ b/tests/unit/moduleapi/auth.tcl @@ -0,0 +1,90 @@ +set testmodule [file normalize tests/modules/auth.so] + +start_server {tags {"modules"}} { + r module load $testmodule + + test {Modules can create a user that can be authenticated} { + # Make sure we start authenticated with default user + r auth default "" + assert_equal [r acl whoami] "default" + r auth.createmoduleuser + + set id [r auth.authmoduleuser] + assert_equal [r client id] $id + + # Verify returned id is the same as our current id and + # we are authenticated with the specified user + assert_equal [r acl whoami] "global" + } + + test {De-authenticating clients is tracked and kills clients} { + assert_equal [r auth.changecount] 0 + r auth.createmoduleuser + + # Catch the I/O exception that was thrown when Redis + # disconnected with us. + catch { [r ping] } e + assert_match {*I/O*} $e + + # Check that a user change was registered + assert_equal [r auth.changecount] 1 + } + + test {Modules can't authenticate with ACLs users that dont exist} { + catch { [r auth.authrealuser auth-module-test-fake] } e + assert_match {*Invalid user*} $e + } + + test {Modules can authenticate with ACL users} { + assert_equal [r acl whoami] "default" + + # Create user to auth into + r acl setuser auth-module-test on allkeys allcommands + + set id [r auth.authrealuser auth-module-test] + + # Verify returned id is the same as our current id and + # we are authenticated with the specified user + assert_equal [r client id] $id + assert_equal [r acl whoami] "auth-module-test" + } + + test {Client callback is called on user switch} { + assert_equal [r auth.changecount] 0 + + # Auth again and validate change count + r auth.authrealuser auth-module-test + assert_equal [r auth.changecount] 1 + + # Re-auth with the default user + r auth default "" + assert_equal [r auth.changecount] 1 + assert_equal [r acl whoami] "default" + + # Re-auth with the default user again, to + # verify the callback isn't fired again + r auth default "" + assert_equal [r auth.changecount] 0 + assert_equal [r acl whoami] "default" + } + + test {modules can redact arguments} { + r config set slowlog-log-slower-than 0 + r slowlog reset + r auth.redact 1 2 3 4 + r auth.redact 1 2 3 + r config set slowlog-log-slower-than -1 + set slowlog_resp [r slowlog get] + + # There will be 3 records, slowlog reset and the + # two auth redact calls. + assert_equal 3 [llength $slowlog_resp] + assert_equal {slowlog reset} [lindex [lindex $slowlog_resp 2] 3] + assert_equal {auth.redact 1 (redacted) 3 (redacted)} [lindex [lindex $slowlog_resp 1] 3] + assert_equal {auth.redact (redacted) 2 (redacted)} [lindex [lindex $slowlog_resp 0] 3] + } + + test "Unload the module - testacl" { + assert_equal {OK} [r module unload testacl] + } +} diff --git a/tests/unit/moduleapi/basics.tcl b/tests/unit/moduleapi/basics.tcl new file mode 100644 index 0000000..042e347 --- /dev/null +++ b/tests/unit/moduleapi/basics.tcl @@ -0,0 +1,46 @@ +set testmodule [file normalize tests/modules/basics.so] + +start_server {tags {"modules"}} { + r module load $testmodule + + test {test module api basics} { + r test.basics + } {ALL TESTS PASSED} + + test {test rm_call auto mode} { + r hello 2 + set reply [r test.rmcallautomode] + assert_equal [lindex $reply 0] f1 + assert_equal [lindex $reply 1] v1 + assert_equal [lindex $reply 2] f2 + assert_equal [lindex $reply 3] v2 + r hello 3 + set reply [r test.rmcallautomode] + assert_equal [dict get $reply f1] v1 + assert_equal [dict get $reply f2] v2 + } + + test {test get resp} { + foreach resp {3 2} { + if {[lsearch $::denytags "resp3"] >= 0} { + if {$resp == 3} {continue} + } elseif {$::force_resp3} { + if {$resp == 2} {continue} + } + r hello $resp + set reply [r test.getresp] + assert_equal $reply $resp + r hello 2 + } + } + + test "Unload the module - test" { + assert_equal {OK} [r module unload test] + } +} + +start_server {tags {"modules external:skip"} overrides {enable-module-command no}} { + test {module command disabled} { + assert_error "ERR *MODULE command not allowed*" {r module load $testmodule} + } +} diff --git a/tests/unit/moduleapi/blockedclient.tcl b/tests/unit/moduleapi/blockedclient.tcl new file mode 100644 index 0000000..9d475eb --- /dev/null +++ b/tests/unit/moduleapi/blockedclient.tcl @@ -0,0 +1,287 @@ +set testmodule [file normalize tests/modules/blockedclient.so] + +start_server {tags {"modules"}} { + r module load $testmodule + + test {Locked GIL acquisition} { + assert_match "OK" [r acquire_gil] + } + + test {Locked GIL acquisition during multi} { + r multi + r acquire_gil + assert_equal {{Blocked client is not supported inside multi}} [r exec] + } + + test {Locked GIL acquisition from RM_Call} { + assert_equal {Blocked client is not allowed} [r do_rm_call acquire_gil] + } + + test {Blocking command are not block the client on RM_Call} { + r lpush l test + assert_equal [r do_rm_call blpop l 0] {l test} + + r lpush l test + assert_equal [r do_rm_call brpop l 0] {l test} + + r lpush l1 test + assert_equal [r do_rm_call brpoplpush l1 l2 0] {test} + assert_equal [r do_rm_call brpop l2 0] {l2 test} + + r lpush l1 test + assert_equal [r do_rm_call blmove l1 l2 LEFT LEFT 0] {test} + assert_equal [r do_rm_call brpop l2 0] {l2 test} + + r ZADD zset1 0 a 1 b 2 c + assert_equal [r do_rm_call bzpopmin zset1 0] {zset1 a 0} + assert_equal [r do_rm_call bzpopmax zset1 0] {zset1 c 2} + + r xgroup create s g $ MKSTREAM + r xadd s * foo bar + assert {[r do_rm_call xread BLOCK 0 STREAMS s 0-0] ne {}} + assert {[r do_rm_call xreadgroup group g c BLOCK 0 STREAMS s >] ne {}} + + assert {[r do_rm_call blpop empty_list 0] eq {}} + assert {[r do_rm_call brpop empty_list 0] eq {}} + assert {[r do_rm_call brpoplpush empty_list1 empty_list2 0] eq {}} + assert {[r do_rm_call blmove empty_list1 empty_list2 LEFT LEFT 0] eq {}} + + assert {[r do_rm_call bzpopmin empty_zset 0] eq {}} + assert {[r do_rm_call bzpopmax empty_zset 0] eq {}} + + r xgroup create empty_stream g $ MKSTREAM + assert {[r do_rm_call xread BLOCK 0 STREAMS empty_stream $] eq {}} + assert {[r do_rm_call xreadgroup group g c BLOCK 0 STREAMS empty_stream >] eq {}} + + } + + test {Monitor disallow inside RM_Call} { + set e {} + catch { + r do_rm_call monitor + } e + set e + } {*ERR*DENY BLOCKING*} + + test {subscribe disallow inside RM_Call} { + set e {} + catch { + r do_rm_call subscribe x + } e + set e + } {*ERR*DENY BLOCKING*} + + test {RM_Call from blocked client} { + r hset hash foo bar + r do_bg_rm_call hgetall hash + } {foo bar} + + test {RM_Call from blocked client with script mode} { + r do_bg_rm_call_format S hset k foo bar + } {1} + + test {RM_Call from blocked client with oom mode} { + r config set maxmemory 1 + # will set server.pre_command_oom_state to 1 + assert_error {OOM command not allowed*} {r hset hash foo bar} + r config set maxmemory 0 + # now its should be OK to call OOM commands + r do_bg_rm_call_format M hset k1 foo bar + } {1} {needs:config-maxmemory} + + test {RESP version carries through to blocked client} { + for {set client_proto 2} {$client_proto <= 3} {incr client_proto} { + if {[lsearch $::denytags "resp3"] >= 0} { + if {$client_proto == 3} {continue} + } elseif {$::force_resp3} { + if {$client_proto == 2} {continue} + } + r hello $client_proto + r readraw 1 + set ret [r do_fake_bg_true] + if {$client_proto == 2} { + assert_equal $ret {:1} + } else { + assert_equal $ret "#t" + } + r readraw 0 + r hello 2 + } + } + +foreach call_type {nested normal} { + test "Busy module command - $call_type" { + set busy_time_limit 50 + set old_time_limit [lindex [r config get busy-reply-threshold] 1] + r config set busy-reply-threshold $busy_time_limit + set rd [redis_deferring_client] + + # run command that blocks until released + set start [clock clicks -milliseconds] + if {$call_type == "nested"} { + $rd do_rm_call slow_fg_command 0 + } else { + $rd slow_fg_command 0 + } + $rd flush + + # send another command after the blocked one, to make sure we don't attempt to process it + $rd ping + $rd flush + + # make sure we get BUSY error, and that we didn't get it too early + assert_error {*BUSY Slow module operation*} {r ping} + assert_morethan_equal [expr [clock clicks -milliseconds]-$start] $busy_time_limit + + # abort the blocking operation + r stop_slow_fg_command + wait_for_condition 50 100 { + [catch {r ping} e] == 0 + } else { + fail "Failed waiting for busy command to end" + } + assert_equal [$rd read] "1" + assert_equal [$rd read] "PONG" + + # run command that blocks for 200ms + set start [clock clicks -milliseconds] + if {$call_type == "nested"} { + $rd do_rm_call slow_fg_command 200000 + } else { + $rd slow_fg_command 200000 + } + $rd flush + after 10 ;# try to make sure redis started running the command before we proceed + + # make sure we didn't get BUSY error, it simply blocked till the command was done + r ping + assert_morethan_equal [expr [clock clicks -milliseconds]-$start] 200 + $rd read + + $rd close + r config set busy-reply-threshold $old_time_limit + } +} + + test {RM_Call from blocked client} { + set busy_time_limit 50 + set old_time_limit [lindex [r config get busy-reply-threshold] 1] + r config set busy-reply-threshold $busy_time_limit + + # trigger slow operation + r set_slow_bg_operation 1 + r hset hash foo bar + set rd [redis_deferring_client] + set start [clock clicks -milliseconds] + $rd do_bg_rm_call hgetall hash + + # send another command after the blocked one, to make sure we don't attempt to process it + $rd ping + $rd flush + + # wait till we know we're blocked inside the module + wait_for_condition 50 100 { + [r is_in_slow_bg_operation] eq 1 + } else { + fail "Failed waiting for slow operation to start" + } + + # make sure we get BUSY error, and that we didn't get here too early + assert_error {*BUSY Slow module operation*} {r ping} + assert_morethan [expr [clock clicks -milliseconds]-$start] $busy_time_limit + # abort the blocking operation + r set_slow_bg_operation 0 + + wait_for_condition 50 100 { + [r is_in_slow_bg_operation] eq 0 + } else { + fail "Failed waiting for slow operation to stop" + } + assert_equal [r ping] {PONG} + + r config set busy-reply-threshold $old_time_limit + assert_equal [$rd read] {foo bar} + assert_equal [$rd read] {PONG} + $rd close + } + + test {blocked client reaches client output buffer limit} { + r hset hash big [string repeat x 50000] + r hset hash bada [string repeat x 50000] + r hset hash boom [string repeat x 50000] + r config set client-output-buffer-limit {normal 100000 0 0} + r client setname myclient + catch {r do_bg_rm_call hgetall hash} e + assert_match "*I/O error*" $e + reconnect + set clients [r client list] + assert_no_match "*name=myclient*" $clients + } + + test {module client error stats} { + r config resetstat + + # simple module command that replies with string error + assert_error "ERR unknown command 'hgetalllll', with args beginning with:" {r do_rm_call hgetalllll} + assert_equal [errorrstat ERR r] {count=1} + + # simple module command that replies with string error + assert_error "ERR unknown subcommand 'bla'. Try CONFIG HELP." {r do_rm_call config bla} + assert_equal [errorrstat ERR r] {count=2} + + # module command that replies with string error from bg thread + assert_error "NULL reply returned" {r do_bg_rm_call hgetalllll} + assert_equal [errorrstat NULL r] {count=1} + + # module command that returns an arity error + r do_rm_call set x x + assert_error "ERR wrong number of arguments for 'do_rm_call' command" {r do_rm_call} + assert_equal [errorrstat ERR r] {count=3} + + # RM_Call that propagates an error + assert_error "WRONGTYPE*" {r do_rm_call hgetall x} + assert_equal [errorrstat WRONGTYPE r] {count=1} + assert_match {*calls=1,*,rejected_calls=0,failed_calls=1} [cmdrstat hgetall r] + + # RM_Call from bg thread that propagates an error + assert_error "WRONGTYPE*" {r do_bg_rm_call hgetall x} + assert_equal [errorrstat WRONGTYPE r] {count=2} + assert_match {*calls=2,*,rejected_calls=0,failed_calls=2} [cmdrstat hgetall r] + + assert_equal [s total_error_replies] 6 + assert_match {*calls=5,*,rejected_calls=0,failed_calls=4} [cmdrstat do_rm_call r] + assert_match {*calls=2,*,rejected_calls=0,failed_calls=2} [cmdrstat do_bg_rm_call r] + } + + set master [srv 0 client] + set master_host [srv 0 host] + set master_port [srv 0 port] + start_server [list overrides [list loadmodule "$testmodule"]] { + set replica [srv 0 client] + set replica_host [srv 0 host] + set replica_port [srv 0 port] + + # Start the replication process... + $replica replicaof $master_host $master_port + wait_for_sync $replica + + test {WAIT command on module blocked client} { + pause_process [srv 0 pid] + + $master do_bg_rm_call_format ! hset bk1 foo bar + + assert_equal [$master wait 1 1000] 0 + resume_process [srv 0 pid] + assert_equal [$master wait 1 1000] 1 + assert_equal [$replica hget bk1 foo] bar + } + } + + test {Unblock by timer} { + assert_match "OK" [r unblock_by_timer 100] + } + + test "Unload the module - blockedclient" { + assert_equal {OK} [r module unload blockedclient] + } +} diff --git a/tests/unit/moduleapi/blockonbackground.tcl b/tests/unit/moduleapi/blockonbackground.tcl new file mode 100644 index 0000000..fcd7f1d --- /dev/null +++ b/tests/unit/moduleapi/blockonbackground.tcl @@ -0,0 +1,126 @@ +set testmodule [file normalize tests/modules/blockonbackground.so] + +source tests/support/util.tcl + +proc latency_percentiles_usec {cmd} { + return [latencyrstat_percentiles $cmd r] +} + +start_server {tags {"modules"}} { + r module load $testmodule + + test { blocked clients time tracking - check blocked command that uses RedisModule_BlockedClientMeasureTimeStart() is tracking background time} { + r slowlog reset + r config set slowlog-log-slower-than 200000 + if {!$::no_latency} { + assert_equal [r slowlog len] 0 + } + r block.debug 0 10000 + if {!$::no_latency} { + assert_equal [r slowlog len] 0 + } + r config resetstat + r config set latency-tracking yes + r config set latency-tracking-info-percentiles "50.0" + r block.debug 200 10000 + if {!$::no_latency} { + assert_equal [r slowlog len] 1 + } + + set cmdstatline [cmdrstat block.debug r] + set latencystatline_debug [latency_percentiles_usec block.debug] + + regexp "calls=1,usec=(.*?),usec_per_call=(.*?),rejected_calls=0,failed_calls=0" $cmdstatline -> usec usec_per_call + regexp "p50=(.+\..+)" $latencystatline_debug -> p50 + assert {$usec >= 100000} + assert {$usec_per_call >= 100000} + assert {$p50 >= 100000} + } + + test { blocked clients time tracking - check blocked command that uses RedisModule_BlockedClientMeasureTimeStart() is tracking background time even in timeout } { + r slowlog reset + r config set slowlog-log-slower-than 200000 + if {!$::no_latency} { + assert_equal [r slowlog len] 0 + } + r block.debug 0 20000 + if {!$::no_latency} { + assert_equal [r slowlog len] 0 + } + r config resetstat + r block.debug 20000 500 + if {!$::no_latency} { + assert_equal [r slowlog len] 1 + } + + set cmdstatline [cmdrstat block.debug r] + + regexp "calls=1,usec=(.*?),usec_per_call=(.*?),rejected_calls=0,failed_calls=0" $cmdstatline usec usec_per_call + assert {$usec >= 250000} + assert {$usec_per_call >= 250000} + } + + test { blocked clients time tracking - check blocked command with multiple calls RedisModule_BlockedClientMeasureTimeStart() is tracking the total background time } { + r slowlog reset + r config set slowlog-log-slower-than 200000 + if {!$::no_latency} { + assert_equal [r slowlog len] 0 + } + r block.double_debug 0 + if {!$::no_latency} { + assert_equal [r slowlog len] 0 + } + r config resetstat + r block.double_debug 100 + if {!$::no_latency} { + assert_equal [r slowlog len] 1 + } + set cmdstatline [cmdrstat block.double_debug r] + + regexp "calls=1,usec=(.*?),usec_per_call=(.*?),rejected_calls=0,failed_calls=0" $cmdstatline usec usec_per_call + assert {$usec >= 60000} + assert {$usec_per_call >= 60000} + } + + test { blocked clients time tracking - check blocked command without calling RedisModule_BlockedClientMeasureTimeStart() is not reporting background time } { + r slowlog reset + r config set slowlog-log-slower-than 200000 + if {!$::no_latency} { + assert_equal [r slowlog len] 0 + } + r block.debug_no_track 200 1000 + # ensure slowlog is still empty + if {!$::no_latency} { + assert_equal [r slowlog len] 0 + } + } + + test "client unblock works only for modules with timeout support" { + set rd [redis_deferring_client] + $rd client id + set id [$rd read] + + # Block with a timeout function - may unblock + $rd block.block 20000 + wait_for_condition 50 100 { + [r block.is_blocked] == 1 + } else { + fail "Module did not block" + } + + assert_equal 1 [r client unblock $id] + assert_match {*Timed out*} [$rd read] + + # Block without a timeout function - cannot unblock + $rd block.block 0 + wait_for_condition 50 100 { + [r block.is_blocked] == 1 + } else { + fail "Module did not block" + } + + assert_equal 0 [r client unblock $id] + assert_equal "OK" [r block.release foobar] + assert_equal "foobar" [$rd read] + } +} diff --git a/tests/unit/moduleapi/blockonkeys.tcl b/tests/unit/moduleapi/blockonkeys.tcl new file mode 100644 index 0000000..66a94dc --- /dev/null +++ b/tests/unit/moduleapi/blockonkeys.tcl @@ -0,0 +1,366 @@ +set testmodule [file normalize tests/modules/blockonkeys.so] + +start_server {tags {"modules"}} { + r module load $testmodule + + test "Module client blocked on keys: Circular BPOPPUSH" { + set rd1 [redis_deferring_client] + set rd2 [redis_deferring_client] + + r del src dst + + $rd1 fsl.bpoppush src dst 0 + wait_for_blocked_clients_count 1 + + $rd2 fsl.bpoppush dst src 0 + wait_for_blocked_clients_count 2 + + r fsl.push src 42 + wait_for_blocked_clients_count 0 + + assert_equal {42} [r fsl.getall src] + assert_equal {} [r fsl.getall dst] + } + + test "Module client blocked on keys: Self-referential BPOPPUSH" { + set rd1 [redis_deferring_client] + + r del src + + $rd1 fsl.bpoppush src src 0 + wait_for_blocked_clients_count 1 + r fsl.push src 42 + + assert_equal {42} [r fsl.getall src] + } + + test "Module client blocked on keys: BPOPPUSH unblocked by timer" { + set rd1 [redis_deferring_client] + + r del src dst + + set repl [attach_to_replication_stream] + + $rd1 fsl.bpoppush src dst 0 + wait_for_blocked_clients_count 1 + + r fsl.pushtimer src 9000 10 + wait_for_blocked_clients_count 0 + + assert_equal {9000} [r fsl.getall dst] + assert_equal {} [r fsl.getall src] + + assert_replication_stream $repl { + {select *} + {fsl.push src 9000} + {fsl.bpoppush src dst 0} + } + + close_replication_stream $repl + } {} {needs:repl} + + test {Module client blocked on keys (no metadata): No block} { + r del k + r fsl.push k 33 + r fsl.push k 34 + r fsl.bpop k 0 + } {34} + + test {Module client blocked on keys (no metadata): Timeout} { + r del k + set rd [redis_deferring_client] + $rd fsl.bpop k 1 + assert_equal {Request timedout} [$rd read] + } + + test {Module client blocked on keys (no metadata): Blocked} { + r del k + set rd [redis_deferring_client] + $rd fsl.bpop k 0 + wait_for_blocked_clients_count 1 + r fsl.push k 34 + assert_equal {34} [$rd read] + } + + test {Module client blocked on keys (with metadata): No block} { + r del k + r fsl.push k 34 + r fsl.bpopgt k 30 0 + } {34} + + test {Module client blocked on keys (with metadata): Timeout} { + r del k + set rd [redis_deferring_client] + $rd client id + set cid [$rd read] + r fsl.push k 33 + $rd fsl.bpopgt k 35 1 + assert_equal {Request timedout} [$rd read] + r client kill id $cid ;# try to smoke-out client-related memory leak + } + + test {Module client blocked on keys (with metadata): Blocked, case 1} { + r del k + set rd [redis_deferring_client] + $rd client id + set cid [$rd read] + r fsl.push k 33 + $rd fsl.bpopgt k 33 0 + wait_for_blocked_clients_count 1 + r fsl.push k 34 + assert_equal {34} [$rd read] + r client kill id $cid ;# try to smoke-out client-related memory leak + } + + test {Module client blocked on keys (with metadata): Blocked, case 2} { + r del k + r fsl.push k 32 + set rd [redis_deferring_client] + $rd fsl.bpopgt k 35 0 + wait_for_blocked_clients_count 1 + r fsl.push k 33 + r fsl.push k 34 + r fsl.push k 35 + r fsl.push k 36 + assert_equal {36} [$rd read] + } + + test {Module client blocked on keys (with metadata): Blocked, DEL} { + r del k + r fsl.push k 32 + set rd [redis_deferring_client] + $rd fsl.bpopgt k 35 0 + wait_for_blocked_clients_count 1 + r del k + assert_error {*UNBLOCKED key no longer exists*} {$rd read} + } + + test {Module client blocked on keys (with metadata): Blocked, FLUSHALL} { + r del k + r fsl.push k 32 + set rd [redis_deferring_client] + $rd fsl.bpopgt k 35 0 + wait_for_blocked_clients_count 1 + r flushall + assert_error {*UNBLOCKED key no longer exists*} {$rd read} + } + + test {Module client blocked on keys (with metadata): Blocked, SWAPDB, no key} { + r select 9 + r del k + r fsl.push k 32 + set rd [redis_deferring_client] + $rd fsl.bpopgt k 35 0 + wait_for_blocked_clients_count 1 + r swapdb 0 9 + assert_error {*UNBLOCKED key no longer exists*} {$rd read} + } + + test {Module client blocked on keys (with metadata): Blocked, SWAPDB, key exists, case 1} { + ;# Key exists on other db, but wrong type + r flushall + r select 9 + r fsl.push k 32 + r select 0 + r lpush k 38 + r select 9 + set rd [redis_deferring_client] + $rd fsl.bpopgt k 35 0 + wait_for_blocked_clients_count 1 + r swapdb 0 9 + assert_error {*UNBLOCKED key no longer exists*} {$rd read} + r select 9 + } + + test {Module client blocked on keys (with metadata): Blocked, SWAPDB, key exists, case 2} { + ;# Key exists on other db, with the right type, but the value doesn't allow to unblock + r flushall + r select 9 + r fsl.push k 32 + r select 0 + r fsl.push k 34 + r select 9 + set rd [redis_deferring_client] + $rd fsl.bpopgt k 35 0 + wait_for_blocked_clients_count 1 + r swapdb 0 9 + assert_equal {1} [s 0 blocked_clients] + r fsl.push k 38 + assert_equal {38} [$rd read] + r select 9 + } + + test {Module client blocked on keys (with metadata): Blocked, SWAPDB, key exists, case 3} { + ;# Key exists on other db, with the right type, the value allows to unblock + r flushall + r select 9 + r fsl.push k 32 + r select 0 + r fsl.push k 38 + r select 9 + set rd [redis_deferring_client] + $rd fsl.bpopgt k 35 0 + wait_for_blocked_clients_count 1 + r swapdb 0 9 + assert_equal {38} [$rd read] + r select 9 + } + + test {Module client blocked on keys (with metadata): Blocked, CLIENT KILL} { + r del k + r fsl.push k 32 + set rd [redis_deferring_client] + $rd client id + set cid [$rd read] + $rd fsl.bpopgt k 35 0 + wait_for_blocked_clients_count 1 + r client kill id $cid ;# try to smoke-out client-related memory leak + } + + test {Module client blocked on keys (with metadata): Blocked, CLIENT UNBLOCK TIMEOUT} { + r del k + r fsl.push k 32 + set rd [redis_deferring_client] + $rd client id + set cid [$rd read] + $rd fsl.bpopgt k 35 0 + wait_for_blocked_clients_count 1 + r client unblock $cid timeout ;# try to smoke-out client-related memory leak + assert_equal {Request timedout} [$rd read] + } + + test {Module client blocked on keys (with metadata): Blocked, CLIENT UNBLOCK ERROR} { + r del k + r fsl.push k 32 + set rd [redis_deferring_client] + $rd client id + set cid [$rd read] + $rd fsl.bpopgt k 35 0 + wait_for_blocked_clients_count 1 + r client unblock $cid error ;# try to smoke-out client-related memory leak + assert_error "*unblocked*" {$rd read} + } + + test {Module client blocked on keys, no timeout CB, CLIENT UNBLOCK TIMEOUT} { + r del k + set rd [redis_deferring_client] + $rd client id + set cid [$rd read] + $rd fsl.bpop k 0 NO_TO_CB + wait_for_blocked_clients_count 1 + assert_equal [r client unblock $cid timeout] {0} + $rd close + } + + test {Module client blocked on keys, no timeout CB, CLIENT UNBLOCK ERROR} { + r del k + set rd [redis_deferring_client] + $rd client id + set cid [$rd read] + $rd fsl.bpop k 0 NO_TO_CB + wait_for_blocked_clients_count 1 + assert_equal [r client unblock $cid error] {0} + $rd close + } + + test {Module client re-blocked on keys after woke up on wrong type} { + r del k + set rd [redis_deferring_client] + $rd fsl.bpop k 0 + wait_for_blocked_clients_count 1 + r lpush k 12 + r lpush k 13 + r lpush k 14 + r del k + r fsl.push k 34 + assert_equal {34} [$rd read] + assert_equal {1} [r get fsl_wrong_type] ;# first lpush caused one wrong-type wake-up + } + + test {Module client blocked on keys woken up by LPUSH} { + r del k + set rd [redis_deferring_client] + $rd blockonkeys.popall k + wait_for_blocked_clients_count 1 + r lpush k 42 squirrel banana + assert_equal {banana squirrel 42} [$rd read] + $rd close + } + + test {Module client unblocks BLPOP} { + r del k + set rd [redis_deferring_client] + $rd blpop k 3 + wait_for_blocked_clients_count 1 + r blockonkeys.lpush k 42 + assert_equal {k 42} [$rd read] + $rd close + } + + test {Module unblocks module blocked on non-empty list} { + r del k + r lpush k aa + # Module client blocks to pop 5 elements from list + set rd [redis_deferring_client] + $rd blockonkeys.blpopn k 5 + wait_for_blocked_clients_count 1 + # Check that RM_SignalKeyAsReady() can wake up BLPOPN + r blockonkeys.lpush_unblock k bb cc ;# Not enough elements for BLPOPN + r lpush k dd ee ff ;# Doesn't unblock module + r blockonkeys.lpush_unblock k gg ;# Unblocks module + assert_equal {gg ff ee dd cc} [$rd read] + $rd close + } + + test {Module explicit unblock when blocked on keys} { + r del k + r set somekey someval + # Module client blocks to pop 5 elements from list + set rd [redis_deferring_client] + $rd blockonkeys.blpopn_or_unblock k 5 0 + wait_for_blocked_clients_count 1 + # will now cause the module to trigger pop but instead will unblock the client from the reply_callback + r lpush k dd + # we should still get unblocked as the command should not reprocess + wait_for_blocked_clients_count 0 + assert_equal {Action aborted} [$rd read] + $rd get somekey + assert_equal {someval} [$rd read] + $rd close + } + + set master [srv 0 client] + set master_host [srv 0 host] + set master_port [srv 0 port] + start_server [list overrides [list loadmodule "$testmodule"]] { + set replica [srv 0 client] + set replica_host [srv 0 host] + set replica_port [srv 0 port] + + # Start the replication process... + $replica replicaof $master_host $master_port + wait_for_sync $replica + + test {WAIT command on module blocked client on keys} { + set rd [redis_deferring_client -1] + $rd set x y + $rd read + + pause_process [srv 0 pid] + + $master del k + $rd fsl.bpop k 0 + wait_for_blocked_client -1 + $master fsl.push k 34 + $master fsl.push k 35 + assert_equal {34} [$rd read] + + assert_equal [$master wait 1 1000] 0 + resume_process [srv 0 pid] + assert_equal [$master wait 1 1000] 1 + $rd close + assert_equal {35} [$replica fsl.getall k] + } + } + +} diff --git a/tests/unit/moduleapi/cluster.tcl b/tests/unit/moduleapi/cluster.tcl new file mode 100644 index 0000000..8075083 --- /dev/null +++ b/tests/unit/moduleapi/cluster.tcl @@ -0,0 +1,222 @@ +# Primitive tests on cluster-enabled redis with modules + +source tests/support/cli.tcl + +# cluster creation is complicated with TLS, and the current tests don't really need that coverage +tags {tls:skip external:skip cluster modules} { + +set testmodule_nokey [file normalize tests/modules/blockonbackground.so] +set testmodule_blockedclient [file normalize tests/modules/blockedclient.so] +set testmodule [file normalize tests/modules/blockonkeys.so] + +set modules [list loadmodule $testmodule loadmodule $testmodule_nokey loadmodule $testmodule_blockedclient] +start_cluster 3 0 [list config_lines $modules] { + + set node1 [srv 0 client] + set node2 [srv -1 client] + set node3 [srv -2 client] + set node3_pid [srv -2 pid] + + test "Run blocking command (blocked on key) on cluster node3" { + # key9184688 is mapped to slot 10923 (first slot of node 3) + set node3_rd [redis_deferring_client -2] + $node3_rd fsl.bpop key9184688 0 + $node3_rd flush + wait_for_condition 50 100 { + [s -2 blocked_clients] eq {1} + } else { + fail "Client executing blocking command (blocked on key) not blocked" + } + } + + test "Run blocking command (no keys) on cluster node2" { + set node2_rd [redis_deferring_client -1] + $node2_rd block.block 0 + $node2_rd flush + + wait_for_condition 50 100 { + [s -1 blocked_clients] eq {1} + } else { + fail "Client executing blocking command (no keys) not blocked" + } + } + + + test "Perform a Resharding" { + exec src/redis-cli --cluster-yes --cluster reshard 127.0.0.1:[srv -2 port] \ + --cluster-to [$node1 cluster myid] \ + --cluster-from [$node3 cluster myid] \ + --cluster-slots 1 + } + + test "Verify command (no keys) is unaffected after resharding" { + # verify there are blocked clients on node2 + assert_equal [s -1 blocked_clients] {1} + + #release client + $node2 block.release 0 + } + + test "Verify command (blocked on key) got unblocked after resharding" { + # this (read) will wait for the node3 to realize the new topology + assert_error {*MOVED*} {$node3_rd read} + + # verify there are no blocked clients + assert_equal [s 0 blocked_clients] {0} + assert_equal [s -1 blocked_clients] {0} + assert_equal [s -2 blocked_clients] {0} + } + + test "Wait for cluster to be stable" { + wait_for_condition 1000 50 { + [catch {exec src/redis-cli --cluster check 127.0.0.1:[srv 0 port]}] == 0 && + [catch {exec src/redis-cli --cluster check 127.0.0.1:[srv -1 port]}] == 0 && + [catch {exec src/redis-cli --cluster check 127.0.0.1:[srv -2 port]}] == 0 && + [CI 0 cluster_state] eq {ok} && + [CI 1 cluster_state] eq {ok} && + [CI 2 cluster_state] eq {ok} + } else { + fail "Cluster doesn't stabilize" + } + } + + test "Sanity test push cmd after resharding" { + assert_error {*MOVED*} {$node3 fsl.push key9184688 1} + + set node1_rd [redis_deferring_client 0] + $node1_rd fsl.bpop key9184688 0 + $node1_rd flush + + wait_for_condition 50 100 { + [s 0 blocked_clients] eq {1} + } else { + puts "Client not blocked" + puts "read from blocked client: [$node1_rd read]" + fail "Client not blocked" + } + + $node1 fsl.push key9184688 2 + assert_equal {2} [$node1_rd read] + } + + $node1_rd close + $node2_rd close + $node3_rd close + + test "Run blocking command (blocked on key) again on cluster node1" { + $node1 del key9184688 + # key9184688 is mapped to slot 10923 which has been moved to node1 + set node1_rd [redis_deferring_client 0] + $node1_rd fsl.bpop key9184688 0 + $node1_rd flush + + wait_for_condition 50 100 { + [s 0 blocked_clients] eq {1} + } else { + fail "Client executing blocking command (blocked on key) again not blocked" + } + } + + test "Run blocking command (no keys) again on cluster node2" { + set node2_rd [redis_deferring_client -1] + + $node2_rd block.block 0 + $node2_rd flush + + wait_for_condition 50 100 { + [s -1 blocked_clients] eq {1} + } else { + fail "Client executing blocking command (no keys) again not blocked" + } + } + + test "Kill a cluster node and wait for fail state" { + # kill node3 in cluster + pause_process $node3_pid + + wait_for_condition 1000 50 { + [CI 0 cluster_state] eq {fail} && + [CI 1 cluster_state] eq {fail} + } else { + fail "Cluster doesn't fail" + } + } + + test "Verify command (blocked on key) got unblocked after cluster failure" { + assert_error {*CLUSTERDOWN*} {$node1_rd read} + } + + test "Verify command (no keys) got unblocked after cluster failure" { + assert_error {*CLUSTERDOWN*} {$node2_rd read} + + # verify there are no blocked clients + assert_equal [s 0 blocked_clients] {0} + assert_equal [s -1 blocked_clients] {0} + } + + test "Verify command RM_Call is rejected when cluster is down" { + assert_error "ERR Can not execute a command 'set' while the cluster is down" {$node1 do_rm_call set x 1} + } + + resume_process $node3_pid + $node1_rd close + $node2_rd close +} + +set modules [list loadmodule [file normalize tests/modules/keyspace_events.so]] +start_cluster 2 2 [list config_lines $modules] { + + set master1 [srv 0 client] + set master2 [srv -1 client] + set replica1 [srv -2 client] + set replica2 [srv -3 client] + + test "Verify keys deletion and notification effects happened on cluster slots change are replicated inside multi exec" { + $master2 set count_dels_{4oi} 1 + $master2 del count_dels_{4oi} + assert_equal 1 [$master2 keyspace.get_dels] + assert_equal 1 [$replica2 keyspace.get_dels] + $master2 set count_dels_{4oi} 1 + + set repl [attach_to_replication_stream_on_connection -3] + + $master1 cluster bumpepoch + $master1 cluster setslot 16382 node [$master1 cluster myid] + + wait_for_cluster_propagation + wait_for_condition 50 100 { + [$master2 keyspace.get_dels] eq 2 + } else { + fail "master did not delete the key" + } + wait_for_condition 50 100 { + [$replica2 keyspace.get_dels] eq 2 + } else { + fail "replica did not increase del counter" + } + + assert_replication_stream $repl { + {multi} + {del count_dels_{4oi}} + {keyspace.incr_dels} + {exec} + } + close_replication_stream $repl + } +} + +} + +set testmodule [file normalize tests/modules/basics.so] +set modules [list loadmodule $testmodule] +start_cluster 3 0 [list config_lines $modules] { + set node1 [srv 0 client] + set node2 [srv -1 client] + set node3 [srv -2 client] + + test "Verify RM_Call inside module load function on cluster mode" { + assert_equal {PONG} [$node1 PING] + assert_equal {PONG} [$node2 PING] + assert_equal {PONG} [$node3 PING] + } +} diff --git a/tests/unit/moduleapi/cmdintrospection.tcl b/tests/unit/moduleapi/cmdintrospection.tcl new file mode 100644 index 0000000..6ba69a1 --- /dev/null +++ b/tests/unit/moduleapi/cmdintrospection.tcl @@ -0,0 +1,50 @@ +set testmodule [file normalize tests/modules/cmdintrospection.so] + +start_server {tags {"modules"}} { + r module load $testmodule + + # cmdintrospection.xadd mimics XADD with regards to how + # what COMMAND exposes. There are two differences: + # + # 1. cmdintrospection.xadd (and all module commands) do not have ACL categories + # 2. cmdintrospection.xadd's `group` is "module" + # + # This tests verify that, apart from the above differences, the output of + # COMMAND INFO and COMMAND DOCS are identical for the two commands. + test "Module command introspection via COMMAND INFO" { + set redis_reply [lindex [r command info xadd] 0] + set module_reply [lindex [r command info cmdintrospection.xadd] 0] + for {set i 1} {$i < [llength $redis_reply]} {incr i} { + if {$i == 2} { + # Remove the "module" flag + set mylist [lindex $module_reply $i] + set idx [lsearch $mylist "module"] + set mylist [lreplace $mylist $idx $idx] + lset module_reply $i $mylist + } + if {$i == 6} { + # Skip ACL categories + continue + } + assert_equal [lindex $redis_reply $i] [lindex $module_reply $i] + } + } + + test "Module command introspection via COMMAND DOCS" { + set redis_reply [dict create {*}[lindex [r command docs xadd] 1]] + set module_reply [dict create {*}[lindex [r command docs cmdintrospection.xadd] 1]] + # Compare the maps. We need to pop "group" first. + dict unset redis_reply group + dict unset module_reply group + dict unset module_reply module + if {$::log_req_res} { + dict unset redis_reply reply_schema + } + + assert_equal $redis_reply $module_reply + } + + test "Unload the module - cmdintrospection" { + assert_equal {OK} [r module unload cmdintrospection] + } +} diff --git a/tests/unit/moduleapi/commandfilter.tcl b/tests/unit/moduleapi/commandfilter.tcl new file mode 100644 index 0000000..72b16ec --- /dev/null +++ b/tests/unit/moduleapi/commandfilter.tcl @@ -0,0 +1,175 @@ +set testmodule [file normalize tests/modules/commandfilter.so] + +start_server {tags {"modules"}} { + r module load $testmodule log-key 0 + + test {Retain a command filter argument} { + # Retain an argument now. Later we'll try to re-read it and make sure + # it is not corrupt and that valgrind does not complain. + r rpush some-list @retain my-retained-string + r commandfilter.retained + } {my-retained-string} + + test {Command Filter handles redirected commands} { + r set mykey @log + r lrange log-key 0 -1 + } "{set mykey @log}" + + test {Command Filter can call RedisModule_CommandFilterArgDelete} { + r rpush mylist elem1 @delme elem2 + r lrange mylist 0 -1 + } {elem1 elem2} + + test {Command Filter can call RedisModule_CommandFilterArgInsert} { + r del mylist + r rpush mylist elem1 @insertbefore elem2 @insertafter elem3 + r lrange mylist 0 -1 + } {elem1 --inserted-before-- @insertbefore elem2 @insertafter --inserted-after-- elem3} + + test {Command Filter can call RedisModule_CommandFilterArgReplace} { + r del mylist + r rpush mylist elem1 @replaceme elem2 + r lrange mylist 0 -1 + } {elem1 --replaced-- elem2} + + test {Command Filter applies on RM_Call() commands} { + r del log-key + r commandfilter.ping + r lrange log-key 0 -1 + } "{ping @log}" + + test {Command Filter applies on Lua redis.call()} { + r del log-key + r eval "redis.call('ping', '@log')" 0 + r lrange log-key 0 -1 + } "{ping @log}" + + test {Command Filter applies on Lua redis.call() that calls a module} { + r del log-key + r eval "redis.call('commandfilter.ping')" 0 + r lrange log-key 0 -1 + } "{ping @log}" + + test {Command Filter strings can be retained} { + r commandfilter.retained + } {my-retained-string} + + test {Command Filter is unregistered implicitly on module unload} { + r del log-key + r module unload commandfilter + r set mykey @log + r lrange log-key 0 -1 + } {} + + r module load $testmodule log-key 0 + + test {Command Filter unregister works as expected} { + # Validate reloading succeeded + r del log-key + r set mykey @log + assert_equal "{set mykey @log}" [r lrange log-key 0 -1] + + # Unregister + r commandfilter.unregister + r del log-key + + r set mykey @log + r lrange log-key 0 -1 + } {} + + r module unload commandfilter + r module load $testmodule log-key 1 + + test {Command Filter REDISMODULE_CMDFILTER_NOSELF works as expected} { + r set mykey @log + assert_equal "{set mykey @log}" [r lrange log-key 0 -1] + + r del log-key + r commandfilter.ping + assert_equal {} [r lrange log-key 0 -1] + + r eval "redis.call('commandfilter.ping')" 0 + assert_equal {} [r lrange log-key 0 -1] + } + + test "Unload the module - commandfilter" { + assert_equal {OK} [r module unload commandfilter] + } +} + +test {RM_CommandFilterArgInsert and script argv caching} { + # coverage for scripts calling commands that expand the argv array + # an attempt to add coverage for a possible bug in luaArgsToRedisArgv + # this test needs a fresh server so that lua_argv_size is 0. + # glibc realloc can return the same pointer even when the size changes + # still this test isn't able to trigger the issue, but we keep it anyway. + start_server {tags {"modules"}} { + r module load $testmodule log-key 0 + r del mylist + # command with 6 args + r eval {redis.call('rpush', KEYS[1], 'elem1', 'elem2', 'elem3', 'elem4')} 1 mylist + # command with 3 args that is changed to 4 + r eval {redis.call('rpush', KEYS[1], '@insertafter')} 1 mylist + # command with 6 args again + r eval {redis.call('rpush', KEYS[1], 'elem1', 'elem2', 'elem3', 'elem4')} 1 mylist + assert_equal [r lrange mylist 0 -1] {elem1 elem2 elem3 elem4 @insertafter --inserted-after-- elem1 elem2 elem3 elem4} + } +} + +# previously, there was a bug that command filters would be rerun (which would cause args to swap back) +# this test is meant to protect against that bug +test {Blocking Commands don't run through command filter when reprocessed} { + start_server {tags {"modules"}} { + r module load $testmodule log-key 0 + + r del list1{t} + r del list2{t} + + r lpush list2{t} a b c d e + + set rd [redis_deferring_client] + # we're asking to pop from the left, but the command filter swaps the two arguments, + # if it didn't swap it, we would end up with e d c b a 5 (5 being the left most of the following lpush) + # but since we swap the arguments, we end up with 1 e d c b a (1 being the right most of it). + # if the command filter would run again on unblock, they would be swapped back. + $rd blmove list1{t} list2{t} left right 0 + wait_for_blocked_client + r lpush list1{t} 1 2 3 4 5 + # validate that we moved the correct element with the swapped args + assert_equal [$rd read] 1 + # validate that we moved the correct elements to the correct side of the list + assert_equal [r lpop list2{t}] 1 + + $rd close + } +} + +test {Filtering based on client id} { + start_server {tags {"modules"}} { + r module load $testmodule log-key 0 + + set rr [redis_client] + set cid [$rr client id] + r unfilter_clientid $cid + + r rpush mylist elem1 @replaceme elem2 + assert_equal [r lrange mylist 0 -1] {elem1 --replaced-- elem2} + + r del mylist + + assert_equal [$rr rpush mylist elem1 @replaceme elem2] 3 + assert_equal [r lrange mylist 0 -1] {elem1 @replaceme elem2} + + $rr close + } +} + +start_server {} { + test {OnLoad failure will handle un-registration} { + catch {r module load $testmodule log-key 0 noload} + r set mykey @log + assert_equal [r lrange log-key 0 -1] {} + r rpush mylist elem1 @delme elem2 + assert_equal [r lrange mylist 0 -1] {elem1 @delme elem2} + } +} diff --git a/tests/unit/moduleapi/datatype.tcl b/tests/unit/moduleapi/datatype.tcl new file mode 100644 index 0000000..951c060 --- /dev/null +++ b/tests/unit/moduleapi/datatype.tcl @@ -0,0 +1,134 @@ +set testmodule [file normalize tests/modules/datatype.so] + +start_server {tags {"modules"}} { + r module load $testmodule + + test {DataType: Test module is sane, GET/SET work.} { + r datatype.set dtkey 100 stringval + assert {[r datatype.get dtkey] eq {100 stringval}} + } + + test {test blocking of datatype creation outside of OnLoad} { + assert_equal [r block.create.datatype.outside.onload] OK + } + + test {DataType: RM_SaveDataTypeToString(), RM_LoadDataTypeFromStringEncver() work} { + r datatype.set dtkey -1111 MyString + set encoded [r datatype.dump dtkey] + + assert {[r datatype.restore dtkeycopy $encoded 4] eq {4}} + assert {[r datatype.get dtkeycopy] eq {-1111 MyString}} + } + + test {DataType: Handle truncated RM_LoadDataTypeFromStringEncver()} { + r datatype.set dtkey -1111 MyString + set encoded [r datatype.dump dtkey] + set truncated [string range $encoded 0 end-1] + + catch {r datatype.restore dtkeycopy $truncated 4} e + set e + } {*Invalid*} + + test {DataType: ModuleTypeReplaceValue() happy path works} { + r datatype.set key-a 1 AAA + r datatype.set key-b 2 BBB + + assert {[r datatype.swap key-a key-b] eq {OK}} + assert {[r datatype.get key-a] eq {2 BBB}} + assert {[r datatype.get key-b] eq {1 AAA}} + } + + test {DataType: ModuleTypeReplaceValue() fails on non-module keys} { + r datatype.set key-a 1 AAA + r set key-b RedisString + + catch {r datatype.swap key-a key-b} e + set e + } {*ERR*} + + test {DataType: Copy command works for modules} { + # Test failed copies + r datatype.set answer-to-universe 42 AAA + catch {r copy answer-to-universe answer2} e + assert_match {*module key failed to copy*} $e + + # Our module's data type copy function copies the int value as-is + # but appends /<from-key>/<to-key> to the string value so we can + # track passed arguments. + r datatype.set sourcekey 1234 AAA + r copy sourcekey targetkey + r datatype.get targetkey + } {1234 AAA/sourcekey/targetkey} + + test {DataType: Slow Loading} { + r config set busy-reply-threshold 5000 ;# make sure we're using a high default + # trigger slow loading + r datatype.slow_loading 1 + set rd [redis_deferring_client] + set start [clock clicks -milliseconds] + $rd debug reload + + # wait till we know we're blocked inside the module + wait_for_condition 50 100 { + [r datatype.is_in_slow_loading] eq 1 + } else { + fail "Failed waiting for slow loading to start" + } + + # make sure we get LOADING error, and that we didn't get here late (not waiting for busy-reply-threshold) + assert_error {*LOADING*} {r ping} + assert_lessthan [expr [clock clicks -milliseconds]-$start] 2000 + + # abort the blocking operation + r datatype.slow_loading 0 + wait_for_condition 50 100 { + [s loading] eq {0} + } else { + fail "Failed waiting for loading to end" + } + $rd read + $rd close + } + + test {DataType: check the type name} { + r flushdb + r datatype.set foo 111 bar + assert_type test___dt foo + } + + test {SCAN module datatype} { + r flushdb + populate 1000 + r datatype.set foo 111 bar + set type [r type foo] + set cur 0 + set keys {} + while 1 { + set res [r scan $cur type $type] + set cur [lindex $res 0] + set k [lindex $res 1] + lappend keys {*}$k + if {$cur == 0} break + } + + assert_equal 1 [llength $keys] + } + + test {SCAN module datatype with case sensitive} { + r flushdb + populate 1000 + r datatype.set foo 111 bar + set type "tEsT___dT" + set cur 0 + set keys {} + while 1 { + set res [r scan $cur type $type] + set cur [lindex $res 0] + set k [lindex $res 1] + lappend keys {*}$k + if {$cur == 0} break + } + + assert_equal 1 [llength $keys] + } +} diff --git a/tests/unit/moduleapi/datatype2.tcl b/tests/unit/moduleapi/datatype2.tcl new file mode 100644 index 0000000..95acc9a --- /dev/null +++ b/tests/unit/moduleapi/datatype2.tcl @@ -0,0 +1,232 @@ +set testmodule [file normalize tests/modules/datatype2.so] + +start_server {tags {"modules"}} { + r module load $testmodule + + test "datatype2: test mem alloc and free" { + r flushall + + r select 0 + assert_equal 3 [r mem.alloc k1 3] + assert_equal 2 [r mem.alloc k2 2] + + r select 1 + assert_equal 1 [r mem.alloc k1 1] + assert_equal 5 [r mem.alloc k2 5] + + r select 0 + assert_equal 1 [r mem.free k1] + assert_equal 1 [r mem.free k2] + + r select 1 + assert_equal 1 [r mem.free k1] + assert_equal 1 [r mem.free k2] + } + + test "datatype2: test del and unlink" { + r flushall + + assert_equal 100 [r mem.alloc k1 100] + assert_equal 60 [r mem.alloc k2 60] + + assert_equal 1 [r unlink k1] + assert_equal 1 [r del k2] + } + + test "datatype2: test read and write" { + r flushall + + assert_equal 3 [r mem.alloc k1 3] + + set data datatype2 + assert_equal [string length $data] [r mem.write k1 0 $data] + assert_equal $data [r mem.read k1 0] + } + + test "datatype2: test rdb save and load" { + r flushall + + r select 0 + set data k1 + assert_equal 3 [r mem.alloc k1 3] + assert_equal [string length $data] [r mem.write k1 1 $data] + + set data k2 + assert_equal 2 [r mem.alloc k2 2] + assert_equal [string length $data] [r mem.write k2 0 $data] + + r select 1 + set data k3 + assert_equal 3 [r mem.alloc k3 3] + assert_equal [string length $data] [r mem.write k3 1 $data] + + set data k4 + assert_equal 2 [r mem.alloc k4 2] + assert_equal [string length $data] [r mem.write k4 0 $data] + + r bgsave + waitForBgsave r + r debug reload + + r select 0 + assert_equal k1 [r mem.read k1 1] + assert_equal k2 [r mem.read k2 0] + + r select 1 + assert_equal k3 [r mem.read k3 1] + assert_equal k4 [r mem.read k4 0] + } + + test "datatype2: test aof rewrite" { + r flushall + + r select 0 + set data k1 + assert_equal 3 [r mem.alloc k1 3] + assert_equal [string length $data] [r mem.write k1 1 $data] + + set data k2 + assert_equal 2 [r mem.alloc k2 2] + assert_equal [string length $data] [r mem.write k2 0 $data] + + r select 1 + set data k3 + assert_equal 3 [r mem.alloc k3 3] + assert_equal [string length $data] [r mem.write k3 1 $data] + + set data k4 + assert_equal 2 [r mem.alloc k4 2] + assert_equal [string length $data] [r mem.write k4 0 $data] + + r bgrewriteaof + waitForBgrewriteaof r + r debug loadaof + + r select 0 + assert_equal k1 [r mem.read k1 1] + assert_equal k2 [r mem.read k2 0] + + r select 1 + assert_equal k3 [r mem.read k3 1] + assert_equal k4 [r mem.read k4 0] + } + + test "datatype2: test copy" { + r flushall + + r select 0 + set data k1 + assert_equal 3 [r mem.alloc k1 3] + assert_equal [string length $data] [r mem.write k1 1 $data] + assert_equal $data [r mem.read k1 1] + + set data k2 + assert_equal 2 [r mem.alloc k2 2] + assert_equal [string length $data] [r mem.write k2 0 $data] + assert_equal $data [r mem.read k2 0] + + r select 1 + set data k3 + assert_equal 3 [r mem.alloc k3 3] + assert_equal [string length $data] [r mem.write k3 1 $data] + + set data k4 + assert_equal 2 [r mem.alloc k4 2] + assert_equal [string length $data] [r mem.write k4 0 $data] + + assert_equal {total 5 used 2} [r mem.usage 0] + assert_equal {total 5 used 2} [r mem.usage 1] + + r select 0 + assert_equal 1 [r copy k1 k3] + assert_equal k1 [r mem.read k3 1] + assert_equal {total 8 used 3} [r mem.usage 0] + assert_equal 1 [r copy k2 k1 db 1] + + r select 1 + assert_equal k2 [r mem.read k1 0] + assert_equal {total 8 used 3} [r mem.usage 0] + assert_equal {total 7 used 3} [r mem.usage 1] + } + + test "datatype2: test swapdb" { + r flushall + + r select 0 + set data k1 + assert_equal 5 [r mem.alloc k1 5] + assert_equal [string length $data] [r mem.write k1 1 $data] + assert_equal $data [r mem.read k1 1] + + set data k2 + assert_equal 4 [r mem.alloc k2 4] + assert_equal [string length $data] [r mem.write k2 0 $data] + assert_equal $data [r mem.read k2 0] + + r select 1 + set data k1 + assert_equal 3 [r mem.alloc k3 3] + assert_equal [string length $data] [r mem.write k3 1 $data] + + set data k2 + assert_equal 2 [r mem.alloc k4 2] + assert_equal [string length $data] [r mem.write k4 0 $data] + + assert_equal {total 9 used 2} [r mem.usage 0] + assert_equal {total 5 used 2} [r mem.usage 1] + + assert_equal OK [r swapdb 0 1] + assert_equal {total 9 used 2} [r mem.usage 1] + assert_equal {total 5 used 2} [r mem.usage 0] + } + + test "datatype2: test digest" { + r flushall + + r select 0 + set data k1 + assert_equal 3 [r mem.alloc k1 3] + assert_equal [string length $data] [r mem.write k1 1 $data] + assert_equal $data [r mem.read k1 1] + + set data k2 + assert_equal 2 [r mem.alloc k2 2] + assert_equal [string length $data] [r mem.write k2 0 $data] + assert_equal $data [r mem.read k2 0] + + r select 1 + set data k1 + assert_equal 3 [r mem.alloc k1 3] + assert_equal [string length $data] [r mem.write k1 1 $data] + assert_equal $data [r mem.read k1 1] + + set data k2 + assert_equal 2 [r mem.alloc k2 2] + assert_equal [string length $data] [r mem.write k2 0 $data] + assert_equal $data [r mem.read k2 0] + + r select 0 + set digest0 [debug_digest] + + r select 1 + set digest1 [debug_digest] + + assert_equal $digest0 $digest1 + } + + test "datatype2: test memusage" { + r flushall + + set data k1 + assert_equal 3 [r mem.alloc k1 3] + assert_equal [string length $data] [r mem.write k1 1 $data] + assert_equal $data [r mem.read k1 1] + + set data k2 + assert_equal 3 [r mem.alloc k2 3] + assert_equal [string length $data] [r mem.write k2 0 $data] + assert_equal $data [r mem.read k2 0] + + assert_equal [memory_usage k1] [memory_usage k2] + } +}
\ No newline at end of file diff --git a/tests/unit/moduleapi/defrag.tcl b/tests/unit/moduleapi/defrag.tcl new file mode 100644 index 0000000..b2e2396 --- /dev/null +++ b/tests/unit/moduleapi/defrag.tcl @@ -0,0 +1,46 @@ +set testmodule [file normalize tests/modules/defragtest.so] + +start_server {tags {"modules"} overrides {{save ""}}} { + r module load $testmodule 10000 + r config set hz 100 + r config set active-defrag-ignore-bytes 1 + r config set active-defrag-threshold-lower 0 + r config set active-defrag-cycle-min 99 + + # try to enable active defrag, it will fail if redis was compiled without it + catch {r config set activedefrag yes} e + if {[r config get activedefrag] eq "activedefrag yes"} { + + test {Module defrag: simple key defrag works} { + r frag.create key1 1 1000 0 + + after 2000 + set info [r info defragtest_stats] + assert {[getInfoProperty $info defragtest_datatype_attempts] > 0} + assert_equal 0 [getInfoProperty $info defragtest_datatype_resumes] + } + + test {Module defrag: late defrag with cursor works} { + r flushdb + r frag.resetstats + + # key can only be defragged in no less than 10 iterations + # due to maxstep + r frag.create key2 10000 100 1000 + + after 2000 + set info [r info defragtest_stats] + assert {[getInfoProperty $info defragtest_datatype_resumes] > 10} + assert_equal 0 [getInfoProperty $info defragtest_datatype_wrong_cursor] + } + + test {Module defrag: global defrag works} { + r flushdb + r frag.resetstats + + after 2000 + set info [r info defragtest_stats] + assert {[getInfoProperty $info defragtest_global_attempts] > 0} + } + } +} diff --git a/tests/unit/moduleapi/eventloop.tcl b/tests/unit/moduleapi/eventloop.tcl new file mode 100644 index 0000000..81e01ca --- /dev/null +++ b/tests/unit/moduleapi/eventloop.tcl @@ -0,0 +1,28 @@ +set testmodule [file normalize tests/modules/eventloop.so] + +start_server {tags {"modules"}} { + r module load $testmodule + + test "Module eventloop sendbytes" { + assert_match "OK" [r test.sendbytes 5000000] + assert_match "OK" [r test.sendbytes 2000000] + } + + test "Module eventloop iteration" { + set iteration [r test.iteration] + set next_iteration [r test.iteration] + assert {$next_iteration > $iteration} + } + + test "Module eventloop sanity" { + r test.sanity + } + + test "Module eventloop oneshot" { + r test.oneshot + } + + test "Unload the module - eventloop" { + assert_equal {OK} [r module unload eventloop] + } +} diff --git a/tests/unit/moduleapi/fork.tcl b/tests/unit/moduleapi/fork.tcl new file mode 100644 index 0000000..c89a6c5 --- /dev/null +++ b/tests/unit/moduleapi/fork.tcl @@ -0,0 +1,49 @@ +set testmodule [file normalize tests/modules/fork.so] + +proc count_log_message {pattern} { + set status [catch {exec grep -c $pattern < [srv 0 stdout]} result] + if {$status == 1} { + set result 0 + } + return $result +} + +start_server {tags {"modules"}} { + r module load $testmodule + + test {Module fork} { + # the argument to fork.create is the exitcode on termination + # the second argument to fork.create is passed to usleep + r fork.create 3 100000 ;# 100ms + wait_for_condition 20 100 { + [r fork.exitcode] != -1 + } else { + fail "fork didn't terminate" + } + r fork.exitcode + } {3} + + test {Module fork kill} { + # use a longer time to avoid the child exiting before being killed + r fork.create 3 100000000 ;# 100s + wait_for_condition 20 100 { + [count_log_message "fork child started"] == 2 + } else { + fail "fork didn't start" + } + + # module fork twice + assert_error {Fork failed} {r fork.create 0 1} + assert {[count_log_message "Can't fork for module: File exists"] eq "1"} + + r fork.kill + + assert {[count_log_message "Received SIGUSR1 in child"] eq "1"} + # check that it wasn't printed again (the print belong to the previous test) + assert {[count_log_message "fork child exiting"] eq "1"} + } + + test "Unload the module - fork" { + assert_equal {OK} [r module unload fork] + } +} diff --git a/tests/unit/moduleapi/getchannels.tcl b/tests/unit/moduleapi/getchannels.tcl new file mode 100644 index 0000000..e8f557d --- /dev/null +++ b/tests/unit/moduleapi/getchannels.tcl @@ -0,0 +1,40 @@ +set testmodule [file normalize tests/modules/getchannels.so] + +start_server {tags {"modules"}} { + r module load $testmodule + + # Channels are currently used to just validate ACLs, so test them here + r ACL setuser testuser +@all resetchannels &channel &pattern* + + test "module getchannels-api with literals - ACL" { + assert_equal "OK" [r ACL DRYRUN testuser getchannels.command subscribe literal channel subscribe literal pattern1] + assert_equal "OK" [r ACL DRYRUN testuser getchannels.command publish literal channel publish literal pattern1] + assert_equal "OK" [r ACL DRYRUN testuser getchannels.command unsubscribe literal channel unsubscribe literal pattern1] + + assert_equal "This user has no permissions to access the 'nopattern1' channel" [r ACL DRYRUN testuser getchannels.command subscribe literal channel subscribe literal nopattern1] + assert_equal "This user has no permissions to access the 'nopattern1' channel" [r ACL DRYRUN testuser getchannels.command publish literal channel subscribe literal nopattern1] + assert_equal "OK" [r ACL DRYRUN testuser getchannels.command unsubscribe literal channel unsubscribe literal nopattern1] + + assert_equal "This user has no permissions to access the 'otherchannel' channel" [r ACL DRYRUN testuser getchannels.command subscribe literal otherchannel subscribe literal pattern1] + assert_equal "This user has no permissions to access the 'otherchannel' channel" [r ACL DRYRUN testuser getchannels.command publish literal otherchannel subscribe literal pattern1] + assert_equal "OK" [r ACL DRYRUN testuser getchannels.command unsubscribe literal otherchannel unsubscribe literal pattern1] + } + + test "module getchannels-api with patterns - ACL" { + assert_equal "OK" [r ACL DRYRUN testuser getchannels.command subscribe pattern pattern*] + assert_equal "OK" [r ACL DRYRUN testuser getchannels.command publish pattern pattern*] + assert_equal "OK" [r ACL DRYRUN testuser getchannels.command unsubscribe pattern pattern*] + + assert_equal "This user has no permissions to access the 'pattern1' channel" [r ACL DRYRUN testuser getchannels.command subscribe pattern pattern1 subscribe pattern pattern*] + assert_equal "This user has no permissions to access the 'pattern1' channel" [r ACL DRYRUN testuser getchannels.command publish pattern pattern1 subscribe pattern pattern*] + assert_equal "OK" [r ACL DRYRUN testuser getchannels.command unsubscribe pattern pattern1 unsubscribe pattern pattern*] + + assert_equal "This user has no permissions to access the 'otherpattern*' channel" [r ACL DRYRUN testuser getchannels.command subscribe pattern otherpattern* subscribe pattern pattern*] + assert_equal "This user has no permissions to access the 'otherpattern*' channel" [r ACL DRYRUN testuser getchannels.command publish pattern otherpattern* subscribe pattern pattern*] + assert_equal "OK" [r ACL DRYRUN testuser getchannels.command unsubscribe pattern otherpattern* unsubscribe pattern pattern*] + } + + test "Unload the module - getchannels" { + assert_equal {OK} [r module unload getchannels] + } +} diff --git a/tests/unit/moduleapi/getkeys.tcl b/tests/unit/moduleapi/getkeys.tcl new file mode 100644 index 0000000..b84bb0f --- /dev/null +++ b/tests/unit/moduleapi/getkeys.tcl @@ -0,0 +1,80 @@ +set testmodule [file normalize tests/modules/getkeys.so] + +start_server {tags {"modules"}} { + r module load $testmodule + + test {COMMAND INFO correctly reports a movable keys module command} { + set info [lindex [r command info getkeys.command] 0] + + assert_equal {module movablekeys} [lindex $info 2] + assert_equal {0} [lindex $info 3] + assert_equal {0} [lindex $info 4] + assert_equal {0} [lindex $info 5] + } + + test {COMMAND GETKEYS correctly reports a movable keys module command} { + r command getkeys getkeys.command arg1 arg2 key key1 arg3 key key2 key key3 + } {key1 key2 key3} + + test {COMMAND GETKEYS correctly reports a movable keys module command using flags} { + r command getkeys getkeys.command_with_flags arg1 arg2 key key1 arg3 key key2 key key3 + } {key1 key2 key3} + + test {COMMAND GETKEYSANDFLAGS correctly reports a movable keys module command not using flags} { + r command getkeysandflags getkeys.command arg1 arg2 key key1 arg3 key key2 + } {{key1 {RW access update}} {key2 {RW access update}}} + + test {COMMAND GETKEYSANDFLAGS correctly reports a movable keys module command using flags} { + r command getkeysandflags getkeys.command_with_flags arg1 arg2 key key1 arg3 key key2 key key3 + } {{key1 {RO access}} {key2 {RO access}} {key3 {RO access}}} + + test {RM_GetCommandKeys on non-existing command} { + catch {r getkeys.introspect 0 non-command key1 key2} e + set _ $e + } {*ENOENT*} + + test {RM_GetCommandKeys on built-in fixed keys command} { + r getkeys.introspect 0 set key1 value1 + } {key1} + + test {RM_GetCommandKeys on built-in fixed keys command with flags} { + r getkeys.introspect 1 set key1 value1 + } {{key1 OW}} + + test {RM_GetCommandKeys on EVAL} { + r getkeys.introspect 0 eval "" 4 key1 key2 key3 key4 arg1 arg2 + } {key1 key2 key3 key4} + + test {RM_GetCommandKeys on a movable keys module command} { + r getkeys.introspect 0 getkeys.command arg1 arg2 key key1 arg3 key key2 key key3 + } {key1 key2 key3} + + test {RM_GetCommandKeys on a non-movable module command} { + r getkeys.introspect 0 getkeys.fixed arg1 key1 key2 key3 arg2 + } {key1 key2 key3} + + test {RM_GetCommandKeys with bad arity} { + catch {r getkeys.introspect 0 set key} e + set _ $e + } {*EINVAL*} + + # user that can only read from "read" keys, write to "write" keys, and read+write to "RW" keys + r ACL setuser testuser +@all %R~read* %W~write* %RW~rw* + + test "module getkeys-api - ACL" { + # legacy triple didn't provide flags, so they require both read and write + assert_equal "OK" [r ACL DRYRUN testuser getkeys.command key rw] + assert_match {*has no permissions to access the 'read' key*} [r ACL DRYRUN testuser getkeys.command key read] + assert_match {*has no permissions to access the 'write' key*} [r ACL DRYRUN testuser getkeys.command key write] + } + + test "module getkeys-api with flags - ACL" { + assert_equal "OK" [r ACL DRYRUN testuser getkeys.command_with_flags key rw] + assert_equal "OK" [r ACL DRYRUN testuser getkeys.command_with_flags key read] + assert_match {*has no permissions to access the 'write' key*} [r ACL DRYRUN testuser getkeys.command_with_flags key write] + } + + test "Unload the module - getkeys" { + assert_equal {OK} [r module unload getkeys] + } +} diff --git a/tests/unit/moduleapi/hash.tcl b/tests/unit/moduleapi/hash.tcl new file mode 100644 index 0000000..116b1c5 --- /dev/null +++ b/tests/unit/moduleapi/hash.tcl @@ -0,0 +1,27 @@ +set testmodule [file normalize tests/modules/hash.so] + +start_server {tags {"modules"}} { + r module load $testmodule + + test {Module hash set} { + r set k mystring + assert_error "WRONGTYPE*" {r hash.set k "" hello world} + r del k + # "" = count updates and deletes of existing fields only + assert_equal 0 [r hash.set k "" squirrel yes] + # "a" = COUNT_ALL = count inserted, modified and deleted fields + assert_equal 2 [r hash.set k "a" banana no sushi whynot] + # "n" = NX = only add fields not already existing in the hash + # "x" = XX = only replace the value for existing fields + assert_equal 0 [r hash.set k "n" squirrel hoho what nothing] + assert_equal 1 [r hash.set k "na" squirrel hoho something nice] + assert_equal 0 [r hash.set k "xa" new stuff not inserted] + assert_equal 1 [r hash.set k "x" squirrel ofcourse] + assert_equal 1 [r hash.set k "" sushi :delete: none :delete:] + r hgetall k + } {squirrel ofcourse banana no what nothing something nice} + + test "Unload the module - hash" { + assert_equal {OK} [r module unload hash] + } +} diff --git a/tests/unit/moduleapi/hooks.tcl b/tests/unit/moduleapi/hooks.tcl new file mode 100644 index 0000000..94b0f6f --- /dev/null +++ b/tests/unit/moduleapi/hooks.tcl @@ -0,0 +1,321 @@ +set testmodule [file normalize tests/modules/hooks.so] + +tags "modules" { + start_server [list overrides [list loadmodule "$testmodule" appendonly yes]] { + test {Test module aof save on server start from empty} { + assert {[r hooks.event_count persistence-syncaof-start] == 1} + } + + test {Test clients connection / disconnection hooks} { + for {set j 0} {$j < 2} {incr j} { + set rd1 [redis_deferring_client] + $rd1 close + } + assert {[r hooks.event_count client-connected] > 1} + assert {[r hooks.event_count client-disconnected] > 1} + } + + test {Test module client change event for blocked client} { + set rd [redis_deferring_client] + # select db other than 0 + $rd select 1 + # block on key + $rd brpop foo 0 + # kill blocked client + r client kill skipme yes + # assert server is still up + assert_equal [r ping] PONG + $rd close + } + + test {Test module cron hook} { + after 100 + assert {[r hooks.event_count cron-loop] > 0} + set hz [r hooks.event_last cron-loop] + assert_equal $hz 10 + } + + test {Test module loaded / unloaded hooks} { + set othermodule [file normalize tests/modules/infotest.so] + r module load $othermodule + r module unload infotest + assert_equal [r hooks.event_last module-loaded] "infotest" + assert_equal [r hooks.event_last module-unloaded] "infotest" + } + + test {Test module aofrw hook} { + r debug populate 1000 foo 10000 ;# 10mb worth of data + r config set rdbcompression no ;# rdb progress is only checked once in 2mb + r BGREWRITEAOF + waitForBgrewriteaof r + assert_equal [string match {*module-event-persistence-aof-start*} [exec tail -20 < [srv 0 stdout]]] 1 + assert_equal [string match {*module-event-persistence-end*} [exec tail -20 < [srv 0 stdout]]] 1 + } + + test {Test module aof load and rdb/aof progress hooks} { + # create some aof tail (progress is checked only once in 1000 commands) + for {set j 0} {$j < 4000} {incr j} { + r set "bar$j" x + } + # set some configs that will cause many loading progress events during aof loading + r config set key-load-delay 500 + r config set dynamic-hz no + r config set hz 500 + r DEBUG LOADAOF + assert_equal [r hooks.event_last loading-aof-start] 0 + assert_equal [r hooks.event_last loading-end] 0 + assert {[r hooks.event_count loading-rdb-start] == 0} + assert_lessthan 2 [r hooks.event_count loading-progress-rdb] ;# comes from the preamble section + assert_lessthan 2 [r hooks.event_count loading-progress-aof] + if {$::verbose} { + puts "rdb progress events [r hooks.event_count loading-progress-rdb]" + puts "aof progress events [r hooks.event_count loading-progress-aof]" + } + } + # undo configs before next test + r config set dynamic-hz yes + r config set key-load-delay 0 + + test {Test module rdb save hook} { + # debug reload does: save, flush, load: + assert {[r hooks.event_count persistence-syncrdb-start] == 0} + assert {[r hooks.event_count loading-rdb-start] == 0} + r debug reload + assert {[r hooks.event_count persistence-syncrdb-start] == 1} + assert {[r hooks.event_count loading-rdb-start] == 1} + } + + test {Test key unlink hook} { + r set testkey1 hello + r del testkey1 + assert {[r hooks.event_count key-info-testkey1] == 1} + assert_equal [r hooks.event_last key-info-testkey1] testkey1 + r lpush testkey1 hello + r lpop testkey1 + assert {[r hooks.event_count key-info-testkey1] == 2} + assert_equal [r hooks.event_last key-info-testkey1] testkey1 + r set testkey2 world + r unlink testkey2 + assert {[r hooks.event_count key-info-testkey2] == 1} + assert_equal [r hooks.event_last key-info-testkey2] testkey2 + } + + test {Test removed key event} { + r set str abcd + r set str abcde + # For String Type value is returned + assert_equal {abcd overwritten} [r hooks.is_key_removed str] + assert_equal -1 [r hooks.pexpireat str] + + r del str + assert_equal {abcde deleted} [r hooks.is_key_removed str] + assert_equal -1 [r hooks.pexpireat str] + + # test int encoded string + r set intstr 12345678 + # incr doesn't fire event + r incr intstr + catch {[r hooks.is_key_removed intstr]} output + assert_match {ERR * removed} $output + r del intstr + assert_equal {12345679 deleted} [r hooks.is_key_removed intstr] + + catch {[r hooks.is_key_removed not-exists]} output + assert_match {ERR * removed} $output + + r hset hash f v + r hdel hash f + assert_equal {0 deleted} [r hooks.is_key_removed hash] + + r hset hash f v a b + r del hash + assert_equal {2 deleted} [r hooks.is_key_removed hash] + + r lpush list 1 + r lpop list + assert_equal {0 deleted} [r hooks.is_key_removed list] + + r lpush list 1 2 3 + r del list + assert_equal {3 deleted} [r hooks.is_key_removed list] + + r sadd set 1 + r spop set + assert_equal {0 deleted} [r hooks.is_key_removed set] + + r sadd set 1 2 3 4 + r del set + assert_equal {4 deleted} [r hooks.is_key_removed set] + + r zadd zset 1 f + r zpopmin zset + assert_equal {0 deleted} [r hooks.is_key_removed zset] + + r zadd zset 1 f 2 d + r del zset + assert_equal {2 deleted} [r hooks.is_key_removed zset] + + r xadd stream 1-1 f v + r xdel stream 1-1 + # Stream does not delete object when del entry + catch {[r hooks.is_key_removed stream]} output + assert_match {ERR * removed} $output + r del stream + assert_equal {0 deleted} [r hooks.is_key_removed stream] + + r xadd stream 2-1 f v + r del stream + assert_equal {1 deleted} [r hooks.is_key_removed stream] + + # delete key because of active expire + set size [r dbsize] + r set active-expire abcd px 1 + #ensure active expire + wait_for_condition 50 100 { + [r dbsize] == $size + } else { + fail "Active expire not trigger" + } + assert_equal {abcd expired} [r hooks.is_key_removed active-expire] + # current time is greater than pexpireat + set now [r time] + set mill [expr ([lindex $now 0]*1000)+([lindex $now 1]/1000)] + assert {$mill >= [r hooks.pexpireat active-expire]} + + # delete key because of lazy expire + r debug set-active-expire 0 + r set lazy-expire abcd px 1 + after 10 + r get lazy-expire + assert_equal {abcd expired} [r hooks.is_key_removed lazy-expire] + set now [r time] + set mill [expr ([lindex $now 0]*1000)+([lindex $now 1]/1000)] + assert {$mill >= [r hooks.pexpireat lazy-expire]} + r debug set-active-expire 1 + + # delete key not yet expired + set now [r time] + set expireat [expr ([lindex $now 0]*1000)+([lindex $now 1]/1000)+1000000] + r set not-expire abcd pxat $expireat + r del not-expire + assert_equal {abcd deleted} [r hooks.is_key_removed not-expire] + assert_equal $expireat [r hooks.pexpireat not-expire] + + # Test key evict + set used [expr {[s used_memory] - [s mem_not_counted_for_evict]}] + set limit [expr {$used+100*1024}] + set old_policy [lindex [r config get maxmemory-policy] 1] + r config set maxmemory $limit + # We set policy volatile-random, so only keys with ttl will be evicted + r config set maxmemory-policy volatile-random + r setex volatile-key 10000 x + # We use SETBIT here, so we can set a big key and get the used_memory + # bigger than maxmemory. Next command will evict volatile keys. We + # can't use SET, as SET uses big input buffer, so it will fail. + r setbit big-key 1600000 0 ;# this will consume 200kb + r getbit big-key 0 + assert_equal {x evicted} [r hooks.is_key_removed volatile-key] + r config set maxmemory-policy $old_policy + r config set maxmemory 0 + } {OK} {needs:debug} + + test {Test flushdb hooks} { + r flushdb + assert_equal [r hooks.event_last flush-start] 9 + assert_equal [r hooks.event_last flush-end] 9 + r flushall + assert_equal [r hooks.event_last flush-start] -1 + assert_equal [r hooks.event_last flush-end] -1 + } + + # replication related tests + set master [srv 0 client] + set master_host [srv 0 host] + set master_port [srv 0 port] + start_server {} { + r module load $testmodule + set replica [srv 0 client] + set replica_host [srv 0 host] + set replica_port [srv 0 port] + $replica replicaof $master_host $master_port + + wait_replica_online $master + + test {Test master link up hook} { + assert_equal [r hooks.event_count masterlink-up] 1 + assert_equal [r hooks.event_count masterlink-down] 0 + } + + test {Test role-replica hook} { + assert_equal [r hooks.event_count role-replica] 1 + assert_equal [r hooks.event_count role-master] 0 + assert_equal [r hooks.event_last role-replica] [s 0 master_host] + } + + test {Test replica-online hook} { + assert_equal [r -1 hooks.event_count replica-online] 1 + assert_equal [r -1 hooks.event_count replica-offline] 0 + } + + test {Test master link down hook} { + r client kill type master + assert_equal [r hooks.event_count masterlink-down] 1 + + wait_for_condition 50 100 { + [string match {*master_link_status:up*} [r info replication]] + } else { + fail "Replica didn't reconnect" + } + + assert_equal [r hooks.event_count masterlink-down] 1 + assert_equal [r hooks.event_count masterlink-up] 2 + } + + wait_for_condition 50 10 { + [string match {*master_link_status:up*} [r info replication]] + } else { + fail "Can't turn the instance into a replica" + } + + $replica replicaof no one + + test {Test role-master hook} { + assert_equal [r hooks.event_count role-replica] 1 + assert_equal [r hooks.event_count role-master] 1 + assert_equal [r hooks.event_last role-master] {} + } + + test {Test replica-offline hook} { + assert_equal [r -1 hooks.event_count replica-online] 2 + assert_equal [r -1 hooks.event_count replica-offline] 2 + } + # get the replica stdout, to be used by the next test + set replica_stdout [srv 0 stdout] + } + + test {Test swapdb hooks} { + r swapdb 0 10 + assert_equal [r hooks.event_last swapdb-first] 0 + assert_equal [r hooks.event_last swapdb-second] 10 + } + + test {Test configchange hooks} { + r config set rdbcompression no + assert_equal [r hooks.event_last config-change-count] 1 + assert_equal [r hooks.event_last config-change-first] rdbcompression + } + + # look into the log file of the server that just exited + test {Test shutdown hook} { + assert_equal [string match {*module-event-shutdown*} [exec tail -5 < $replica_stdout]] 1 + } + } + + start_server {} { + test {OnLoad failure will handle un-registration} { + catch {r module load $testmodule noload} + r flushall + r ping + } + } +} diff --git a/tests/unit/moduleapi/infotest.tcl b/tests/unit/moduleapi/infotest.tcl new file mode 100644 index 0000000..ccd8c4e --- /dev/null +++ b/tests/unit/moduleapi/infotest.tcl @@ -0,0 +1,131 @@ +set testmodule [file normalize tests/modules/infotest.so] + +# Return value for INFO property +proc field {info property} { + if {[regexp "\r\n$property:(.*?)\r\n" $info _ value]} { + set _ $value + } +} + +start_server {tags {"modules"}} { + r module load $testmodule log-key 0 + + test {module reading info} { + # check string, integer and float fields + assert_equal [r info.gets replication role] "master" + assert_equal [r info.getc replication role] "master" + assert_equal [r info.geti stats expired_keys] 0 + assert_equal [r info.getd stats expired_stale_perc] 0 + + # check signed and unsigned + assert_equal [r info.geti infotest infotest_global] -2 + assert_equal [r info.getu infotest infotest_uglobal] -2 + + # the above are always 0, try module info that is non-zero + assert_equal [r info.geti infotest_italian infotest_due] 2 + set tre [r info.getd infotest_italian infotest_tre] + assert {$tre > 3.2 && $tre < 3.4 } + + # search using the wrong section + catch { [r info.gets badname redis_version] } e + assert_match {*not found*} $e + + # check that section filter works + assert { [string match "*usec_per_call*" [r info.gets all cmdstat_info.gets] ] } + catch { [r info.gets default cmdstat_info.gets] ] } e + assert_match {*not found*} $e + } + + test {module info all} { + set info [r info all] + # info all does not contain modules + assert { ![string match "*Spanish*" $info] } + assert { ![string match "*infotest_*" $info] } + assert { [string match "*used_memory*" $info] } + } + + test {module info all infotest} { + set info [r info all infotest] + # info all infotest should contain both ALL and the module information + assert { [string match "*Spanish*" $info] } + assert { [string match "*infotest_*" $info] } + assert { [string match "*used_memory*" $info] } + } + + test {module info everything} { + set info [r info everything] + # info everything contains all default sections, but not ones for crash report + assert { [string match "*infotest_global*" $info] } + assert { [string match "*Spanish*" $info] } + assert { [string match "*Italian*" $info] } + assert { [string match "*used_memory*" $info] } + assert { ![string match "*Klingon*" $info] } + field $info infotest_dos + } {2} + + test {module info modules} { + set info [r info modules] + # info all does not contain modules + assert { [string match "*Spanish*" $info] } + assert { [string match "*infotest_global*" $info] } + assert { ![string match "*used_memory*" $info] } + } + + test {module info one module} { + set info [r info INFOtest] ;# test case insensitive compare + # info all does not contain modules + assert { [string match "*Spanish*" $info] } + assert { ![string match "*used_memory*" $info] } + field $info infotest_global + } {-2} + + test {module info one section} { + set info [r info INFOtest_SpanisH] ;# test case insensitive compare + assert { ![string match "*used_memory*" $info] } + assert { ![string match "*Italian*" $info] } + assert { ![string match "*infotest_global*" $info] } + field $info infotest_uno + } {one} + + test {module info dict} { + set info [r info infotest_keyspace] + set keyspace [field $info infotest_db0] + set keys [scan [regexp -inline {keys\=([\d]*)} $keyspace] keys=%d] + } {3} + + test {module info unsafe fields} { + set info [r info infotest_unsafe] + assert_match {*infotest_unsafe_field:value=1*} $info + } + + test {module info multiply sections without all, everything, default keywords} { + set info [r info replication INFOTEST] + assert { [string match "*Spanish*" $info] } + assert { ![string match "*used_memory*" $info] } + assert { [string match "*repl_offset*" $info] } + } + + test {module info multiply sections with all keyword and modules} { + set info [r info all modules] + assert { [string match "*cluster*" $info] } + assert { [string match "*cmdstat_info*" $info] } + assert { [string match "*infotest_global*" $info] } + } + + test {module info multiply sections with everything keyword} { + set info [r info replication everything cpu] + assert { [string match "*client_recent*" $info] } + assert { [string match "*cmdstat_info*" $info] } + assert { [string match "*Italian*" $info] } + # check that we didn't get the same info twice + assert { ![string match "*used_cpu_user_children*used_cpu_user_children*" $info] } + assert { ![string match "*Italian*Italian*" $info] } + field $info infotest_dos + } {2} + + test "Unload the module - infotest" { + assert_equal {OK} [r module unload infotest] + } + + # TODO: test crash report. +} diff --git a/tests/unit/moduleapi/infra.tcl b/tests/unit/moduleapi/infra.tcl new file mode 100644 index 0000000..1140e5a --- /dev/null +++ b/tests/unit/moduleapi/infra.tcl @@ -0,0 +1,25 @@ +set testmodule [file normalize tests/modules/infotest.so] + +test {modules config rewrite} { + + start_server {tags {"modules"}} { + r module load $testmodule + + set modules [lmap x [r module list] {dict get $x name}] + assert_not_equal [lsearch $modules infotest] -1 + + r config rewrite + restart_server 0 true false + + set modules [lmap x [r module list] {dict get $x name}] + assert_not_equal [lsearch $modules infotest] -1 + + assert_equal {OK} [r module unload infotest] + + r config rewrite + restart_server 0 true false + + set modules [lmap x [r module list] {dict get $x name}] + assert_equal [lsearch $modules infotest] -1 + } +} diff --git a/tests/unit/moduleapi/keyspace_events.tcl b/tests/unit/moduleapi/keyspace_events.tcl new file mode 100644 index 0000000..1323b12 --- /dev/null +++ b/tests/unit/moduleapi/keyspace_events.tcl @@ -0,0 +1,118 @@ +set testmodule [file normalize tests/modules/keyspace_events.so] + +tags "modules" { + start_server [list overrides [list loadmodule "$testmodule"]] { + + test {Test loaded key space event} { + r set x 1 + r hset y f v + r lpush z 1 2 3 + r sadd p 1 2 3 + r zadd t 1 f1 2 f2 + r xadd s * f v + r debug reload + assert_equal {1 x} [r keyspace.is_key_loaded x] + assert_equal {1 y} [r keyspace.is_key_loaded y] + assert_equal {1 z} [r keyspace.is_key_loaded z] + assert_equal {1 p} [r keyspace.is_key_loaded p] + assert_equal {1 t} [r keyspace.is_key_loaded t] + assert_equal {1 s} [r keyspace.is_key_loaded s] + } + + test {Nested multi due to RM_Call} { + r del multi + r del lua + + r set x 1 + r set x_copy 1 + r keyspace.del_key_copy x + r keyspace.incr_case1 x + r keyspace.incr_case2 x + r keyspace.incr_case3 x + assert_equal {} [r get multi] + assert_equal {} [r get lua] + r get x + } {3} + + test {Nested multi due to RM_Call, with client MULTI} { + r del multi + r del lua + + r set x 1 + r set x_copy 1 + r multi + r keyspace.del_key_copy x + r keyspace.incr_case1 x + r keyspace.incr_case2 x + r keyspace.incr_case3 x + r exec + assert_equal {1} [r get multi] + assert_equal {} [r get lua] + r get x + } {3} + + test {Nested multi due to RM_Call, with EVAL} { + r del multi + r del lua + + r set x 1 + r set x_copy 1 + r eval { + redis.pcall('keyspace.del_key_copy', KEYS[1]) + redis.pcall('keyspace.incr_case1', KEYS[1]) + redis.pcall('keyspace.incr_case2', KEYS[1]) + redis.pcall('keyspace.incr_case3', KEYS[1]) + } 1 x + assert_equal {} [r get multi] + assert_equal {1} [r get lua] + r get x + } {3} + + test {Test module key space event} { + r keyspace.notify x + assert_equal {1 x} [r keyspace.is_module_key_notified x] + } + + test "Keyspace notifications: module events test" { + r config set notify-keyspace-events Kd + r del x + set rd1 [redis_deferring_client] + assert_equal {1} [psubscribe $rd1 *] + r keyspace.notify x + assert_equal {pmessage * __keyspace@9__:x notify} [$rd1 read] + $rd1 close + } + + test {Test expired key space event} { + set prev_expired [s expired_keys] + r set exp 1 PX 10 + wait_for_condition 100 10 { + [s expired_keys] eq $prev_expired + 1 + } else { + fail "key not expired" + } + assert_equal [r get testkeyspace:expired] 1 + } + + test "Unload the module - testkeyspace" { + assert_equal {OK} [r module unload testkeyspace] + } + + test "Verify RM_StringDMA with expiration are not causing invalid memory access" { + assert_equal {OK} [r set x 1 EX 1] + } + } + + start_server {} { + test {OnLoad failure will handle un-registration} { + catch {r module load $testmodule noload} + r set x 1 + r hset y f v + r lpush z 1 2 3 + r sadd p 1 2 3 + r zadd t 1 f1 2 f2 + r xadd s * f v + r ping + } + } +} diff --git a/tests/unit/moduleapi/keyspecs.tcl b/tests/unit/moduleapi/keyspecs.tcl new file mode 100644 index 0000000..9e68e97 --- /dev/null +++ b/tests/unit/moduleapi/keyspecs.tcl @@ -0,0 +1,160 @@ +set testmodule [file normalize tests/modules/keyspecs.so] + +start_server {tags {"modules"}} { + r module load $testmodule + + test "Module key specs: No spec, only legacy triple" { + set reply [lindex [r command info kspec.none] 0] + # Verify (first, last, step) and not movablekeys + assert_equal [lindex $reply 2] {module} + assert_equal [lindex $reply 3] 1 + assert_equal [lindex $reply 4] -1 + assert_equal [lindex $reply 5] 2 + # Verify key-spec auto-generated from the legacy triple + set keyspecs [lindex $reply 8] + assert_equal [llength $keyspecs] 1 + assert_equal [lindex $keyspecs 0] {flags {RW access update} begin_search {type index spec {index 1}} find_keys {type range spec {lastkey -1 keystep 2 limit 0}}} + assert_equal [r command getkeys kspec.none key1 val1 key2 val2] {key1 key2} + } + + test "Module key specs: No spec, only legacy triple with getkeys-api" { + set reply [lindex [r command info kspec.nonewithgetkeys] 0] + # Verify (first, last, step) and movablekeys + assert_equal [lindex $reply 2] {module movablekeys} + assert_equal [lindex $reply 3] 1 + assert_equal [lindex $reply 4] -1 + assert_equal [lindex $reply 5] 2 + # Verify key-spec auto-generated from the legacy triple + set keyspecs [lindex $reply 8] + assert_equal [llength $keyspecs] 1 + assert_equal [lindex $keyspecs 0] {flags {RW access update variable_flags} begin_search {type index spec {index 1}} find_keys {type range spec {lastkey -1 keystep 2 limit 0}}} + assert_equal [r command getkeys kspec.nonewithgetkeys key1 val1 key2 val2] {key1 key2} + } + + test "Module key specs: Two ranges" { + set reply [lindex [r command info kspec.tworanges] 0] + # Verify (first, last, step) and not movablekeys + assert_equal [lindex $reply 2] {module} + assert_equal [lindex $reply 3] 1 + assert_equal [lindex $reply 4] 2 + assert_equal [lindex $reply 5] 1 + # Verify key-specs + set keyspecs [lindex $reply 8] + assert_equal [lindex $keyspecs 0] {flags {RO access} begin_search {type index spec {index 1}} find_keys {type range spec {lastkey 0 keystep 1 limit 0}}} + assert_equal [lindex $keyspecs 1] {flags {RW update} begin_search {type index spec {index 2}} find_keys {type range spec {lastkey 0 keystep 1 limit 0}}} + assert_equal [r command getkeys kspec.tworanges foo bar baz quux] {foo bar} + } + + test "Module key specs: Two ranges with gap" { + set reply [lindex [r command info kspec.tworangeswithgap] 0] + # Verify (first, last, step) and movablekeys + assert_equal [lindex $reply 2] {module movablekeys} + assert_equal [lindex $reply 3] 1 + assert_equal [lindex $reply 4] 1 + assert_equal [lindex $reply 5] 1 + # Verify key-specs + set keyspecs [lindex $reply 8] + assert_equal [lindex $keyspecs 0] {flags {RO access} begin_search {type index spec {index 1}} find_keys {type range spec {lastkey 0 keystep 1 limit 0}}} + assert_equal [lindex $keyspecs 1] {flags {RW update} begin_search {type index spec {index 3}} find_keys {type range spec {lastkey 0 keystep 1 limit 0}}} + assert_equal [r command getkeys kspec.tworangeswithgap foo bar baz quux] {foo baz} + } + + test "Module key specs: Keyword-only spec clears the legacy triple" { + set reply [lindex [r command info kspec.keyword] 0] + # Verify (first, last, step) and movablekeys + assert_equal [lindex $reply 2] {module movablekeys} + assert_equal [lindex $reply 3] 0 + assert_equal [lindex $reply 4] 0 + assert_equal [lindex $reply 5] 0 + # Verify key-specs + set keyspecs [lindex $reply 8] + assert_equal [lindex $keyspecs 0] {flags {RO access} begin_search {type keyword spec {keyword KEYS startfrom 1}} find_keys {type range spec {lastkey -1 keystep 1 limit 0}}} + assert_equal [r command getkeys kspec.keyword foo KEYS bar baz] {bar baz} + } + + test "Module key specs: Complex specs, case 1" { + set reply [lindex [r command info kspec.complex1] 0] + # Verify (first, last, step) and movablekeys + assert_equal [lindex $reply 2] {module movablekeys} + assert_equal [lindex $reply 3] 1 + assert_equal [lindex $reply 4] 1 + assert_equal [lindex $reply 5] 1 + # Verify key-specs + set keyspecs [lindex $reply 8] + assert_equal [lindex $keyspecs 0] {flags RO begin_search {type index spec {index 1}} find_keys {type range spec {lastkey 0 keystep 1 limit 0}}} + assert_equal [lindex $keyspecs 1] {flags {RW update} begin_search {type keyword spec {keyword STORE startfrom 2}} find_keys {type range spec {lastkey 0 keystep 1 limit 0}}} + assert_equal [lindex $keyspecs 2] {flags {RO access} begin_search {type keyword spec {keyword KEYS startfrom 2}} find_keys {type keynum spec {keynumidx 0 firstkey 1 keystep 1}}} + assert_equal [r command getkeys kspec.complex1 foo dummy KEYS 1 bar baz STORE quux] {foo quux bar} + } + + test "Module key specs: Complex specs, case 2" { + set reply [lindex [r command info kspec.complex2] 0] + # Verify (first, last, step) and movablekeys + assert_equal [lindex $reply 2] {module movablekeys} + assert_equal [lindex $reply 3] 1 + assert_equal [lindex $reply 4] 2 + assert_equal [lindex $reply 5] 1 + # Verify key-specs + set keyspecs [lindex $reply 8] + assert_equal [lindex $keyspecs 0] {flags {RW update} begin_search {type keyword spec {keyword STORE startfrom 5}} find_keys {type range spec {lastkey 0 keystep 1 limit 0}}} + assert_equal [lindex $keyspecs 1] {flags {RO access} begin_search {type index spec {index 1}} find_keys {type range spec {lastkey 0 keystep 1 limit 0}}} + assert_equal [lindex $keyspecs 2] {flags {RO access} begin_search {type index spec {index 2}} find_keys {type range spec {lastkey 0 keystep 1 limit 0}}} + assert_equal [lindex $keyspecs 3] {flags {RW update} begin_search {type index spec {index 3}} find_keys {type keynum spec {keynumidx 0 firstkey 1 keystep 1}}} + assert_equal [lindex $keyspecs 4] {flags {RW update} begin_search {type keyword spec {keyword MOREKEYS startfrom 5}} find_keys {type range spec {lastkey -1 keystep 1 limit 0}}} + assert_equal [r command getkeys kspec.complex2 foo bar 2 baz quux banana STORE dst dummy MOREKEYS hey ho] {dst foo bar baz quux hey ho} + } + + test "Module command list filtering" { + ;# Note: we piggyback this tcl file to test the general functionality of command list filtering + set reply [r command list filterby module keyspecs] + assert_equal [lsort $reply] {kspec.complex1 kspec.complex2 kspec.keyword kspec.none kspec.nonewithgetkeys kspec.tworanges kspec.tworangeswithgap} + assert_equal [r command getkeys kspec.complex2 foo bar 2 baz quux banana STORE dst dummy MOREKEYS hey ho] {dst foo bar baz quux hey ho} + } + + test {COMMAND GETKEYSANDFLAGS correctly reports module key-spec without flags} { + r command getkeysandflags kspec.none key1 val1 key2 val2 + } {{key1 {RW access update}} {key2 {RW access update}}} + + test {COMMAND GETKEYSANDFLAGS correctly reports module key-spec with flags} { + r command getkeysandflags kspec.nonewithgetkeys key1 val1 key2 val2 + } {{key1 {RO access}} {key2 {RO access}}} + + test {COMMAND GETKEYSANDFLAGS correctly reports module key-spec flags} { + r command getkeysandflags kspec.keyword keys key1 key2 key3 + } {{key1 {RO access}} {key2 {RO access}} {key3 {RO access}}} + + # user that can only read from "read" keys, write to "write" keys, and read+write to "RW" keys + r ACL setuser testuser +@all %R~read* %W~write* %RW~rw* + + test "Module key specs: No spec, only legacy triple - ACL" { + # legacy triple didn't provide flags, so they require both read and write + assert_equal "OK" [r ACL DRYRUN testuser kspec.none rw val1] + assert_match {*has no permissions to access the 'read' key*} [r ACL DRYRUN testuser kspec.none read val1] + assert_match {*has no permissions to access the 'write' key*} [r ACL DRYRUN testuser kspec.none write val1] + } + + test "Module key specs: tworanges - ACL" { + assert_equal "OK" [r ACL DRYRUN testuser kspec.tworanges read write] + assert_equal "OK" [r ACL DRYRUN testuser kspec.tworanges rw rw] + assert_match {*has no permissions to access the 'read' key*} [r ACL DRYRUN testuser kspec.tworanges rw read] + assert_match {*has no permissions to access the 'write' key*} [r ACL DRYRUN testuser kspec.tworanges write rw] + } + + foreach cmd {kspec.none kspec.tworanges} { + test "$cmd command will not be marked with movablekeys" { + set info [lindex [r command info $cmd] 0] + assert_no_match {*movablekeys*} [lindex $info 2] + } + } + + foreach cmd {kspec.keyword kspec.complex1 kspec.complex2 kspec.nonewithgetkeys} { + test "$cmd command is marked with movablekeys" { + set info [lindex [r command info $cmd] 0] + assert_match {*movablekeys*} [lindex $info 2] + } + } + + test "Unload the module - keyspecs" { + assert_equal {OK} [r module unload keyspecs] + } +} diff --git a/tests/unit/moduleapi/list.tcl b/tests/unit/moduleapi/list.tcl new file mode 100644 index 0000000..11f3b75 --- /dev/null +++ b/tests/unit/moduleapi/list.tcl @@ -0,0 +1,160 @@ +set testmodule [file normalize tests/modules/list.so] + +# The following arguments can be passed to args: +# i -- the number of inserts +# d -- the number of deletes +# r -- the number of replaces +# index -- the last index +# entry -- The entry pointed to by index +proc verify_list_edit_reply {reply argv} { + foreach {k v} $argv { + assert_equal [dict get $reply $k] $v + } +} + +start_server {tags {"modules"}} { + r module load $testmodule + + test {Module list set, get, insert, delete} { + r del k + assert_error {WRONGTYPE Operation against a key holding the wrong kind of value*} {r list.set k 1 xyz} + r rpush k x + # insert, set, get + r list.insert k 0 foo + r list.insert k -1 bar + r list.set k 1 xyz + assert_equal {foo xyz bar} [r list.getall k] + assert_equal {foo} [r list.get k 0] + assert_equal {xyz} [r list.get k 1] + assert_equal {bar} [r list.get k 2] + assert_equal {bar} [r list.get k -1] + assert_equal {foo} [r list.get k -3] + assert_error {ERR index out*} {r list.get k -4} + assert_error {ERR index out*} {r list.get k 3} + # remove + assert_error {ERR index out*} {r list.delete k -4} + assert_error {ERR index out*} {r list.delete k 3} + r list.delete k 0 + r list.delete k -1 + assert_equal {xyz} [r list.getall k] + # removing the last element deletes the list + r list.delete k 0 + assert_equal 0 [r exists k] + } + + test {Module list iteration} { + r del k + r rpush k x y z + assert_equal {x y z} [r list.getall k] + assert_equal {z y x} [r list.getall k REVERSE] + } + + test {Module list insert & delete} { + r del k + r rpush k x y z + verify_list_edit_reply [r list.edit k ikikdi foo bar baz] {i 3 index 5} + r list.getall k + } {foo x bar y baz} + + test {Module list insert & delete, neg index} { + r del k + r rpush k x y z + verify_list_edit_reply [r list.edit k REVERSE ikikdi foo bar baz] {i 3 index -6} + r list.getall k + } {baz y bar z foo} + + test {Module list set while iterating} { + r del k + r rpush k x y z + verify_list_edit_reply [r list.edit k rkr foo bar] {r 2 index 3} + r list.getall k + } {foo y bar} + + test {Module list set while iterating, neg index} { + r del k + r rpush k x y z + verify_list_edit_reply [r list.edit k reverse rkr foo bar] {r 2 index -4} + r list.getall k + } {bar y foo} + + test {Module list - encoding conversion while inserting} { + r config set list-max-listpack-size 4 + r del k + r rpush k a b c d + assert_encoding listpack k + + # Converts to quicklist after inserting. + r list.edit k dii foo bar + assert_encoding quicklist k + assert_equal [r list.getall k] {foo bar b c d} + + # Converts to listpack after deleting three entries. + r list.edit k ddd e + assert_encoding listpack k + assert_equal [r list.getall k] {c d} + } + + test {Module list - encoding conversion while replacing} { + r config set list-max-listpack-size -1 + r del k + r rpush k x y z + assert_encoding listpack k + + # Converts to quicklist after replacing. + set big [string repeat "x" 4096] + r list.edit k r $big + assert_encoding quicklist k + assert_equal [r list.getall k] "$big y z" + + # Converts to listpack after deleting the big entry. + r list.edit k d + assert_encoding listpack k + assert_equal [r list.getall k] {y z} + } + + test {Module list - list entry and index should be updated when deletion} { + set original_config [config_get_set list-max-listpack-size 1] + + # delete from start (index 0) + r del l + r rpush l x y z + verify_list_edit_reply [r list.edit l dd] {d 2 index 0 entry z} + assert_equal [r list.getall l] {z} + + # delete from start (index -3) + r del l + r rpush l x y z + verify_list_edit_reply [r list.edit l reverse kkd] {d 1 index -3} + assert_equal [r list.getall l] {y z} + + # # delete from tail (index 2) + r del l + r rpush l x y z + verify_list_edit_reply [r list.edit l kkd] {d 1 index 2} + assert_equal [r list.getall l] {x y} + + # # delete from tail (index -1) + r del l + r rpush l x y z + verify_list_edit_reply [r list.edit l reverse dd] {d 2 index -1 entry x} + assert_equal [r list.getall l] {x} + + # # delete from middle (index 1) + r del l + r rpush l x y z + verify_list_edit_reply [r list.edit l kdd] {d 2 index 1} + assert_equal [r list.getall l] {x} + + # # delete from middle (index -2) + r del l + r rpush l x y z + verify_list_edit_reply [r list.edit l reverse kdd] {d 2 index -2} + assert_equal [r list.getall l] {z} + + config_set list-max-listpack-size $original_config + } + + test "Unload the module - list" { + assert_equal {OK} [r module unload list] + } +} diff --git a/tests/unit/moduleapi/mallocsize.tcl b/tests/unit/moduleapi/mallocsize.tcl new file mode 100644 index 0000000..359a7ae --- /dev/null +++ b/tests/unit/moduleapi/mallocsize.tcl @@ -0,0 +1,21 @@ +set testmodule [file normalize tests/modules/mallocsize.so] + + +start_server {tags {"modules"}} { + r module load $testmodule + + test {MallocSize of raw bytes} { + assert_equal [r mallocsize.setraw key 40] {OK} + assert_morethan [r memory usage key] 40 + } + + test {MallocSize of string} { + assert_equal [r mallocsize.setstr key abcdefg] {OK} + assert_morethan [r memory usage key] 7 ;# Length of "abcdefg" + } + + test {MallocSize of dict} { + assert_equal [r mallocsize.setdict key f1 v1 f2 v2] {OK} + assert_morethan [r memory usage key] 8 ;# Length of "f1v1f2v2" + } +} diff --git a/tests/unit/moduleapi/misc.tcl b/tests/unit/moduleapi/misc.tcl new file mode 100644 index 0000000..cf20546 --- /dev/null +++ b/tests/unit/moduleapi/misc.tcl @@ -0,0 +1,555 @@ +set testmodule [file normalize tests/modules/misc.so] + +start_server {overrides {save {900 1}} tags {"modules"}} { + r module load $testmodule + + test {test RM_Call} { + set info [r test.call_info commandstats] + # cmdstat is not in a default section, so we also test an argument was passed + assert { [string match "*cmdstat_module*" $info] } + } + + test {test RM_Call args array} { + set info [r test.call_generic info commandstats] + # cmdstat is not in a default section, so we also test an argument was passed + assert { [string match "*cmdstat_module*" $info] } + } + + test {test RM_Call recursive} { + set info [r test.call_generic test.call_generic info commandstats] + assert { [string match "*cmdstat_module*" $info] } + } + + test {test redis version} { + set version [s redis_version] + assert_equal $version [r test.redisversion] + } + + test {test long double conversions} { + set ld [r test.ld_conversion] + assert {[string match $ld "0.00000000000000001"]} + } + + test {test unsigned long long conversions} { + set ret [r test.ull_conversion] + assert {[string match $ret "ok"]} + } + + test {test module db commands} { + r set x foo + set key [r test.randomkey] + assert_equal $key "x" + assert_equal [r test.dbsize] 1 + r test.flushall + assert_equal [r test.dbsize] 0 + } + + test {test RedisModule_ResetDataset do not reset functions} { + r function load {#!lua name=lib + redis.register_function('test', function() return 1 end) + } + assert_equal [r function list] {{library_name lib engine LUA functions {{name test description {} flags {}}}}} + r test.flushall + assert_equal [r function list] {{library_name lib engine LUA functions {{name test description {} flags {}}}}} + r function flush + } + + test {test module keyexists} { + r set x foo + assert_equal 1 [r test.keyexists x] + r del x + assert_equal 0 [r test.keyexists x] + } + + test {test module lru api} { + r config set maxmemory-policy allkeys-lru + r set x foo + set lru [r test.getlru x] + assert { $lru <= 1000 } + set was_set [r test.setlru x 100000] + assert { $was_set == 1 } + set idle [r object idletime x] + assert { $idle >= 100 } + set lru [r test.getlru x] + assert { $lru >= 100000 } + r config set maxmemory-policy allkeys-lfu + set lru [r test.getlru x] + assert { $lru == -1 } + set was_set [r test.setlru x 100000] + assert { $was_set == 0 } + } + r config set maxmemory-policy allkeys-lru + + test {test module lfu api} { + r config set maxmemory-policy allkeys-lfu + r set x foo + set lfu [r test.getlfu x] + assert { $lfu >= 1 } + set was_set [r test.setlfu x 100] + assert { $was_set == 1 } + set freq [r object freq x] + assert { $freq <= 100 } + set lfu [r test.getlfu x] + assert { $lfu <= 100 } + r config set maxmemory-policy allkeys-lru + set lfu [r test.getlfu x] + assert { $lfu == -1 } + set was_set [r test.setlfu x 100] + assert { $was_set == 0 } + } + + test {test module clientinfo api} { + # Test basic sanity and SSL flag + set info [r test.clientinfo] + set ssl_flag [expr $::tls ? {"ssl:"} : {":"}] + + assert { [dict get $info db] == 9 } + assert { [dict get $info flags] == "${ssl_flag}::::" } + + # Test MULTI flag + r multi + r test.clientinfo + set info [lindex [r exec] 0] + assert { [dict get $info flags] == "${ssl_flag}::::multi" } + + # Test TRACKING flag + r client tracking on + set info [r test.clientinfo] + assert { [dict get $info flags] == "${ssl_flag}::tracking::" } + r CLIENT TRACKING off + } + + test {tracking with rm_call sanity} { + set rd_trk [redis_client] + $rd_trk HELLO 3 + $rd_trk CLIENT TRACKING on + r MSET key1{t} 1 key2{t} 1 + + # GET triggers tracking, SET does not + $rd_trk test.rm_call GET key1{t} + $rd_trk test.rm_call SET key2{t} 2 + r MSET key1{t} 2 key2{t} 2 + assert_equal {invalidate key1{t}} [$rd_trk read] + assert_equal "PONG" [$rd_trk ping] + $rd_trk close + } + + test {tracking with rm_call with script} { + set rd_trk [redis_client] + $rd_trk HELLO 3 + $rd_trk CLIENT TRACKING on + r MSET key1{t} 1 key2{t} 1 + + # GET triggers tracking, SET does not + $rd_trk test.rm_call EVAL "redis.call('get', 'key1{t}')" 2 key1{t} key2{t} + r MSET key1{t} 2 key2{t} 2 + assert_equal {invalidate key1{t}} [$rd_trk read] + assert_equal "PONG" [$rd_trk ping] + $rd_trk close + } + + test {publish to self inside rm_call} { + r hello 3 + r subscribe foo + + # published message comes after the response of the command that issued it. + assert_equal [r test.rm_call publish foo bar] {1} + assert_equal [r read] {message foo bar} + + r unsubscribe foo + r hello 2 + set _ "" + } {} {resp3} + + test {test module get/set client name by id api} { + catch { r test.getname } e + assert_equal "-ERR No name" $e + r client setname nobody + catch { r test.setname "name with spaces" } e + assert_match "*Invalid argument*" $e + assert_equal nobody [r client getname] + assert_equal nobody [r test.getname] + r test.setname somebody + assert_equal somebody [r client getname] + } + + test {test module getclientcert api} { + set cert [r test.getclientcert] + + if {$::tls} { + assert {$cert != ""} + } else { + assert {$cert == ""} + } + } + + test {test detached thread safe cnotext} { + r test.log_tsctx "info" "Test message" + verify_log_message 0 "*<misc> Test message*" 0 + } + + test {test RM_Call CLIENT INFO} { + assert_match "*fd=-1*" [r test.call_generic client info] + } + + test {Unsafe command names are sanitized in INFO output} { + r test.weird:cmd + set info [r info commandstats] + assert_match {*cmdstat_test.weird_cmd:calls=1*} $info + } + + test {test monotonic time} { + set x [r test.monotonic_time] + assert { [r test.monotonic_time] >= $x } + } + + test {rm_call OOM} { + r config set maxmemory 1 + r config set maxmemory-policy volatile-lru + + # sanity test plain call + assert_equal {OK} [ + r test.rm_call set x 1 + ] + + # add the M flag + assert_error {OOM *} { + r test.rm_call_flags M set x 1 + + } + + # test a non deny-oom command + assert_equal {1} [ + r test.rm_call_flags M get x + ] + + r config set maxmemory 0 + } {OK} {needs:config-maxmemory} + + test {rm_call clear OOM} { + r config set maxmemory 1 + + # verify rm_call fails with OOM + assert_error {OOM *} { + r test.rm_call_flags M set x 1 + } + + # clear OOM state + r config set maxmemory 0 + + # test set command is allowed + r test.rm_call_flags M set x 1 + } {OK} {needs:config-maxmemory} + + test {rm_call OOM Eval} { + r config set maxmemory 1 + r config set maxmemory-policy volatile-lru + + # use the M flag without allow-oom shebang flag + assert_error {OOM *} { + r test.rm_call_flags M eval {#!lua + redis.call('set','x',1) + return 1 + } 1 x + } + + # add the M flag with allow-oom shebang flag + assert_equal {1} [ + r test.rm_call_flags M eval {#!lua flags=allow-oom + redis.call('set','x',1) + return 1 + } 1 x + ] + + r config set maxmemory 0 + } {OK} {needs:config-maxmemory} + + test {rm_call write flag} { + # add the W flag + assert_error {ERR Write command 'set' was called while write is not allowed.} { + r test.rm_call_flags W set x 1 + } + + # test a non deny-oom command + r test.rm_call_flags W get x + } {1} + + test {rm_call EVAL} { + r test.rm_call eval { + redis.call('set','x',1) + return 1 + } 1 x + + assert_error {ERR Write commands are not allowed from read-only scripts.*} { + r test.rm_call eval {#!lua flags=no-writes + redis.call('set','x',1) + return 1 + } 1 x + } + } + + # Note: each script is unique, to check that flags are extracted correctly + test {rm_call EVAL - OOM - with M flag} { + r config set maxmemory 1 + + # script without shebang, but uses SET, so fails + assert_error {*OOM command not allowed when used memory > 'maxmemory'*} { + r test.rm_call_flags M eval { + redis.call('set','x',1) + return 1 + } 1 x + } + + # script with an allow-oom flag, succeeds despite using SET + r test.rm_call_flags M eval {#!lua flags=allow-oom + redis.call('set','x', 1) + return 2 + } 1 x + + # script with no-writes flag, implies allow-oom, succeeds + r test.rm_call_flags M eval {#!lua flags=no-writes + redis.call('get','x') + return 2 + } 1 x + + # script with shebang using default flags, so fails regardless of using only GET + assert_error {*OOM command not allowed when used memory > 'maxmemory'*} { + r test.rm_call_flags M eval {#!lua + redis.call('get','x') + return 3 + } 1 x + } + + # script without shebang, but uses GET, so succeeds + r test.rm_call_flags M eval { + redis.call('get','x') + return 4 + } 1 x + + r config set maxmemory 0 + } {OK} {needs:config-maxmemory} + + # All RM_Call for script succeeds in OOM state without using the M flag + test {rm_call EVAL - OOM - without M flag} { + r config set maxmemory 1 + + # no shebang at all + r test.rm_call eval { + redis.call('set','x',1) + return 6 + } 1 x + + # Shebang without flags + r test.rm_call eval {#!lua + redis.call('set','x', 1) + return 7 + } 1 x + + # with allow-oom flag + r test.rm_call eval {#!lua flags=allow-oom + redis.call('set','x', 1) + return 8 + } 1 x + + r config set maxmemory 0 + } {OK} {needs:config-maxmemory} + + test "not enough good replicas" { + r set x "some value" + r config set min-replicas-to-write 1 + + # rm_call in script mode + assert_error {NOREPLICAS *} {r test.rm_call_flags S set x s} + + assert_equal [ + r test.rm_call eval {#!lua flags=no-writes + return redis.call('get','x') + } 1 x + ] "some value" + + assert_equal [ + r test.rm_call eval { + return redis.call('get','x') + } 1 x + ] "some value" + + assert_error {NOREPLICAS *} { + r test.rm_call eval {#!lua + return redis.call('get','x') + } 1 x + } + + assert_error {NOREPLICAS *} { + r test.rm_call eval { + return redis.call('set','x', 1) + } 1 x + } + + r config set min-replicas-to-write 0 + } + + test {rm_call EVAL - read-only replica} { + r replicaof 127.0.0.1 1 + + # rm_call in script mode + assert_error {READONLY *} {r test.rm_call_flags S set x 1} + + assert_error {READONLY You can't write against a read only replica. script*} { + r test.rm_call eval { + redis.call('set','x',1) + return 1 + } 1 x + } + + r test.rm_call eval {#!lua flags=no-writes + redis.call('get','x') + return 2 + } 1 x + + assert_error {READONLY Can not run script with write flag on readonly replica*} { + r test.rm_call eval {#!lua + redis.call('get','x') + return 3 + } 1 x + } + + r test.rm_call eval { + redis.call('get','x') + return 4 + } 1 x + + r replicaof no one + } {OK} {needs:config-maxmemory} + + test {rm_call EVAL - stale replica} { + r replicaof 127.0.0.1 1 + r config set replica-serve-stale-data no + + # rm_call in script mode + assert_error {MASTERDOWN *} { + r test.rm_call_flags S get x + } + + assert_error {MASTERDOWN *} { + r test.rm_call eval {#!lua flags=no-writes + redis.call('get','x') + return 2 + } 1 x + } + + assert_error {MASTERDOWN *} { + r test.rm_call eval { + redis.call('get','x') + return 4 + } 1 x + } + + r replicaof no one + r config set replica-serve-stale-data yes + } {OK} {needs:config-maxmemory} + + test "rm_call EVAL - failed bgsave prevents writes" { + r config set rdb-key-save-delay 10000000 + populate 1000 + r set x x + r bgsave + set pid1 [get_child_pid 0] + catch {exec kill -9 $pid1} + waitForBgsave r + + # make sure a read command succeeds + assert_equal [r get x] x + + # make sure a write command fails + assert_error {MISCONF *} {r set x y} + + # rm_call in script mode + assert_error {MISCONF *} {r test.rm_call_flags S set x 1} + + # repeate with script + assert_error {MISCONF *} {r test.rm_call eval { + return redis.call('set','x',1) + } 1 x + } + assert_equal {x} [r test.rm_call eval { + return redis.call('get','x') + } 1 x + ] + + # again with script using shebang + assert_error {MISCONF *} {r test.rm_call eval {#!lua + return redis.call('set','x',1) + } 1 x + } + assert_equal {x} [r test.rm_call eval {#!lua flags=no-writes + return redis.call('get','x') + } 1 x + ] + + r config set rdb-key-save-delay 0 + r bgsave + waitForBgsave r + + # server is writable again + r set x y + } {OK} +} + +start_server {tags {"modules"}} { + r module load $testmodule + + test {test Dry Run - OK OOM/ACL} { + set x 5 + r set x $x + catch {r test.rm_call_flags DMC set x 10} e + assert_match {*NULL reply returned*} $e + assert_equal [r get x] 5 + } + + test {test Dry Run - Fail OOM} { + set x 5 + r set x $x + r config set maxmemory 1 + catch {r test.rm_call_flags DM set x 10} e + assert_match {*OOM*} $e + assert_equal [r get x] $x + r config set maxmemory 0 + } {OK} {needs:config-maxmemory} + + test {test Dry Run - Fail ACL} { + set x 5 + r set x $x + # deny all permissions besides the dryrun command + r acl setuser default resetkeys + + catch {r test.rm_call_flags DC set x 10} e + assert_match {*NOPERM No permissions to access a key*} $e + r acl setuser default +@all ~* + assert_equal [r get x] $x + } + + test {test silent open key} { + r debug set-active-expire 0 + r test.clear_n_events + r set x 1 PX 10 + after 1000 + # now the key has been expired, open it silently and make sure not event were fired. + assert_error {key not found} {r test.silent_open_key x} + assert_equal {0} [r test.get_n_events] + } + +if {[string match {*jemalloc*} [s mem_allocator]]} { + test {test RM_Call with large arg for SET command} { + # set a big value to trigger increasing the query buf + r set foo [string repeat A 100000] + # set a smaller value but > PROTO_MBULK_BIG_ARG (32*1024) Redis will try to save the query buf itself on the DB. + r test.call_generic set bar [string repeat A 33000] + # asset the value was trimmed + assert {[r memory usage bar] < 42000}; # 42K to count for Jemalloc's additional memory overhead. + } +} ;# if jemalloc + + test "Unload the module - misc" { + assert_equal {OK} [r module unload misc] + } +} diff --git a/tests/unit/moduleapi/moduleauth.tcl b/tests/unit/moduleapi/moduleauth.tcl new file mode 100644 index 0000000..82f42f5 --- /dev/null +++ b/tests/unit/moduleapi/moduleauth.tcl @@ -0,0 +1,405 @@ +set testmodule [file normalize tests/modules/auth.so] +set testmoduletwo [file normalize tests/modules/moduleauthtwo.so] +set miscmodule [file normalize tests/modules/misc.so] + +proc cmdstat {cmd} { + return [cmdrstat $cmd r] +} + +start_server {tags {"modules"}} { + r module load $testmodule + r module load $testmoduletwo + + set hello2_response [r HELLO 2] + set hello3_response [r HELLO 3] + + test {test registering module auth callbacks} { + assert_equal {OK} [r testmoduleone.rm_register_blocking_auth_cb] + assert_equal {OK} [r testmoduletwo.rm_register_auth_cb] + assert_equal {OK} [r testmoduleone.rm_register_auth_cb] + } + + test {test module AUTH for non existing / disabled users} { + r config resetstat + # Validate that an error is thrown for non existing users. + assert_error {*WRONGPASS*} {r AUTH foo pwd} + assert_match {*calls=1,*,rejected_calls=0,failed_calls=1} [cmdstat auth] + # Validate that an error is thrown for disabled users. + r acl setuser foo >pwd off ~* &* +@all + assert_error {*WRONGPASS*} {r AUTH foo pwd} + assert_match {*calls=2,*,rejected_calls=0,failed_calls=2} [cmdstat auth] + } + + test {test non blocking module AUTH} { + r config resetstat + # Test for a fixed password user + r acl setuser foo >pwd on ~* &* +@all + assert_equal {OK} [r AUTH foo allow] + assert_error {*Auth denied by Misc Module*} {r AUTH foo deny} + assert_match {*calls=2,*,rejected_calls=0,failed_calls=1} [cmdstat auth] + assert_error {*WRONGPASS*} {r AUTH foo nomatch} + assert_match {*calls=3,*,rejected_calls=0,failed_calls=2} [cmdstat auth] + assert_equal {OK} [r AUTH foo pwd] + # Test for No Pass user + r acl setuser foo on ~* &* +@all nopass + assert_equal {OK} [r AUTH foo allow] + assert_error {*Auth denied by Misc Module*} {r AUTH foo deny} + assert_match {*calls=6,*,rejected_calls=0,failed_calls=3} [cmdstat auth] + assert_equal {OK} [r AUTH foo nomatch] + + # Validate that the Module added an ACL Log entry. + set entry [lindex [r ACL LOG] 0] + assert {[dict get $entry username] eq {foo}} + assert {[dict get $entry context] eq {module}} + assert {[dict get $entry reason] eq {auth}} + assert {[dict get $entry object] eq {Module Auth}} + assert_match {*cmd=auth*} [dict get $entry client-info] + r ACL LOG RESET + } + + test {test non blocking module HELLO AUTH} { + r config resetstat + r acl setuser foo >pwd on ~* &* +@all + # Validate proto 2 and 3 in case of success + assert_equal $hello2_response [r HELLO 2 AUTH foo pwd] + assert_equal $hello2_response [r HELLO 2 AUTH foo allow] + assert_equal $hello3_response [r HELLO 3 AUTH foo pwd] + assert_equal $hello3_response [r HELLO 3 AUTH foo allow] + # Validate denying AUTH for the HELLO cmd + assert_error {*Auth denied by Misc Module*} {r HELLO 2 AUTH foo deny} + assert_match {*calls=5,*,rejected_calls=0,failed_calls=1} [cmdstat hello] + assert_error {*WRONGPASS*} {r HELLO 2 AUTH foo nomatch} + assert_match {*calls=6,*,rejected_calls=0,failed_calls=2} [cmdstat hello] + assert_error {*Auth denied by Misc Module*} {r HELLO 3 AUTH foo deny} + assert_match {*calls=7,*,rejected_calls=0,failed_calls=3} [cmdstat hello] + assert_error {*WRONGPASS*} {r HELLO 3 AUTH foo nomatch} + assert_match {*calls=8,*,rejected_calls=0,failed_calls=4} [cmdstat hello] + + # Validate that the Module added an ACL Log entry. + set entry [lindex [r ACL LOG] 1] + assert {[dict get $entry username] eq {foo}} + assert {[dict get $entry context] eq {module}} + assert {[dict get $entry reason] eq {auth}} + assert {[dict get $entry object] eq {Module Auth}} + assert_match {*cmd=hello*} [dict get $entry client-info] + r ACL LOG RESET + } + + test {test non blocking module HELLO AUTH SETNAME} { + r config resetstat + r acl setuser foo >pwd on ~* &* +@all + # Validate clientname is set on success + assert_equal $hello2_response [r HELLO 2 AUTH foo pwd setname client1] + assert {[r client getname] eq {client1}} + assert_equal $hello2_response [r HELLO 2 AUTH foo allow setname client2] + assert {[r client getname] eq {client2}} + # Validate clientname is not updated on failure + r client setname client0 + assert_error {*Auth denied by Misc Module*} {r HELLO 2 AUTH foo deny setname client1} + assert {[r client getname] eq {client0}} + assert_match {*calls=3,*,rejected_calls=0,failed_calls=1} [cmdstat hello] + assert_error {*WRONGPASS*} {r HELLO 2 AUTH foo nomatch setname client2} + assert {[r client getname] eq {client0}} + assert_match {*calls=4,*,rejected_calls=0,failed_calls=2} [cmdstat hello] + } + + test {test blocking module AUTH} { + r config resetstat + # Test for a fixed password user + r acl setuser foo >pwd on ~* &* +@all + assert_equal {OK} [r AUTH foo block_allow] + assert_error {*Auth denied by Misc Module*} {r AUTH foo block_deny} + assert_match {*calls=2,*,rejected_calls=0,failed_calls=1} [cmdstat auth] + assert_error {*WRONGPASS*} {r AUTH foo nomatch} + assert_match {*calls=3,*,rejected_calls=0,failed_calls=2} [cmdstat auth] + assert_equal {OK} [r AUTH foo pwd] + # Test for No Pass user + r acl setuser foo on ~* &* +@all nopass + assert_equal {OK} [r AUTH foo block_allow] + assert_error {*Auth denied by Misc Module*} {r AUTH foo block_deny} + assert_match {*calls=6,*,rejected_calls=0,failed_calls=3} [cmdstat auth] + assert_equal {OK} [r AUTH foo nomatch] + # Validate that every Blocking AUTH command took at least 500000 usec. + set stats [cmdstat auth] + regexp "usec_per_call=(\[0-9]{1,})\.*," $stats all usec_per_call + assert {$usec_per_call >= 500000} + + # Validate that the Module added an ACL Log entry. + set entry [lindex [r ACL LOG] 0] + assert {[dict get $entry username] eq {foo}} + assert {[dict get $entry context] eq {module}} + assert {[dict get $entry reason] eq {auth}} + assert {[dict get $entry object] eq {Module Auth}} + assert_match {*cmd=auth*} [dict get $entry client-info] + r ACL LOG RESET + } + + test {test blocking module HELLO AUTH} { + r config resetstat + r acl setuser foo >pwd on ~* &* +@all + # validate proto 2 and 3 in case of success + assert_equal $hello2_response [r HELLO 2 AUTH foo pwd] + assert_equal $hello2_response [r HELLO 2 AUTH foo block_allow] + assert_equal $hello3_response [r HELLO 3 AUTH foo pwd] + assert_equal $hello3_response [r HELLO 3 AUTH foo block_allow] + # validate denying AUTH for the HELLO cmd + assert_error {*Auth denied by Misc Module*} {r HELLO 2 AUTH foo block_deny} + assert_match {*calls=5,*,rejected_calls=0,failed_calls=1} [cmdstat hello] + assert_error {*WRONGPASS*} {r HELLO 2 AUTH foo nomatch} + assert_match {*calls=6,*,rejected_calls=0,failed_calls=2} [cmdstat hello] + assert_error {*Auth denied by Misc Module*} {r HELLO 3 AUTH foo block_deny} + assert_match {*calls=7,*,rejected_calls=0,failed_calls=3} [cmdstat hello] + assert_error {*WRONGPASS*} {r HELLO 3 AUTH foo nomatch} + assert_match {*calls=8,*,rejected_calls=0,failed_calls=4} [cmdstat hello] + # Validate that every HELLO AUTH command took at least 500000 usec. + set stats [cmdstat hello] + regexp "usec_per_call=(\[0-9]{1,})\.*," $stats all usec_per_call + assert {$usec_per_call >= 500000} + + # Validate that the Module added an ACL Log entry. + set entry [lindex [r ACL LOG] 1] + assert {[dict get $entry username] eq {foo}} + assert {[dict get $entry context] eq {module}} + assert {[dict get $entry reason] eq {auth}} + assert {[dict get $entry object] eq {Module Auth}} + assert_match {*cmd=hello*} [dict get $entry client-info] + r ACL LOG RESET + } + + test {test blocking module HELLO AUTH SETNAME} { + r config resetstat + r acl setuser foo >pwd on ~* &* +@all + # Validate clientname is set on success + assert_equal $hello2_response [r HELLO 2 AUTH foo pwd setname client1] + assert {[r client getname] eq {client1}} + assert_equal $hello2_response [r HELLO 2 AUTH foo block_allow setname client2] + assert {[r client getname] eq {client2}} + # Validate clientname is not updated on failure + r client setname client0 + assert_error {*Auth denied by Misc Module*} {r HELLO 2 AUTH foo block_deny setname client1} + assert {[r client getname] eq {client0}} + assert_match {*calls=3,*,rejected_calls=0,failed_calls=1} [cmdstat hello] + assert_error {*WRONGPASS*} {r HELLO 2 AUTH foo nomatch setname client2} + assert {[r client getname] eq {client0}} + assert_match {*calls=4,*,rejected_calls=0,failed_calls=2} [cmdstat hello] + # Validate that every HELLO AUTH SETNAME command took at least 500000 usec. + set stats [cmdstat hello] + regexp "usec_per_call=(\[0-9]{1,})\.*," $stats all usec_per_call + assert {$usec_per_call >= 500000} + } + + test {test AUTH after registering multiple module auth callbacks} { + r config resetstat + + # Register two more callbacks from the same module. + assert_equal {OK} [r testmoduleone.rm_register_blocking_auth_cb] + assert_equal {OK} [r testmoduleone.rm_register_auth_cb] + + # Register another module auth callback from the second module. + assert_equal {OK} [r testmoduletwo.rm_register_auth_cb] + + r acl setuser foo >pwd on ~* &* +@all + + # Case 1 - Non Blocking Success + assert_equal {OK} [r AUTH foo allow] + + # Case 2 - Non Blocking Deny + assert_error {*Auth denied by Misc Module*} {r AUTH foo deny} + assert_match {*calls=2,*,rejected_calls=0,failed_calls=1} [cmdstat auth] + + r config resetstat + + # Case 3 - Blocking Success + assert_equal {OK} [r AUTH foo block_allow] + + # Case 4 - Blocking Deny + assert_error {*Auth denied by Misc Module*} {r AUTH foo block_deny} + assert_match {*calls=2,*,rejected_calls=0,failed_calls=1} [cmdstat auth] + + # Validate that every Blocking AUTH command took at least 500000 usec. + set stats [cmdstat auth] + regexp "usec_per_call=(\[0-9]{1,})\.*," $stats all usec_per_call + assert {$usec_per_call >= 500000} + + r config resetstat + + # Case 5 - Non Blocking Success via the second module. + assert_equal {OK} [r AUTH foo allow_two] + + # Case 6 - Non Blocking Deny via the second module. + assert_error {*Auth denied by Misc Module*} {r AUTH foo deny_two} + assert_match {*calls=2,*,rejected_calls=0,failed_calls=1} [cmdstat auth] + + r config resetstat + + # Case 7 - All four auth callbacks "Skip" by not explicitly allowing or denying. + assert_error {*WRONGPASS*} {r AUTH foo nomatch} + assert_match {*calls=1,*,rejected_calls=0,failed_calls=1} [cmdstat auth] + assert_equal {OK} [r AUTH foo pwd] + + # Because we had to attempt all 4 callbacks, validate that the AUTH command took at least + # 1000000 usec (each blocking callback takes 500000 usec). + set stats [cmdstat auth] + regexp "usec_per_call=(\[0-9]{1,})\.*," $stats all usec_per_call + assert {$usec_per_call >= 1000000} + } + + test {module auth during blocking module auth} { + r config resetstat + r acl setuser foo >pwd on ~* &* +@all + set rd [redis_deferring_client] + set rd_two [redis_deferring_client] + + # Attempt blocking module auth. While this ongoing, attempt non blocking module auth from + # moduleone/moduletwo and start another blocking module auth from another deferring client. + $rd AUTH foo block_allow + wait_for_blocked_clients_count 1 + assert_equal {OK} [r AUTH foo allow] + assert_equal {OK} [r AUTH foo allow_two] + # Validate that the non blocking module auth cmds finished before any blocking module auth. + set info_clients [r info clients] + assert_match "*blocked_clients:1*" $info_clients + $rd_two AUTH foo block_allow + + # Validate that all of the AUTH commands succeeded. + wait_for_blocked_clients_count 0 500 10 + $rd flush + assert_equal [$rd read] "OK" + $rd_two flush + assert_equal [$rd_two read] "OK" + assert_match {*calls=4,*,rejected_calls=0,failed_calls=0} [cmdstat auth] + } + + test {module auth inside MULTI EXEC} { + r config resetstat + r acl setuser foo >pwd on ~* &* +@all + + # Validate that non blocking module auth inside MULTI succeeds. + r multi + r AUTH foo allow + assert_equal {OK} [r exec] + + # Validate that blocking module auth inside MULTI throws an err. + r multi + r AUTH foo block_allow + assert_error {*ERR Blocking module command called from transaction*} {r exec} + assert_match {*calls=2,*,rejected_calls=0,failed_calls=1} [cmdstat auth] + } + + test {Disabling Redis User during blocking module auth} { + r config resetstat + r acl setuser foo >pwd on ~* &* +@all + set rd [redis_deferring_client] + + # Attempt blocking module auth and disable the Redis user while module auth is in progress. + $rd AUTH foo pwd + wait_for_blocked_clients_count 1 + r acl setuser foo >pwd off ~* &* +@all + + # Validate that module auth failed. + wait_for_blocked_clients_count 0 500 10 + $rd flush + assert_error {*WRONGPASS*} { $rd read } + assert_match {*calls=1,*,rejected_calls=0,failed_calls=1} [cmdstat auth] + } + + test {Killing a client in the middle of blocking module auth} { + r config resetstat + r acl setuser foo >pwd on ~* &* +@all + set rd [redis_deferring_client] + $rd client id + set cid [$rd read] + + # Attempt blocking module auth command on client `cid` and kill the client while module auth + # is in progress. + $rd AUTH foo pwd + wait_for_blocked_clients_count 1 + r client kill id $cid + + # Validate that the blocked client count goes to 0 and no AUTH command is tracked. + wait_for_blocked_clients_count 0 500 10 + $rd flush + assert_error {*I/O error reading reply*} { $rd read } + assert_match {} [cmdstat auth] + } + + test {test RM_AbortBlock Module API during blocking module auth} { + r config resetstat + r acl setuser foo >pwd on ~* &* +@all + + # Attempt module auth. With the "block_abort" as the password, the "testacl.so" module + # blocks the client and uses the RM_AbortBlock API. This should result in module auth + # failing and the client being unblocked with the default AUTH err message. + assert_error {*WRONGPASS*} {r AUTH foo block_abort} + assert_match {*calls=1,*,rejected_calls=0,failed_calls=1} [cmdstat auth] + } + + test {test RM_RegisterAuthCallback Module API during blocking module auth} { + r config resetstat + r acl setuser foo >defaultpwd on ~* &* +@all + set rd [redis_deferring_client] + + # Start the module auth attempt with the standard Redis auth password for the user. This + # will result in all module auth cbs attempted and then standard Redis auth will be tried. + $rd AUTH foo defaultpwd + wait_for_blocked_clients_count 1 + + # Validate that we allow modules to register module auth cbs while module auth is already + # in progress. + assert_equal {OK} [r testmoduleone.rm_register_blocking_auth_cb] + assert_equal {OK} [r testmoduletwo.rm_register_auth_cb] + + # Validate that blocking module auth succeeds. + wait_for_blocked_clients_count 0 500 10 + $rd flush + assert_equal [$rd read] "OK" + set stats [cmdstat auth] + assert_match {*calls=1,*,rejected_calls=0,failed_calls=0} $stats + + # Validate that even the new blocking module auth cb which was registered in the middle of + # blocking module auth is attempted - making it take twice the duration (2x 500000 us). + regexp "usec_per_call=(\[0-9]{1,})\.*," $stats all usec_per_call + assert {$usec_per_call >= 1000000} + } + + test {Module unload during blocking module auth} { + r config resetstat + r module load $miscmodule + set rd [redis_deferring_client] + r acl setuser foo >pwd on ~* &* +@all + + # Start a blocking module auth attempt. + $rd AUTH foo block_allow + wait_for_blocked_clients_count 1 + + # moduleone and moduletwo have module auth cbs registered. Because blocking module auth is + # ongoing, they cannot be unloaded. + catch {r module unload testacl} e + assert_match {*the module has blocked clients*} $e + # The moduleauthtwo module can be unregistered because no client is blocked on it. + assert_equal "OK" [r module unload moduleauthtwo] + + # The misc module does not have module auth cbs registered, so it can be unloaded even when + # blocking module auth is ongoing. + assert_equal "OK" [r module unload misc] + + # Validate that blocking module auth succeeds. + wait_for_blocked_clients_count 0 500 10 + $rd flush + assert_equal [$rd read] "OK" + assert_match {*calls=1,*,rejected_calls=0,failed_calls=0} [cmdstat auth] + + # Validate that unloading the moduleauthtwo module does not unregister module auth cbs of + # of the testacl module. Module based auth should succeed. + assert_equal {OK} [r AUTH foo allow] + + # Validate that the testacl module can be unloaded since blocking module auth is done. + r module unload testacl + + # Validate that since all module auth cbs are unregistered, module auth attempts fail. + assert_error {*WRONGPASS*} {r AUTH foo block_allow} + assert_error {*WRONGPASS*} {r AUTH foo allow_two} + assert_error {*WRONGPASS*} {r AUTH foo allow} + assert_match {*calls=5,*,rejected_calls=0,failed_calls=3} [cmdstat auth] + } +} diff --git a/tests/unit/moduleapi/moduleconfigs.tcl b/tests/unit/moduleapi/moduleconfigs.tcl new file mode 100644 index 0000000..1709e9d --- /dev/null +++ b/tests/unit/moduleapi/moduleconfigs.tcl @@ -0,0 +1,247 @@ +set testmodule [file normalize tests/modules/moduleconfigs.so] +set testmoduletwo [file normalize tests/modules/moduleconfigstwo.so] + +start_server {tags {"modules"}} { + r module load $testmodule + test {Config get commands work} { + # Make sure config get module config works + assert_not_equal [lsearch [lmap x [r module list] {dict get $x name}] moduleconfigs] -1 + assert_equal [r config get moduleconfigs.mutable_bool] "moduleconfigs.mutable_bool yes" + assert_equal [r config get moduleconfigs.immutable_bool] "moduleconfigs.immutable_bool no" + assert_equal [r config get moduleconfigs.memory_numeric] "moduleconfigs.memory_numeric 1024" + assert_equal [r config get moduleconfigs.string] "moduleconfigs.string {secret password}" + assert_equal [r config get moduleconfigs.enum] "moduleconfigs.enum one" + assert_equal [r config get moduleconfigs.flags] "moduleconfigs.flags {one two}" + assert_equal [r config get moduleconfigs.numeric] "moduleconfigs.numeric -1" + } + + test {Config set commands work} { + # Make sure that config sets work during runtime + r config set moduleconfigs.mutable_bool no + assert_equal [r config get moduleconfigs.mutable_bool] "moduleconfigs.mutable_bool no" + r config set moduleconfigs.memory_numeric 1mb + assert_equal [r config get moduleconfigs.memory_numeric] "moduleconfigs.memory_numeric 1048576" + r config set moduleconfigs.string wafflewednesdays + assert_equal [r config get moduleconfigs.string] "moduleconfigs.string wafflewednesdays" + set not_embstr [string repeat A 50] + r config set moduleconfigs.string $not_embstr + assert_equal [r config get moduleconfigs.string] "moduleconfigs.string $not_embstr" + r config set moduleconfigs.string \x73\x75\x70\x65\x72\x20\x00\x73\x65\x63\x72\x65\x74\x20\x70\x61\x73\x73\x77\x6f\x72\x64 + assert_equal [r config get moduleconfigs.string] "moduleconfigs.string {super \0secret password}" + r config set moduleconfigs.enum two + assert_equal [r config get moduleconfigs.enum] "moduleconfigs.enum two" + r config set moduleconfigs.flags two + assert_equal [r config get moduleconfigs.flags] "moduleconfigs.flags two" + r config set moduleconfigs.numeric -2 + assert_equal [r config get moduleconfigs.numeric] "moduleconfigs.numeric -2" + } + + test {Config set commands enum flags} { + r config set moduleconfigs.flags "none" + assert_equal [r config get moduleconfigs.flags] "moduleconfigs.flags none" + + r config set moduleconfigs.flags "two four" + assert_equal [r config get moduleconfigs.flags] "moduleconfigs.flags {two four}" + + r config set moduleconfigs.flags "five" + assert_equal [r config get moduleconfigs.flags] "moduleconfigs.flags five" + + r config set moduleconfigs.flags "one four" + assert_equal [r config get moduleconfigs.flags] "moduleconfigs.flags five" + + r config set moduleconfigs.flags "one two four" + assert_equal [r config get moduleconfigs.flags] "moduleconfigs.flags {five two}" + } + + test {Immutable flag works properly and rejected strings dont leak} { + # Configs flagged immutable should not allow sets + catch {[r config set moduleconfigs.immutable_bool yes]} e + assert_match {*can't set immutable config*} $e + catch {[r config set moduleconfigs.string rejectisfreed]} e + assert_match {*Cannot set string to 'rejectisfreed'*} $e + } + + test {Numeric limits work properly} { + # Configs over/under the limit shouldn't be allowed, and memory configs should only take memory values + catch {[r config set moduleconfigs.memory_numeric 200gb]} e + assert_match {*argument must be between*} $e + catch {[r config set moduleconfigs.memory_numeric -5]} e + assert_match {*argument must be a memory value*} $e + catch {[r config set moduleconfigs.numeric -10]} e + assert_match {*argument must be between*} $e + } + + test {Enums only able to be set to passed in values} { + # Module authors specify what values are valid for enums, check that only those values are ok on a set + catch {[r config set moduleconfigs.enum asdf]} e + assert_match {*must be one of the following*} $e + } + + test {test blocking of config registration and load outside of OnLoad} { + assert_equal [r block.register.configs.outside.onload] OK + } + + test {Unload removes module configs} { + r module unload moduleconfigs + assert_equal [r config get moduleconfigs.*] "" + r module load $testmodule + # these should have reverted back to their module specified values + assert_equal [r config get moduleconfigs.mutable_bool] "moduleconfigs.mutable_bool yes" + assert_equal [r config get moduleconfigs.immutable_bool] "moduleconfigs.immutable_bool no" + assert_equal [r config get moduleconfigs.memory_numeric] "moduleconfigs.memory_numeric 1024" + assert_equal [r config get moduleconfigs.string] "moduleconfigs.string {secret password}" + assert_equal [r config get moduleconfigs.enum] "moduleconfigs.enum one" + assert_equal [r config get moduleconfigs.flags] "moduleconfigs.flags {one two}" + assert_equal [r config get moduleconfigs.numeric] "moduleconfigs.numeric -1" + r module unload moduleconfigs + } + + test {test loadex functionality} { + r module loadex $testmodule CONFIG moduleconfigs.mutable_bool no CONFIG moduleconfigs.immutable_bool yes CONFIG moduleconfigs.memory_numeric 2mb CONFIG moduleconfigs.string tclortickle + assert_not_equal [lsearch [lmap x [r module list] {dict get $x name}] moduleconfigs] -1 + assert_equal [r config get moduleconfigs.mutable_bool] "moduleconfigs.mutable_bool no" + assert_equal [r config get moduleconfigs.immutable_bool] "moduleconfigs.immutable_bool yes" + assert_equal [r config get moduleconfigs.memory_numeric] "moduleconfigs.memory_numeric 2097152" + assert_equal [r config get moduleconfigs.string] "moduleconfigs.string tclortickle" + # Configs that were not changed should still be their module specified value + assert_equal [r config get moduleconfigs.enum] "moduleconfigs.enum one" + assert_equal [r config get moduleconfigs.flags] "moduleconfigs.flags {one two}" + assert_equal [r config get moduleconfigs.numeric] "moduleconfigs.numeric -1" + } + + test {apply function works} { + catch {[r config set moduleconfigs.mutable_bool yes]} e + assert_match {*Bool configs*} $e + assert_equal [r config get moduleconfigs.mutable_bool] "moduleconfigs.mutable_bool no" + catch {[r config set moduleconfigs.memory_numeric 1000 moduleconfigs.numeric 1000]} e + assert_match {*cannot equal*} $e + assert_equal [r config get moduleconfigs.memory_numeric] "moduleconfigs.memory_numeric 2097152" + assert_equal [r config get moduleconfigs.numeric] "moduleconfigs.numeric -1" + r module unload moduleconfigs + } + + test {test double config argument to loadex} { + r module loadex $testmodule CONFIG moduleconfigs.mutable_bool yes CONFIG moduleconfigs.mutable_bool no + assert_equal [r config get moduleconfigs.mutable_bool] "moduleconfigs.mutable_bool no" + r module unload moduleconfigs + } + + test {missing loadconfigs call} { + catch {[r module loadex $testmodule CONFIG moduleconfigs.string "cool" ARGS noload]} e + assert_match {*ERR*} $e + } + + test {test loadex rejects bad configs} { + # Bad config 200gb is over the limit + catch {[r module loadex $testmodule CONFIG moduleconfigs.memory_numeric 200gb ARGS]} e + assert_match {*ERR*} $e + # We should completely remove all configs on a failed load + assert_equal [r config get moduleconfigs.*] "" + # No value for config, should error out + catch {[r module loadex $testmodule CONFIG moduleconfigs.mutable_bool CONFIG moduleconfigs.enum two ARGS]} e + assert_match {*ERR*} $e + assert_equal [r config get moduleconfigs.*] "" + # Asan will catch this if this string is not freed + catch {[r module loadex $testmodule CONFIG moduleconfigs.string rejectisfreed]} + assert_match {*ERR*} $e + assert_equal [r config get moduleconfigs.*] "" + # test we can't set random configs + catch {[r module loadex $testmodule CONFIG maxclients 333]} + assert_match {*ERR*} $e + assert_equal [r config get moduleconfigs.*] "" + assert_not_equal [r config get maxclients] "maxclients 333" + # test we can't set other module's configs + r module load $testmoduletwo + catch {[r module loadex $testmodule CONFIG configs.test no]} + assert_match {*ERR*} $e + assert_equal [r config get configs.test] "configs.test yes" + r module unload configs + } + + test {test config rewrite with dynamic load} { + #translates to: super \0secret password + r module loadex $testmodule CONFIG moduleconfigs.string \x73\x75\x70\x65\x72\x20\x00\x73\x65\x63\x72\x65\x74\x20\x70\x61\x73\x73\x77\x6f\x72\x64 ARGS + assert_not_equal [lsearch [lmap x [r module list] {dict get $x name}] moduleconfigs] -1 + assert_equal [r config get moduleconfigs.string] "moduleconfigs.string {super \0secret password}" + r config set moduleconfigs.mutable_bool yes + r config set moduleconfigs.memory_numeric 750 + r config set moduleconfigs.enum two + r config set moduleconfigs.flags "four two" + r config rewrite + restart_server 0 true false + # Ensure configs we rewrote are present and that the conf file is readable + assert_equal [r config get moduleconfigs.mutable_bool] "moduleconfigs.mutable_bool yes" + assert_equal [r config get moduleconfigs.memory_numeric] "moduleconfigs.memory_numeric 750" + assert_equal [r config get moduleconfigs.string] "moduleconfigs.string {super \0secret password}" + assert_equal [r config get moduleconfigs.enum] "moduleconfigs.enum two" + assert_equal [r config get moduleconfigs.flags] "moduleconfigs.flags {two four}" + assert_equal [r config get moduleconfigs.numeric] "moduleconfigs.numeric -1" + r module unload moduleconfigs + } + + test {test multiple modules with configs} { + r module load $testmodule + r module loadex $testmoduletwo CONFIG configs.test yes + assert_equal [r config get moduleconfigs.mutable_bool] "moduleconfigs.mutable_bool yes" + assert_equal [r config get moduleconfigs.immutable_bool] "moduleconfigs.immutable_bool no" + assert_equal [r config get moduleconfigs.memory_numeric] "moduleconfigs.memory_numeric 1024" + assert_equal [r config get moduleconfigs.string] "moduleconfigs.string {secret password}" + assert_equal [r config get moduleconfigs.enum] "moduleconfigs.enum one" + assert_equal [r config get moduleconfigs.numeric] "moduleconfigs.numeric -1" + assert_equal [r config get configs.test] "configs.test yes" + r config set moduleconfigs.mutable_bool no + r config set moduleconfigs.string nice + r config set moduleconfigs.enum two + r config set configs.test no + assert_equal [r config get moduleconfigs.mutable_bool] "moduleconfigs.mutable_bool no" + assert_equal [r config get moduleconfigs.string] "moduleconfigs.string nice" + assert_equal [r config get moduleconfigs.enum] "moduleconfigs.enum two" + assert_equal [r config get configs.test] "configs.test no" + r config rewrite + # test we can load from conf file with multiple different modules. + restart_server 0 true false + assert_equal [r config get moduleconfigs.mutable_bool] "moduleconfigs.mutable_bool no" + assert_equal [r config get moduleconfigs.string] "moduleconfigs.string nice" + assert_equal [r config get moduleconfigs.enum] "moduleconfigs.enum two" + assert_equal [r config get configs.test] "configs.test no" + r module unload moduleconfigs + r module unload configs + } + + test {test 1.module load 2.config rewrite 3.module unload 4.config rewrite works} { + # Configs need to be removed from the old config file in this case. + r module loadex $testmodule CONFIG moduleconfigs.memory_numeric 500 ARGS + assert_not_equal [lsearch [lmap x [r module list] {dict get $x name}] moduleconfigs] -1 + r config rewrite + r module unload moduleconfigs + r config rewrite + restart_server 0 true false + # Ensure configs we rewrote are no longer present + assert_equal [r config get moduleconfigs.*] "" + } + test {startup moduleconfigs} { + # No loadmodule directive + catch {exec src/redis-server --moduleconfigs.string "hello"} err + assert_match {*Module Configuration detected without loadmodule directive or no ApplyConfig call: aborting*} $err + + # Bad config value + catch {exec src/redis-server --loadmodule "$testmodule" --moduleconfigs.string "rejectisfreed"} err + assert_match {*Issue during loading of configuration moduleconfigs.string : Cannot set string to 'rejectisfreed'*} $err + + # missing LoadConfigs call + catch {exec src/redis-server --loadmodule "$testmodule" noload --moduleconfigs.string "hello"} err + assert_match {*Module Configurations were not set, likely a missing LoadConfigs call. Unloading the module.*} $err + + # successful + start_server [list overrides [list loadmodule "$testmodule" moduleconfigs.string "bootedup" moduleconfigs.enum two moduleconfigs.flags "two four"]] { + assert_equal [r config get moduleconfigs.string] "moduleconfigs.string bootedup" + assert_equal [r config get moduleconfigs.mutable_bool] "moduleconfigs.mutable_bool yes" + assert_equal [r config get moduleconfigs.immutable_bool] "moduleconfigs.immutable_bool no" + assert_equal [r config get moduleconfigs.enum] "moduleconfigs.enum two" + assert_equal [r config get moduleconfigs.flags] "moduleconfigs.flags {two four}" + assert_equal [r config get moduleconfigs.numeric] "moduleconfigs.numeric -1" + assert_equal [r config get moduleconfigs.memory_numeric] "moduleconfigs.memory_numeric 1024" + } + } +} + diff --git a/tests/unit/moduleapi/postnotifications.tcl b/tests/unit/moduleapi/postnotifications.tcl new file mode 100644 index 0000000..7e48c7b --- /dev/null +++ b/tests/unit/moduleapi/postnotifications.tcl @@ -0,0 +1,219 @@ +set testmodule [file normalize tests/modules/postnotifications.so] + +tags "modules" { + start_server {} { + r module load $testmodule with_key_events + + test {Test write on post notification callback} { + set repl [attach_to_replication_stream] + + r set string_x 1 + assert_equal {1} [r get string_changed{string_x}] + assert_equal {1} [r get string_total] + + r set string_x 2 + assert_equal {2} [r get string_changed{string_x}] + assert_equal {2} [r get string_total] + + # the {lpush before_overwritten string_x} is a post notification job registered when 'string_x' was overwritten + assert_replication_stream $repl { + {multi} + {select *} + {set string_x 1} + {incr string_changed{string_x}} + {incr string_total} + {exec} + {multi} + {set string_x 2} + {lpush before_overwritten string_x} + {incr string_changed{string_x}} + {incr string_total} + {exec} + } + close_replication_stream $repl + } + + test {Test write on post notification callback from module thread} { + r flushall + set repl [attach_to_replication_stream] + + assert_equal {OK} [r postnotification.async_set] + assert_equal {1} [r get string_changed{string_x}] + assert_equal {1} [r get string_total] + + assert_replication_stream $repl { + {multi} + {select *} + {set string_x 1} + {incr string_changed{string_x}} + {incr string_total} + {exec} + } + close_replication_stream $repl + } + + test {Test active expire} { + r flushall + set repl [attach_to_replication_stream] + + r set x 1 + r pexpire x 10 + + wait_for_condition 100 50 { + [r keys expired] == {expired} + } else { + puts [r keys *] + fail "Failed waiting for x to expired" + } + + # the {lpush before_expired x} is a post notification job registered before 'x' got expired + assert_replication_stream $repl { + {select *} + {set x 1} + {pexpireat x *} + {multi} + {del x} + {lpush before_expired x} + {incr expired} + {exec} + } + close_replication_stream $repl + } + + test {Test lazy expire} { + r flushall + r DEBUG SET-ACTIVE-EXPIRE 0 + set repl [attach_to_replication_stream] + + r set x 1 + r pexpire x 1 + after 10 + assert_equal {} [r get x] + + # the {lpush before_expired x} is a post notification job registered before 'x' got expired + assert_replication_stream $repl { + {select *} + {set x 1} + {pexpireat x *} + {multi} + {del x} + {lpush before_expired x} + {incr expired} + {exec} + } + close_replication_stream $repl + r DEBUG SET-ACTIVE-EXPIRE 1 + } {OK} {needs:debug} + + test {Test lazy expire inside post job notification} { + r flushall + r DEBUG SET-ACTIVE-EXPIRE 0 + set repl [attach_to_replication_stream] + + r set x 1 + r pexpire x 1 + after 10 + assert_equal {OK} [r set read_x 1] + + # the {lpush before_expired x} is a post notification job registered before 'x' got expired + assert_replication_stream $repl { + {select *} + {set x 1} + {pexpireat x *} + {multi} + {set read_x 1} + {del x} + {lpush before_expired x} + {incr expired} + {exec} + } + close_replication_stream $repl + r DEBUG SET-ACTIVE-EXPIRE 1 + } {OK} {needs:debug} + + test {Test nested keyspace notification} { + r flushall + set repl [attach_to_replication_stream] + + assert_equal {OK} [r set write_sync_write_sync_x 1] + + assert_replication_stream $repl { + {multi} + {select *} + {set x 1} + {set write_sync_x 1} + {set write_sync_write_sync_x 1} + {exec} + } + close_replication_stream $repl + } + + test {Test eviction} { + r flushall + set repl [attach_to_replication_stream] + r set x 1 + r config set maxmemory-policy allkeys-random + r config set maxmemory 1 + + assert_error {OOM *} {r set y 1} + + # the {lpush before_evicted x} is a post notification job registered before 'x' got evicted + assert_replication_stream $repl { + {select *} + {set x 1} + {multi} + {del x} + {lpush before_evicted x} + {incr evicted} + {exec} + } + close_replication_stream $repl + } {} {needs:config-maxmemory} + } +} + +set testmodule2 [file normalize tests/modules/keyspace_events.so] + +tags "modules" { + start_server {} { + r module load $testmodule with_key_events + r module load $testmodule2 + test {Test write on post notification callback} { + set repl [attach_to_replication_stream] + + r set string_x 1 + assert_equal {1} [r get string_changed{string_x}] + assert_equal {1} [r get string_total] + + r set string_x 2 + assert_equal {2} [r get string_changed{string_x}] + assert_equal {2} [r get string_total] + + r set string1_x 1 + assert_equal {1} [r get string_changed{string1_x}] + assert_equal {3} [r get string_total] + + # the {lpush before_overwritten string_x} is a post notification job registered before 'string_x' got overwritten + assert_replication_stream $repl { + {multi} + {select *} + {set string_x 1} + {incr string_changed{string_x}} + {incr string_total} + {exec} + {multi} + {set string_x 2} + {lpush before_overwritten string_x} + {incr string_changed{string_x}} + {incr string_total} + {exec} + {multi} + {set string1_x 1} + {incr string_changed{string1_x}} + {incr string_total} + {exec} + } + close_replication_stream $repl + } + } +} diff --git a/tests/unit/moduleapi/propagate.tcl b/tests/unit/moduleapi/propagate.tcl new file mode 100644 index 0000000..90a369d --- /dev/null +++ b/tests/unit/moduleapi/propagate.tcl @@ -0,0 +1,763 @@ +set testmodule [file normalize tests/modules/propagate.so] +set miscmodule [file normalize tests/modules/misc.so] +set keyspace_events [file normalize tests/modules/keyspace_events.so] + +tags "modules" { + test {Modules can propagate in async and threaded contexts} { + start_server [list overrides [list loadmodule "$testmodule"]] { + set replica [srv 0 client] + set replica_host [srv 0 host] + set replica_port [srv 0 port] + $replica module load $keyspace_events + start_server [list overrides [list loadmodule "$testmodule"]] { + set master [srv 0 client] + set master_host [srv 0 host] + set master_port [srv 0 port] + $master module load $keyspace_events + + # Start the replication process... + $replica replicaof $master_host $master_port + wait_for_sync $replica + after 1000 + + test {module propagates from timer} { + set repl [attach_to_replication_stream] + + $master propagate-test.timer + + wait_for_condition 500 10 { + [$replica get timer] eq "3" + } else { + fail "The two counters don't match the expected value." + } + + assert_replication_stream $repl { + {select *} + {incr timer} + {incr timer} + {incr timer} + } + close_replication_stream $repl + } + + test {module propagation with notifications} { + set repl [attach_to_replication_stream] + + $master set x y + + assert_replication_stream $repl { + {multi} + {select *} + {incr notifications} + {set x y} + {exec} + } + close_replication_stream $repl + } + + test {module propagation with notifications with multi} { + set repl [attach_to_replication_stream] + + $master multi + $master set x1 y1 + $master set x2 y2 + $master exec + + assert_replication_stream $repl { + {multi} + {select *} + {incr notifications} + {set x1 y1} + {incr notifications} + {set x2 y2} + {exec} + } + close_replication_stream $repl + } + + test {module propagation with notifications with active-expire} { + $master debug set-active-expire 1 + set repl [attach_to_replication_stream] + + $master set asdf1 1 PX 300 + $master set asdf2 2 PX 300 + $master set asdf3 3 PX 300 + + wait_for_condition 500 10 { + [$replica keys asdf*] eq {} + } else { + fail "Not all keys have expired" + } + + # Note whenever there's double notification: SET with PX issues two separate + # notifications: one for "set" and one for "expire" + assert_replication_stream $repl { + {multi} + {select *} + {incr notifications} + {incr notifications} + {set asdf1 1 PXAT *} + {exec} + {multi} + {incr notifications} + {incr notifications} + {set asdf2 2 PXAT *} + {exec} + {multi} + {incr notifications} + {incr notifications} + {set asdf3 3 PXAT *} + {exec} + {multi} + {incr notifications} + {incr notifications} + {incr testkeyspace:expired} + {del asdf*} + {exec} + {multi} + {incr notifications} + {incr notifications} + {incr testkeyspace:expired} + {del asdf*} + {exec} + {multi} + {incr notifications} + {incr notifications} + {incr testkeyspace:expired} + {del asdf*} + {exec} + } + close_replication_stream $repl + + $master debug set-active-expire 0 + } + + test {module propagation with notifications with eviction case 1} { + $master flushall + $master set asdf1 1 + $master set asdf2 2 + $master set asdf3 3 + + $master config set maxmemory-policy allkeys-random + $master config set maxmemory 1 + + # Please note the following loop: + # We evict a key and send a notification, which does INCR on the "notifications" key, so + # that every time we evict any key, "notifications" key exist (it happens inside the + # performEvictions loop). So even evicting "notifications" causes INCR on "notifications". + # If maxmemory_eviction_tenacity would have been set to 100 this would be an endless loop, but + # since the default is 10, at some point the performEvictions loop would end. + # Bottom line: "notifications" always exists and we can't really determine the order of evictions + # This test is here only for sanity + + # The replica will get the notification with multi exec and we have a generic notification handler + # that performs `RedisModule_Call(ctx, "INCR", "c", "multi");` if the notification is inside multi exec. + # so we will have 2 keys, "notifications" and "multi". + wait_for_condition 500 10 { + [$replica dbsize] eq 2 + } else { + fail "Not all keys have been evicted" + } + + $master config set maxmemory 0 + $master config set maxmemory-policy noeviction + } + + test {module propagation with notifications with eviction case 2} { + $master flushall + set repl [attach_to_replication_stream] + + $master set asdf1 1 EX 300 + $master set asdf2 2 EX 300 + $master set asdf3 3 EX 300 + + # Please note we use volatile eviction to prevent the loop described in the test above. + # "notifications" is not volatile so it always remains + $master config resetstat + $master config set maxmemory-policy volatile-ttl + $master config set maxmemory 1 + + wait_for_condition 500 10 { + [s evicted_keys] eq 3 + } else { + fail "Not all keys have been evicted" + } + + $master config set maxmemory 0 + $master config set maxmemory-policy noeviction + + $master set asdf4 4 + + # Note whenever there's double notification: SET with EX issues two separate + # notifications: one for "set" and one for "expire" + # Note that although CONFIG SET maxmemory is called in this flow (see issue #10014), + # eviction will happen and will not induce propagation of the CONFIG command (see #10019). + assert_replication_stream $repl { + {multi} + {select *} + {incr notifications} + {incr notifications} + {set asdf1 1 PXAT *} + {exec} + {multi} + {incr notifications} + {incr notifications} + {set asdf2 2 PXAT *} + {exec} + {multi} + {incr notifications} + {incr notifications} + {set asdf3 3 PXAT *} + {exec} + {multi} + {incr notifications} + {del asdf*} + {exec} + {multi} + {incr notifications} + {del asdf*} + {exec} + {multi} + {incr notifications} + {del asdf*} + {exec} + {multi} + {incr notifications} + {set asdf4 4} + {exec} + } + close_replication_stream $repl + } + + test {module propagation with timer and CONFIG SET maxmemory} { + set repl [attach_to_replication_stream] + + $master config resetstat + $master config set maxmemory-policy volatile-random + + $master propagate-test.timer-maxmemory + + # Wait until the volatile keys are evicted + wait_for_condition 500 10 { + [s evicted_keys] eq 2 + } else { + fail "Not all keys have been evicted" + } + + assert_replication_stream $repl { + {multi} + {select *} + {incr notifications} + {incr notifications} + {set timer-maxmemory-volatile-start 1 PXAT *} + {incr timer-maxmemory-middle} + {incr notifications} + {incr notifications} + {set timer-maxmemory-volatile-end 1 PXAT *} + {exec} + {multi} + {incr notifications} + {del timer-maxmemory-volatile-*} + {exec} + {multi} + {incr notifications} + {del timer-maxmemory-volatile-*} + {exec} + } + close_replication_stream $repl + + $master config set maxmemory 0 + $master config set maxmemory-policy noeviction + } + + test {module propagation with timer and EVAL} { + set repl [attach_to_replication_stream] + + $master propagate-test.timer-eval + + assert_replication_stream $repl { + {multi} + {select *} + {incr notifications} + {incrby timer-eval-start 1} + {incr notifications} + {set foo bar} + {incr timer-eval-middle} + {incr notifications} + {incrby timer-eval-end 1} + {exec} + } + close_replication_stream $repl + } + + test {module propagates nested ctx case1} { + set repl [attach_to_replication_stream] + + $master propagate-test.timer-nested + + wait_for_condition 500 10 { + [$replica get timer-nested-end] eq "1" + } else { + fail "The two counters don't match the expected value." + } + + assert_replication_stream $repl { + {multi} + {select *} + {incrby timer-nested-start 1} + {incrby timer-nested-end 1} + {exec} + } + close_replication_stream $repl + + # Note propagate-test.timer-nested just propagates INCRBY, causing an + # inconsistency, so we flush + $master flushall + } + + test {module propagates nested ctx case2} { + set repl [attach_to_replication_stream] + + $master propagate-test.timer-nested-repl + + wait_for_condition 500 10 { + [$replica get timer-nested-end] eq "1" + } else { + fail "The two counters don't match the expected value." + } + + assert_replication_stream $repl { + {multi} + {select *} + {incrby timer-nested-start 1} + {incr notifications} + {incr using-call} + {incr counter-1} + {incr counter-2} + {incr counter-3} + {incr counter-4} + {incr notifications} + {incr after-call} + {incr notifications} + {incr before-call-2} + {incr notifications} + {incr asdf} + {incr notifications} + {del asdf} + {incr notifications} + {incr after-call-2} + {incr notifications} + {incr timer-nested-middle} + {incrby timer-nested-end 1} + {exec} + } + close_replication_stream $repl + + # Note propagate-test.timer-nested-repl just propagates INCRBY, causing an + # inconsistency, so we flush + $master flushall + } + + test {module propagates from thread} { + set repl [attach_to_replication_stream] + + $master propagate-test.thread + + wait_for_condition 500 10 { + [$replica get a-from-thread] eq "3" + } else { + fail "The two counters don't match the expected value." + } + + assert_replication_stream $repl { + {multi} + {select *} + {incr a-from-thread} + {incr notifications} + {incr thread-call} + {incr b-from-thread} + {exec} + {multi} + {incr a-from-thread} + {incr notifications} + {incr thread-call} + {incr b-from-thread} + {exec} + {multi} + {incr a-from-thread} + {incr notifications} + {incr thread-call} + {incr b-from-thread} + {exec} + } + close_replication_stream $repl + } + + test {module propagates from thread with detached ctx} { + set repl [attach_to_replication_stream] + + $master propagate-test.detached-thread + + wait_for_condition 500 10 { + [$replica get thread-detached-after] eq "1" + } else { + fail "The key doesn't match the expected value." + } + + assert_replication_stream $repl { + {multi} + {select *} + {incr thread-detached-before} + {incr notifications} + {incr thread-detached-1} + {incr notifications} + {incr thread-detached-2} + {incr thread-detached-after} + {exec} + } + close_replication_stream $repl + } + + test {module propagates from command} { + set repl [attach_to_replication_stream] + + $master propagate-test.simple + $master propagate-test.mixed + + assert_replication_stream $repl { + {multi} + {select *} + {incr counter-1} + {incr counter-2} + {exec} + {multi} + {incr notifications} + {incr using-call} + {incr counter-1} + {incr counter-2} + {incr notifications} + {incr after-call} + {exec} + } + close_replication_stream $repl + } + + test {module propagates from EVAL} { + set repl [attach_to_replication_stream] + + assert_equal [ $master eval { \ + redis.call("propagate-test.simple"); \ + redis.call("set", "x", "y"); \ + redis.call("propagate-test.mixed"); return "OK" } 0 ] {OK} + + assert_replication_stream $repl { + {multi} + {select *} + {incr counter-1} + {incr counter-2} + {incr notifications} + {set x y} + {incr notifications} + {incr using-call} + {incr counter-1} + {incr counter-2} + {incr notifications} + {incr after-call} + {exec} + } + close_replication_stream $repl + } + + test {module propagates from command after good EVAL} { + set repl [attach_to_replication_stream] + + assert_equal [ $master eval { return "hello" } 0 ] {hello} + $master propagate-test.simple + $master propagate-test.mixed + + assert_replication_stream $repl { + {multi} + {select *} + {incr counter-1} + {incr counter-2} + {exec} + {multi} + {incr notifications} + {incr using-call} + {incr counter-1} + {incr counter-2} + {incr notifications} + {incr after-call} + {exec} + } + close_replication_stream $repl + } + + test {module propagates from command after bad EVAL} { + set repl [attach_to_replication_stream] + + catch { $master eval { return "hello" } -12 } e + assert_equal $e {ERR Number of keys can't be negative} + $master propagate-test.simple + $master propagate-test.mixed + + assert_replication_stream $repl { + {multi} + {select *} + {incr counter-1} + {incr counter-2} + {exec} + {multi} + {incr notifications} + {incr using-call} + {incr counter-1} + {incr counter-2} + {incr notifications} + {incr after-call} + {exec} + } + close_replication_stream $repl + } + + test {module propagates from multi-exec} { + set repl [attach_to_replication_stream] + + $master multi + $master propagate-test.simple + $master propagate-test.mixed + $master propagate-test.timer-nested-repl + $master exec + + wait_for_condition 500 10 { + [$replica get timer-nested-end] eq "1" + } else { + fail "The two counters don't match the expected value." + } + + assert_replication_stream $repl { + {multi} + {select *} + {incr counter-1} + {incr counter-2} + {incr notifications} + {incr using-call} + {incr counter-1} + {incr counter-2} + {incr notifications} + {incr after-call} + {exec} + {multi} + {incrby timer-nested-start 1} + {incr notifications} + {incr using-call} + {incr counter-1} + {incr counter-2} + {incr counter-3} + {incr counter-4} + {incr notifications} + {incr after-call} + {incr notifications} + {incr before-call-2} + {incr notifications} + {incr asdf} + {incr notifications} + {del asdf} + {incr notifications} + {incr after-call-2} + {incr notifications} + {incr timer-nested-middle} + {incrby timer-nested-end 1} + {exec} + } + close_replication_stream $repl + + # Note propagate-test.timer-nested just propagates INCRBY, causing an + # inconsistency, so we flush + $master flushall + } + + test {module RM_Call of expired key propagation} { + $master debug set-active-expire 0 + + $master set k1 900 px 100 + after 110 + + set repl [attach_to_replication_stream] + $master propagate-test.incr k1 + + assert_replication_stream $repl { + {multi} + {select *} + {del k1} + {propagate-test.incr k1} + {exec} + } + close_replication_stream $repl + + assert_equal [$master get k1] 1 + assert_equal [$master ttl k1] -1 + assert_equal [$replica get k1] 1 + assert_equal [$replica ttl k1] -1 + } + + test {module notification on set} { + set repl [attach_to_replication_stream] + + $master SADD s foo + + wait_for_condition 500 10 { + [$replica SCARD s] eq "1" + } else { + fail "Failed to wait for set to be replicated" + } + + $master SPOP s 1 + + wait_for_condition 500 10 { + [$replica SCARD s] eq "0" + } else { + fail "Failed to wait for set to be replicated" + } + + # Currently the `del` command comes after the notification. + # When we fix spop to fire notification at the end (like all other commands), + # the `del` will come first. + assert_replication_stream $repl { + {multi} + {select *} + {incr notifications} + {sadd s foo} + {exec} + {multi} + {incr notifications} + {incr notifications} + {del s} + {exec} + } + close_replication_stream $repl + } + + test {module key miss notification do not cause read command to be replicated} { + set repl [attach_to_replication_stream] + + $master flushall + + $master get unexisting_key + + wait_for_condition 500 10 { + [$replica get missed] eq "1" + } else { + fail "Failed to wait for set to be replicated" + } + + # Test is checking a wrong!!! behavior that causes a read command to be replicated to replica/aof. + # We keep the test to verify that such a wrong behavior does not cause any crashes. + assert_replication_stream $repl { + {select *} + {flushall} + {multi} + {incr notifications} + {incr missed} + {get unexisting_key} + {exec} + } + + close_replication_stream $repl + } + + test "Unload the module - propagate-test/testkeyspace" { + assert_equal {OK} [r module unload propagate-test] + assert_equal {OK} [r module unload testkeyspace] + } + + assert_equal [s -1 unexpected_error_replies] 0 + } + } + } +} + + +tags "modules aof" { + foreach aofload_type {debug_cmd startup} { + test "Modules RM_Replicate replicates MULTI/EXEC correctly: AOF-load type $aofload_type" { + start_server [list overrides [list loadmodule "$testmodule"]] { + # Enable the AOF + r config set appendonly yes + r config set auto-aof-rewrite-percentage 0 ; # Disable auto-rewrite. + waitForBgrewriteaof r + + r propagate-test.simple + r propagate-test.mixed + r multi + r propagate-test.simple + r propagate-test.mixed + r exec + + assert_equal [r get counter-1] {} + assert_equal [r get counter-2] {} + assert_equal [r get using-call] 2 + assert_equal [r get after-call] 2 + assert_equal [r get notifications] 4 + + # Load the AOF + if {$aofload_type == "debug_cmd"} { + r debug loadaof + } else { + r config rewrite + restart_server 0 true false + wait_done_loading r + } + + # This module behaves bad on purpose, it only calls + # RM_Replicate for counter-1 and counter-2 so values + # after AOF-load are different + assert_equal [r get counter-1] 4 + assert_equal [r get counter-2] 4 + assert_equal [r get using-call] 2 + assert_equal [r get after-call] 2 + # 4+4+2+2 commands from AOF (just above) + 4 "INCR notifications" from AOF + 4 notifications for these INCRs + assert_equal [r get notifications] 20 + + assert_equal {OK} [r module unload propagate-test] + assert_equal [s 0 unexpected_error_replies] 0 + } + } + test "Modules RM_Call does not update stats during aof load: AOF-load type $aofload_type" { + start_server [list overrides [list loadmodule "$miscmodule"]] { + # Enable the AOF + r config set appendonly yes + r config set auto-aof-rewrite-percentage 0 ; # Disable auto-rewrite. + waitForBgrewriteaof r + + r config resetstat + r set foo bar + r EVAL {return redis.call('SET', KEYS[1], ARGV[1])} 1 foo bar2 + r test.rm_call_replicate set foo bar3 + r EVAL {return redis.call('test.rm_call_replicate',ARGV[1],KEYS[1],ARGV[2])} 1 foo set bar4 + + r multi + r set foo bar5 + r EVAL {return redis.call('SET', KEYS[1], ARGV[1])} 1 foo bar6 + r test.rm_call_replicate set foo bar7 + r EVAL {return redis.call('test.rm_call_replicate',ARGV[1],KEYS[1],ARGV[2])} 1 foo set bar8 + r exec + + assert_match {*calls=8,*,rejected_calls=0,failed_calls=0} [cmdrstat set r] + + + # Load the AOF + if {$aofload_type == "debug_cmd"} { + r config resetstat + r debug loadaof + } else { + r config rewrite + restart_server 0 true false + wait_done_loading r + } + + assert_no_match {*calls=*} [cmdrstat set r] + + } + } + } +} diff --git a/tests/unit/moduleapi/publish.tcl b/tests/unit/moduleapi/publish.tcl new file mode 100644 index 0000000..a6304ea --- /dev/null +++ b/tests/unit/moduleapi/publish.tcl @@ -0,0 +1,34 @@ +set testmodule [file normalize tests/modules/publish.so] + +start_server {tags {"modules"}} { + r module load $testmodule + + test {PUBLISH and SPUBLISH via a module} { + set rd1 [redis_deferring_client] + set rd2 [redis_deferring_client] + + assert_equal {1} [ssubscribe $rd1 {chan1}] + assert_equal {1} [subscribe $rd2 {chan1}] + assert_equal 1 [r publish.shard chan1 hello] + assert_equal 1 [r publish.classic chan1 world] + assert_equal {smessage chan1 hello} [$rd1 read] + assert_equal {message chan1 world} [$rd2 read] + $rd1 close + $rd2 close + } + + test {module publish to self with multi message} { + r hello 3 + r subscribe foo + + # published message comes after the response of the command that issued it. + assert_equal [r publish.classic_multi foo bar vaz] {1 1} + assert_equal [r read] {message foo bar} + assert_equal [r read] {message foo vaz} + + r unsubscribe foo + r hello 2 + set _ "" + } {} {resp3} + +} diff --git a/tests/unit/moduleapi/rdbloadsave.tcl b/tests/unit/moduleapi/rdbloadsave.tcl new file mode 100644 index 0000000..9319c93 --- /dev/null +++ b/tests/unit/moduleapi/rdbloadsave.tcl @@ -0,0 +1,200 @@ +set testmodule [file normalize tests/modules/rdbloadsave.so] + +start_server {tags {"modules"}} { + r module load $testmodule + + test "Module rdbloadsave sanity" { + r test.sanity + + # Try to load non-existing file + assert_error {*No such file or directory*} {r test.rdbload sanity.rdb} + + r set x 1 + assert_equal OK [r test.rdbsave sanity.rdb] + + r flushdb + assert_equal OK [r test.rdbload sanity.rdb] + assert_equal 1 [r get x] + } + + test "Module rdbloadsave test with pipelining" { + r config set save "" + r config set loading-process-events-interval-bytes 1024 + r config set key-load-delay 50 + r flushdb + + populate 3000 a 1024 + r set x 111 + assert_equal [r dbsize] 3001 + + assert_equal OK [r test.rdbsave blabla.rdb] + r flushdb + assert_equal [r dbsize] 0 + + # Send commands with pipeline. First command will call RM_RdbLoad() in + # the command callback. While loading RDB, Redis can go to networking to + # reply -LOADING. By sending commands in pipeline, we verify it doesn't + # cause a problem. + # e.g. Redis won't try to process next message of the current client + # while it is in the command callback for that client . + set rd1 [redis_deferring_client] + $rd1 test.rdbload blabla.rdb + + wait_for_condition 50 100 { + [s loading] eq 1 + } else { + fail "Redis did not start loading or loaded RDB too fast" + } + + $rd1 get x + $rd1 dbsize + + assert_equal OK [$rd1 read] + assert_equal 111 [$rd1 read] + assert_equal 3001 [$rd1 read] + r flushdb + r config set key-load-delay 0 + } + + test "Module rdbloadsave with aof" { + r config set save "" + + # Enable the AOF + r config set appendonly yes + r config set auto-aof-rewrite-percentage 0 ; # Disable auto-rewrite. + waitForBgrewriteaof r + + r set k v1 + assert_equal OK [r test.rdbsave aoftest.rdb] + + r set k v2 + r config set rdb-key-save-delay 10000000 + r bgrewriteaof + + # RM_RdbLoad() should kill aof fork + assert_equal OK [r test.rdbload aoftest.rdb] + + wait_for_condition 50 100 { + [string match {*Killing*AOF*child*} [exec tail -20 < [srv 0 stdout]]] + } else { + fail "Can't find 'Killing AOF child' in recent log lines" + } + + # Verify the value in the loaded rdb + assert_equal v1 [r get k] + + r flushdb + r config set rdb-key-save-delay 0 + r config set appendonly no + } + + test "Module rdbloadsave with bgsave" { + r flushdb + r config set save "" + + r set k v1 + assert_equal OK [r test.rdbsave bgsave.rdb] + + r set k v2 + r config set rdb-key-save-delay 500000 + r bgsave + + # RM_RdbLoad() should kill RDB fork + assert_equal OK [r test.rdbload bgsave.rdb] + + wait_for_condition 10 1000 { + [string match {*Background*saving*terminated*} [exec tail -20 < [srv 0 stdout]]] + } else { + fail "Can't find 'Background saving terminated' in recent log lines" + } + + assert_equal v1 [r get k] + r flushall + waitForBgsave r + r config set rdb-key-save-delay 0 + } + + test "Module rdbloadsave calls rdbsave in a module fork" { + r flushdb + r config set save "" + r config set rdb-key-save-delay 500000 + + r set k v1 + + # Module will call RM_Fork() before calling RM_RdbSave() + assert_equal OK [r test.rdbsave_fork rdbfork.rdb] + assert_equal [s module_fork_in_progress] 1 + + wait_for_condition 10 1000 { + [status r module_fork_in_progress] == "0" + } else { + fail "Module fork didn't finish" + } + + r set k v2 + assert_equal OK [r test.rdbload rdbfork.rdb] + assert_equal v1 [r get k] + + r config set rdb-key-save-delay 0 + } + + test "Unload the module - rdbloadsave" { + assert_equal {OK} [r module unload rdbloadsave] + } + + tags {repl} { + test {Module rdbloadsave on master and replica} { + start_server [list overrides [list loadmodule "$testmodule"]] { + set replica [srv 0 client] + set replica_host [srv 0 host] + set replica_port [srv 0 port] + start_server [list overrides [list loadmodule "$testmodule"]] { + set master [srv 0 client] + set master_host [srv 0 host] + set master_port [srv 0 port] + + $master set x 10000 + + # Start the replication process... + $replica replicaof $master_host $master_port + + wait_for_condition 100 100 { + [status $master sync_full] == 1 + } else { + fail "Master <-> Replica didn't start the full sync" + } + + # RM_RdbSave() is allowed on replicas + assert_equal OK [$replica test.rdbsave rep.rdb] + + # RM_RdbLoad() is not allowed on replicas + assert_error {*supported*} {$replica test.rdbload rep.rdb} + + assert_equal OK [$master test.rdbsave master.rdb] + $master set x 20000 + + wait_for_condition 100 100 { + [$replica get x] == 20000 + } else { + fail "Replica didn't get the update" + } + + # Loading RDB on master will drop replicas + assert_equal OK [$master test.rdbload master.rdb] + + wait_for_condition 100 100 { + [status $master sync_full] == 2 + } else { + fail "Master <-> Replica didn't start the full sync" + } + + wait_for_condition 100 100 { + [$replica get x] == 10000 + } else { + fail "Replica didn't get the update" + } + } + } + } + } +} diff --git a/tests/unit/moduleapi/reply.tcl b/tests/unit/moduleapi/reply.tcl new file mode 100644 index 0000000..3cf284d --- /dev/null +++ b/tests/unit/moduleapi/reply.tcl @@ -0,0 +1,152 @@ +set testmodule [file normalize tests/modules/reply.so] + +start_server {tags {"modules"}} { + r module load $testmodule + + # test all with hello 2/3 + for {set proto 2} {$proto <= 3} {incr proto} { + if {[lsearch $::denytags "resp3"] >= 0} { + if {$proto == 3} {continue} + } elseif {$::force_resp3} { + if {$proto == 2} {continue} + } + r hello $proto + + test "RESP$proto: RM_ReplyWithString: an string reply" { + # RedisString + set string [r rw.string "Redis"] + assert_equal "Redis" $string + # C string + set string [r rw.cstring] + assert_equal "A simple string" $string + } + + test "RESP$proto: RM_ReplyWithBigNumber: an string reply" { + assert_equal "123456778901234567890" [r rw.bignumber "123456778901234567890"] + } + + test "RESP$proto: RM_ReplyWithInt: an integer reply" { + assert_equal 42 [r rw.int 42] + } + + test "RESP$proto: RM_ReplyWithDouble: a float reply" { + assert_equal 3.141 [r rw.double 3.141] + } + + test "RESP$proto: RM_ReplyWithDouble: inf" { + if {$proto == 2} { + assert_equal "inf" [r rw.double inf] + assert_equal "-inf" [r rw.double -inf] + } else { + # TCL convert inf to different results on different platforms, e.g. inf on mac + # and Inf on others, so use readraw to verify the protocol + r readraw 1 + assert_equal ",inf" [r rw.double inf] + assert_equal ",-inf" [r rw.double -inf] + r readraw 0 + } + } + + test "RESP$proto: RM_ReplyWithDouble: NaN" { + if {$proto == 2} { + assert_equal "nan" [r rw.double 0 0] + assert_equal "nan" [r rw.double] + } else { + # TCL won't convert nan into a double, use readraw to verify the protocol + r readraw 1 + assert_equal ",nan" [r rw.double 0 0] + assert_equal ",nan" [r rw.double] + r readraw 0 + } + } + + set ld 0.00000000000000001 + test "RESP$proto: RM_ReplyWithLongDouble: a float reply" { + if {$proto == 2} { + # here the response gets to TCL as a string + assert_equal $ld [r rw.longdouble $ld] + } else { + # TCL doesn't support long double and the test infra converts it to a + # normal double which causes precision loss. so we use readraw instead + r readraw 1 + assert_equal ",$ld" [r rw.longdouble $ld] + r readraw 0 + } + } + + test "RESP$proto: RM_ReplyWithVerbatimString: a string reply" { + assert_equal "bla\nbla\nbla" [r rw.verbatim "bla\nbla\nbla"] + } + + test "RESP$proto: RM_ReplyWithArray: an array reply" { + assert_equal {0 1 2 3 4} [r rw.array 5] + } + + test "RESP$proto: RM_ReplyWithMap: an map reply" { + set res [r rw.map 3] + if {$proto == 2} { + assert_equal {0 0 1 1.5 2 3} $res + } else { + assert_equal [dict create 0 0.0 1 1.5 2 3.0] $res + } + } + + test "RESP$proto: RM_ReplyWithSet: an set reply" { + assert_equal {0 1 2} [r rw.set 3] + } + + test "RESP$proto: RM_ReplyWithAttribute: an set reply" { + if {$proto == 2} { + catch {[r rw.attribute 3]} e + assert_match "Attributes aren't supported by RESP 2" $e + } else { + r readraw 1 + set res [r rw.attribute 3] + assert_equal [r read] {:0} + assert_equal [r read] {,0} + assert_equal [r read] {:1} + assert_equal [r read] {,1.5} + assert_equal [r read] {:2} + assert_equal [r read] {,3} + assert_equal [r read] {+OK} + r readraw 0 + } + } + + test "RESP$proto: RM_ReplyWithBool: a boolean reply" { + assert_equal {0 1} [r rw.bool] + } + + test "RESP$proto: RM_ReplyWithNull: a NULL reply" { + assert_equal {} [r rw.null] + } + + test "RESP$proto: RM_ReplyWithError: an error reply" { + catch {r rw.error} e + assert_match "An error" $e + } + + test "RESP$proto: RM_ReplyWithErrorFormat: error format reply" { + catch {r rw.error_format "An error: %s" foo} e + assert_match "An error: foo" $e ;# Should not be used by a user, but compatible with RM_ReplyError + + catch {r rw.error_format "-ERR An error: %s" foo2} e + assert_match "-ERR An error: foo2" $e ;# Should not be used by a user, but compatible with RM_ReplyError (There are two hyphens, TCL removes the first one) + + catch {r rw.error_format "-WRONGTYPE A type error: %s" foo3} e + assert_match "-WRONGTYPE A type error: foo3" $e ;# Should not be used by a user, but compatible with RM_ReplyError (There are two hyphens, TCL removes the first one) + + catch {r rw.error_format "ERR An error: %s" foo4} e + assert_match "ERR An error: foo4" $e + + catch {r rw.error_format "WRONGTYPE A type error: %s" foo5} e + assert_match "WRONGTYPE A type error: foo5" $e + } + + r hello 2 + } + + test "Unload the module - replywith" { + assert_equal {OK} [r module unload replywith] + } +} diff --git a/tests/unit/moduleapi/scan.tcl b/tests/unit/moduleapi/scan.tcl new file mode 100644 index 0000000..1efd6ac --- /dev/null +++ b/tests/unit/moduleapi/scan.tcl @@ -0,0 +1,69 @@ +set testmodule [file normalize tests/modules/scan.so] + +start_server {tags {"modules"}} { + r module load $testmodule + + test {Module scan keyspace} { + # the module create a scan command with filtering which also return values + r set x 1 + r set y 2 + r set z 3 + r hset h f v + lsort [r scan.scan_strings] + } {{x 1} {y 2} {z 3}} + + test {Module scan hash listpack} { + r hmset hh f1 v1 f2 v2 + assert_encoding listpack hh + lsort [r scan.scan_key hh] + } {{f1 v1} {f2 v2}} + + test {Module scan hash listpack with int value} { + r hmset hh1 f1 1 + assert_encoding listpack hh1 + lsort [r scan.scan_key hh1] + } {{f1 1}} + + test {Module scan hash dict} { + r config set hash-max-ziplist-entries 2 + r hmset hh f3 v3 + assert_encoding hashtable hh + lsort [r scan.scan_key hh] + } {{f1 v1} {f2 v2} {f3 v3}} + + test {Module scan zset listpack} { + r zadd zz 1 f1 2 f2 + assert_encoding listpack zz + lsort [r scan.scan_key zz] + } {{f1 1} {f2 2}} + + test {Module scan zset skiplist} { + r config set zset-max-ziplist-entries 2 + r zadd zz 3 f3 + assert_encoding skiplist zz + lsort [r scan.scan_key zz] + } {{f1 1} {f2 2} {f3 3}} + + test {Module scan set intset} { + r sadd ss 1 2 + assert_encoding intset ss + lsort [r scan.scan_key ss] + } {{1 {}} {2 {}}} + + test {Module scan set dict} { + r config set set-max-intset-entries 2 + r sadd ss 3 + assert_encoding hashtable ss + lsort [r scan.scan_key ss] + } {{1 {}} {2 {}} {3 {}}} + + test {Module scan set listpack} { + r sadd ss1 a b c + assert_encoding listpack ss1 + lsort [r scan.scan_key ss1] + } {{a {}} {b {}} {c {}}} + + test "Unload the module - scan" { + assert_equal {OK} [r module unload scan] + } +}
\ No newline at end of file diff --git a/tests/unit/moduleapi/stream.tcl b/tests/unit/moduleapi/stream.tcl new file mode 100644 index 0000000..7ad1a30 --- /dev/null +++ b/tests/unit/moduleapi/stream.tcl @@ -0,0 +1,176 @@ +set testmodule [file normalize tests/modules/stream.so] + +start_server {tags {"modules"}} { + r module load $testmodule + + test {Module stream add and delete} { + r del mystream + # add to empty key + set streamid1 [r stream.add mystream item 1 value a] + # add to existing stream + set streamid2 [r stream.add mystream item 2 value b] + # check result + assert { [string match "*-*" $streamid1] } + set items [r XRANGE mystream - +] + assert_equal $items \ + "{$streamid1 {item 1 value a}} {$streamid2 {item 2 value b}}" + # delete one of them and try deleting non-existing ID + assert_equal OK [r stream.delete mystream $streamid1] + assert_error "ERR StreamDelete*" {r stream.delete mystream 123-456} + assert_error "Invalid stream ID*" {r stream.delete mystream foo} + assert_equal "{$streamid2 {item 2 value b}}" [r XRANGE mystream - +] + # check error condition: wrong type + r del mystream + r set mystream mystring + assert_error "ERR StreamAdd*" {r stream.add mystream item 1 value a} + assert_error "ERR StreamDelete*" {r stream.delete mystream 123-456} + } + + test {Module stream add unblocks blocking xread} { + r del mystream + + # Blocking XREAD on an empty key + set rd1 [redis_deferring_client] + $rd1 XREAD BLOCK 3000 STREAMS mystream $ + # wait until client is actually blocked + wait_for_condition 50 100 { + [s 0 blocked_clients] eq {1} + } else { + fail "Client is not blocked" + } + set id [r stream.add mystream field 1 value a] + assert_equal "{mystream {{$id {field 1 value a}}}}" [$rd1 read] + + # Blocking XREAD on an existing stream + set rd2 [redis_deferring_client] + $rd2 XREAD BLOCK 3000 STREAMS mystream $ + # wait until client is actually blocked + wait_for_condition 50 100 { + [s 0 blocked_clients] eq {1} + } else { + fail "Client is not blocked" + } + set id [r stream.add mystream field 2 value b] + assert_equal "{mystream {{$id {field 2 value b}}}}" [$rd2 read] + } + + test {Module stream add benchmark (1M stream add)} { + set n 1000000 + r del mystream + set result [r stream.addn mystream $n field value] + assert_equal $result $n + } + + test {Module stream XADD big fields doesn't create empty key} { + set original_proto [config_get_set proto-max-bulk-len 2147483647] ;#2gb + set original_query [config_get_set client-query-buffer-limit 2147483647] ;#2gb + + r del mystream + r write "*4\r\n\$10\r\nstream.add\r\n\$8\r\nmystream\r\n\$5\r\nfield\r\n" + catch { + write_big_bulk 1073741824 ;#1gb + } err + assert {$err eq "ERR StreamAdd failed"} + assert_equal 0 [r exists mystream] + + # restore defaults + r config set proto-max-bulk-len $original_proto + r config set client-query-buffer-limit $original_query + } {OK} {large-memory} + + test {Module stream iterator} { + r del mystream + set streamid1 [r xadd mystream * item 1 value a] + set streamid2 [r xadd mystream * item 2 value b] + # range result + set result1 [r stream.range mystream "-" "+"] + set expect1 [r xrange mystream "-" "+"] + assert_equal $result1 $expect1 + # reverse range + set result_rev [r stream.range mystream "+" "-"] + set expect_rev [r xrevrange mystream "+" "-"] + assert_equal $result_rev $expect_rev + + # only one item: range with startid = endid + set result2 [r stream.range mystream "-" $streamid1] + assert_equal $result2 "{$streamid1 {item 1 value a}}" + assert_equal $result2 [list [list $streamid1 {item 1 value a}]] + # only one item: range with startid = endid + set result3 [r stream.range mystream $streamid2 $streamid2] + assert_equal $result3 "{$streamid2 {item 2 value b}}" + assert_equal $result3 [list [list $streamid2 {item 2 value b}]] + } + + test {Module stream iterator delete} { + r del mystream + set id1 [r xadd mystream * normal item] + set id2 [r xadd mystream * selfdestruct yes] + set id3 [r xadd mystream * another item] + # stream.range deletes the "selfdestruct" item after returning it + assert_equal \ + "{$id1 {normal item}} {$id2 {selfdestruct yes}} {$id3 {another item}}" \ + [r stream.range mystream - +] + # now, the "selfdestruct" item is gone + assert_equal \ + "{$id1 {normal item}} {$id3 {another item}}" \ + [r stream.range mystream - +] + } + + test {Module stream trim by length} { + r del mystream + # exact maxlen + r xadd mystream * item 1 value a + r xadd mystream * item 2 value b + r xadd mystream * item 3 value c + assert_equal 3 [r xlen mystream] + assert_equal 0 [r stream.trim mystream maxlen = 5] + assert_equal 3 [r xlen mystream] + assert_equal 2 [r stream.trim mystream maxlen = 1] + assert_equal 1 [r xlen mystream] + assert_equal 1 [r stream.trim mystream maxlen = 0] + # check that there is no limit for exact maxlen + r stream.addn mystream 20000 item x value y + assert_equal 20000 [r stream.trim mystream maxlen = 0] + # approx maxlen (100 items per node implies default limit 10K items) + r stream.addn mystream 20000 item x value y + assert_equal 20000 [r xlen mystream] + assert_equal 10000 [r stream.trim mystream maxlen ~ 2] + assert_equal 9900 [r stream.trim mystream maxlen ~ 2] + assert_equal 0 [r stream.trim mystream maxlen ~ 2] + assert_equal 100 [r xlen mystream] + assert_equal 100 [r stream.trim mystream maxlen ~ 0] + assert_equal 0 [r xlen mystream] + } + + test {Module stream trim by ID} { + r del mystream + # exact minid + r xadd mystream * item 1 value a + r xadd mystream * item 2 value b + set minid [r xadd mystream * item 3 value c] + assert_equal 3 [r xlen mystream] + assert_equal 0 [r stream.trim mystream minid = -] + assert_equal 3 [r xlen mystream] + assert_equal 2 [r stream.trim mystream minid = $minid] + assert_equal 1 [r xlen mystream] + assert_equal 1 [r stream.trim mystream minid = +] + # check that there is no limit for exact minid + r stream.addn mystream 20000 item x value y + assert_equal 20000 [r stream.trim mystream minid = +] + # approx minid (100 items per node implies default limit 10K items) + r stream.addn mystream 19980 item x value y + set minid [r xadd mystream * item x value y] + r stream.addn mystream 19 item x value y + assert_equal 20000 [r xlen mystream] + assert_equal 10000 [r stream.trim mystream minid ~ $minid] + assert_equal 9900 [r stream.trim mystream minid ~ $minid] + assert_equal 0 [r stream.trim mystream minid ~ $minid] + assert_equal 100 [r xlen mystream] + assert_equal 100 [r stream.trim mystream minid ~ +] + assert_equal 0 [r xlen mystream] + } + + test "Unload the module - stream" { + assert_equal {OK} [r module unload stream] + } +} diff --git a/tests/unit/moduleapi/subcommands.tcl b/tests/unit/moduleapi/subcommands.tcl new file mode 100644 index 0000000..62de593 --- /dev/null +++ b/tests/unit/moduleapi/subcommands.tcl @@ -0,0 +1,57 @@ +set testmodule [file normalize tests/modules/subcommands.so] + +start_server {tags {"modules"}} { + r module load $testmodule + + test "Module subcommands via COMMAND" { + # Verify that module subcommands are displayed correctly in COMMAND + set command_reply [r command info subcommands.bitarray] + set first_cmd [lindex $command_reply 0] + set subcmds_in_command [lsort [lindex $first_cmd 9]] + assert_equal [lindex $subcmds_in_command 0] {subcommands.bitarray|get -2 module 1 1 1 {} {} {{flags {RO access} begin_search {type index spec {index 1}} find_keys {type range spec {lastkey 0 keystep 1 limit 0}}}} {}} + assert_equal [lindex $subcmds_in_command 1] {subcommands.bitarray|set -2 module 1 1 1 {} {} {{flags {RW update} begin_search {type index spec {index 1}} find_keys {type range spec {lastkey 0 keystep 1 limit 0}}}} {}} + + # Verify that module subcommands are displayed correctly in COMMAND DOCS + set docs_reply [r command docs subcommands.bitarray] + set docs [dict create {*}[lindex $docs_reply 1]] + set subcmds_in_cmd_docs [dict create {*}[dict get $docs subcommands]] + assert_equal [dict get $subcmds_in_cmd_docs "subcommands.bitarray|get"] {group module module subcommands} + assert_equal [dict get $subcmds_in_cmd_docs "subcommands.bitarray|set"] {group module module subcommands} + } + + test "Module pure-container command fails on arity error" { + catch {r subcommands.bitarray} e + assert_match {*wrong number of arguments for 'subcommands.bitarray' command} $e + + # Subcommands can be called + assert_equal [r subcommands.bitarray get k1] {OK} + + # Subcommand arity error + catch {r subcommands.bitarray get k1 8 90} e + assert_match {*wrong number of arguments for 'subcommands.bitarray|get' command} $e + } + + test "Module get current command fullname" { + assert_equal [r subcommands.parent_get_fullname] {subcommands.parent_get_fullname} + } + + test "Module get current subcommand fullname" { + assert_equal [r subcommands.sub get_fullname] {subcommands.sub|get_fullname} + } + + test "COMMAND LIST FILTERBY MODULE" { + assert_equal {} [r command list filterby module non_existing] + + set commands [r command list filterby module subcommands] + assert_not_equal [lsearch $commands "subcommands.bitarray"] -1 + assert_not_equal [lsearch $commands "subcommands.bitarray|set"] -1 + assert_not_equal [lsearch $commands "subcommands.parent_get_fullname"] -1 + assert_not_equal [lsearch $commands "subcommands.sub|get_fullname"] -1 + + assert_equal [lsearch $commands "set"] -1 + } + + test "Unload the module - subcommands" { + assert_equal {OK} [r module unload subcommands] + } +} diff --git a/tests/unit/moduleapi/test_lazyfree.tcl b/tests/unit/moduleapi/test_lazyfree.tcl new file mode 100644 index 0000000..8d2c55a --- /dev/null +++ b/tests/unit/moduleapi/test_lazyfree.tcl @@ -0,0 +1,32 @@ +set testmodule [file normalize tests/modules/test_lazyfree.so] + +start_server {tags {"modules"}} { + r module load $testmodule + + test "modules allocated memory can be reclaimed in the background" { + set orig_mem [s used_memory] + set rd [redis_deferring_client] + + # LAZYFREE_THRESHOLD is 64 + for {set i 0} {$i < 10000} {incr i} { + $rd lazyfreelink.insert lazykey $i + } + + for {set j 0} {$j < 10000} {incr j} { + $rd read + } + + assert {[r lazyfreelink.len lazykey] == 10000} + + set peak_mem [s used_memory] + assert {[r unlink lazykey] == 1} + assert {$peak_mem > $orig_mem+10000} + wait_for_condition 50 100 { + [s used_memory] < $peak_mem && + [s used_memory] < $orig_mem*2 && + [string match {*lazyfreed_objects:1*} [r info Memory]] + } else { + fail "Module memory is not reclaimed by UNLINK" + } + } +} diff --git a/tests/unit/moduleapi/testrdb.tcl b/tests/unit/moduleapi/testrdb.tcl new file mode 100644 index 0000000..ae3036f --- /dev/null +++ b/tests/unit/moduleapi/testrdb.tcl @@ -0,0 +1,306 @@ +# This module can be configure with multiple options given as flags on module load time +# 0 - not aux fields will be declared (this is the default) +# 1 << 0 - use aux_save2 api +# 1 << 1 - call aux callback before key space +# 1 << 2 - call aux callback after key space +# 1 << 3 - do not save data on aux callback +set testmodule [file normalize tests/modules/testrdb.so] + +tags "modules" { + test {modules are able to persist types} { + start_server [list overrides [list loadmodule "$testmodule"]] { + r testrdb.set.key key1 value1 + assert_equal "value1" [r testrdb.get.key key1] + r debug reload + assert_equal "value1" [r testrdb.get.key key1] + } + } + + test {modules global are lost without aux} { + set server_path [tmpdir "server.module-testrdb"] + start_server [list overrides [list loadmodule "$testmodule" "dir" $server_path] keep_persistence true] { + r testrdb.set.before global1 + assert_equal "global1" [r testrdb.get.before] + } + start_server [list overrides [list loadmodule "$testmodule" "dir" $server_path]] { + assert_equal "" [r testrdb.get.before] + } + } + + test {aux that saves no data are not saved to the rdb when aux_save2 is used} { + set server_path [tmpdir "server.module-testrdb"] + puts $server_path + # 15 == 1111 - use aux_save2 before and after key space without data + start_server [list overrides [list loadmodule "$testmodule 15" "dir" $server_path] keep_persistence true] { + r set x 1 + r save + } + start_server [list overrides [list "dir" $server_path] keep_persistence true] { + # make sure server started successfully without the module. + assert_equal {1} [r get x] + } + } + + test {aux that saves no data are saved to the rdb when aux_save is used} { + set server_path [tmpdir "server.module-testrdb"] + puts $server_path + # 14 == 1110 - use aux_save before and after key space without data + start_server [list overrides [list loadmodule "$testmodule 14" "dir" $server_path] keep_persistence true] { + r set x 1 + r save + } + start_server [list overrides [list loadmodule "$testmodule 14" "dir" $server_path] keep_persistence true] { + # make sure server started successfully and aux_save was called twice. + assert_equal {1} [r get x] + assert_equal {2} [r testrdb.get.n_aux_load_called] + } + } + + foreach test_case {6 7} { + # 6 == 0110 - use aux_save before and after key space with data + # 7 == 0111 - use aux_save2 before and after key space with data + test {modules are able to persist globals before and after} { + set server_path [tmpdir "server.module-testrdb"] + start_server [list overrides [list loadmodule "$testmodule $test_case" "dir" $server_path "save" "900 1"] keep_persistence true] { + r testrdb.set.before global1 + r testrdb.set.after global2 + assert_equal "global1" [r testrdb.get.before] + assert_equal "global2" [r testrdb.get.after] + } + start_server [list overrides [list loadmodule "$testmodule $test_case" "dir" $server_path "save" "900 1"]] { + assert_equal "global1" [r testrdb.get.before] + assert_equal "global2" [r testrdb.get.after] + } + + } + } + + foreach test_case {4 5} { + # 4 == 0100 - use aux_save after key space with data + # 5 == 0101 - use aux_save2 after key space with data + test {modules are able to persist globals just after} { + set server_path [tmpdir "server.module-testrdb"] + start_server [list overrides [list loadmodule "$testmodule $test_case" "dir" $server_path "save" "900 1"] keep_persistence true] { + r testrdb.set.after global2 + assert_equal "global2" [r testrdb.get.after] + } + start_server [list overrides [list loadmodule "$testmodule $test_case" "dir" $server_path "save" "900 1"]] { + assert_equal "global2" [r testrdb.get.after] + } + } + } + + test {Verify module options info} { + start_server [list overrides [list loadmodule "$testmodule"]] { + assert_match "*\[handle-io-errors|handle-repl-async-load\]*" [r info modules] + } + } + + tags {repl} { + test {diskless loading short read with module} { + start_server [list overrides [list loadmodule "$testmodule"]] { + set replica [srv 0 client] + set replica_host [srv 0 host] + set replica_port [srv 0 port] + start_server [list overrides [list loadmodule "$testmodule"]] { + set master [srv 0 client] + set master_host [srv 0 host] + set master_port [srv 0 port] + + # Set master and replica to use diskless replication + $master config set repl-diskless-sync yes + $master config set rdbcompression no + $replica config set repl-diskless-load swapdb + $master config set hz 500 + $replica config set hz 500 + $master config set dynamic-hz no + $replica config set dynamic-hz no + set start [clock clicks -milliseconds] + for {set k 0} {$k < 30} {incr k} { + r testrdb.set.key key$k [string repeat A [expr {int(rand()*1000000)}]] + } + + if {$::verbose} { + set end [clock clicks -milliseconds] + set duration [expr $end - $start] + puts "filling took $duration ms (TODO: use pipeline)" + set start [clock clicks -milliseconds] + } + + # Start the replication process... + set loglines [count_log_lines -1] + $master config set repl-diskless-sync-delay 0 + $replica replicaof $master_host $master_port + + # kill the replication at various points + set attempts 100 + if {$::accurate} { set attempts 500 } + for {set i 0} {$i < $attempts} {incr i} { + # wait for the replica to start reading the rdb + # using the log file since the replica only responds to INFO once in 2mb + set res [wait_for_log_messages -1 {"*Loading DB in memory*"} $loglines 2000 1] + set loglines [lindex $res 1] + + # add some additional random sleep so that we kill the master on a different place each time + after [expr {int(rand()*50)}] + + # kill the replica connection on the master + set killed [$master client kill type replica] + + set res [wait_for_log_messages -1 {"*Internal error in RDB*" "*Finished with success*" "*Successful partial resynchronization*"} $loglines 500 10] + if {$::verbose} { puts $res } + set log_text [lindex $res 0] + set loglines [lindex $res 1] + if {![string match "*Internal error in RDB*" $log_text]} { + # force the replica to try another full sync + $master multi + $master client kill type replica + $master set asdf asdf + # fill replication backlog with new content + $master config set repl-backlog-size 16384 + for {set keyid 0} {$keyid < 10} {incr keyid} { + $master set "$keyid string_$keyid" [string repeat A 16384] + } + $master exec + } + + # wait for loading to stop (fail) + # After a loading successfully, next loop will enter `async_loading` + wait_for_condition 1000 1 { + [s -1 async_loading] eq 0 && + [s -1 loading] eq 0 + } else { + fail "Replica didn't disconnect" + } + } + if {$::verbose} { + set end [clock clicks -milliseconds] + set duration [expr $end - $start] + puts "test took $duration ms" + } + # enable fast shutdown + $master config set rdb-key-save-delay 0 + } + } + } + + # Module events for diskless load swapdb when async_loading (matching master replid) + foreach test_case {6 7} { + # 6 == 0110 - use aux_save before and after key space with data + # 7 == 0111 - use aux_save2 before and after key space with data + foreach testType {Successful Aborted} { + start_server [list overrides [list loadmodule "$testmodule $test_case"] tags [list external:skip]] { + set replica [srv 0 client] + set replica_host [srv 0 host] + set replica_port [srv 0 port] + set replica_log [srv 0 stdout] + start_server [list overrides [list loadmodule "$testmodule $test_case"]] { + set master [srv 0 client] + set master_host [srv 0 host] + set master_port [srv 0 port] + + set start [clock clicks -milliseconds] + + # Set master and replica to use diskless replication on swapdb mode + $master config set repl-diskless-sync yes + $master config set repl-diskless-sync-delay 0 + $master config set save "" + $replica config set repl-diskless-load swapdb + $replica config set save "" + + # Initial sync to have matching replids between master and replica + $replica replicaof $master_host $master_port + + # Let replica finish initial sync with master + wait_for_condition 100 100 { + [s -1 master_link_status] eq "up" + } else { + fail "Master <-> Replica didn't finish sync" + } + + # Set global values on module so we can check if module event callbacks will pick it up correctly + $master testrdb.set.before value1_master + $replica testrdb.set.before value1_replica + + # Put different data sets on the master and replica + # We need to put large keys on the master since the replica replies to info only once in 2mb + $replica debug populate 200 slave 10 + $master debug populate 1000 master 100000 + $master config set rdbcompression no + + # Force the replica to try another full sync (this time it will have matching master replid) + $master multi + $master client kill type replica + # Fill replication backlog with new content + $master config set repl-backlog-size 16384 + for {set keyid 0} {$keyid < 10} {incr keyid} { + $master set "$keyid string_$keyid" [string repeat A 16384] + } + $master exec + + switch $testType { + "Aborted" { + # Set master with a slow rdb generation, so that we can easily intercept loading + # 10ms per key, with 1000 keys is 10 seconds + $master config set rdb-key-save-delay 10000 + + test {Diskless load swapdb RedisModuleEvent_ReplAsyncLoad handling: during loading, can keep module variable same as before} { + # Wait for the replica to start reading the rdb and module for acknowledgement + # We wanna abort only after the temp db was populated by REDISMODULE_AUX_BEFORE_RDB + wait_for_condition 100 100 { + [s -1 async_loading] eq 1 && [$replica testrdb.async_loading.get.before] eq "value1_master" + } else { + fail "Module didn't receive or react to REDISMODULE_SUBEVENT_REPL_ASYNC_LOAD_STARTED" + } + + assert_equal [$replica dbsize] 200 + assert_equal value1_replica [$replica testrdb.get.before] + } + + # Make sure that next sync will not start immediately so that we can catch the replica in between syncs + $master config set repl-diskless-sync-delay 5 + + # Kill the replica connection on the master + set killed [$master client kill type replica] + + test {Diskless load swapdb RedisModuleEvent_ReplAsyncLoad handling: when loading aborted, can keep module variable same as before} { + # Wait for loading to stop (fail) and module for acknowledgement + wait_for_condition 100 100 { + [s -1 async_loading] eq 0 && [$replica testrdb.async_loading.get.before] eq "" + } else { + fail "Module didn't receive or react to REDISMODULE_SUBEVENT_REPL_ASYNC_LOAD_ABORTED" + } + + assert_equal [$replica dbsize] 200 + assert_equal value1_replica [$replica testrdb.get.before] + } + + # Speed up shutdown + $master config set rdb-key-save-delay 0 + } + "Successful" { + # Let replica finish sync with master + wait_for_condition 100 100 { + [s -1 master_link_status] eq "up" + } else { + fail "Master <-> Replica didn't finish sync" + } + + test {Diskless load swapdb RedisModuleEvent_ReplAsyncLoad handling: after db loaded, can set module variable with new value} { + assert_equal [$replica dbsize] 1010 + assert_equal value1_master [$replica testrdb.get.before] + } + } + } + + if {$::verbose} { + set end [clock clicks -milliseconds] + set duration [expr $end - $start] + puts "test took $duration ms" + } + } + } + } + } + } +} diff --git a/tests/unit/moduleapi/timer.tcl b/tests/unit/moduleapi/timer.tcl new file mode 100644 index 0000000..4e9dd0f --- /dev/null +++ b/tests/unit/moduleapi/timer.tcl @@ -0,0 +1,99 @@ +set testmodule [file normalize tests/modules/timer.so] + +start_server {tags {"modules"}} { + r module load $testmodule + + test {RM_CreateTimer: a sequence of timers work} { + # We can't guarantee same-ms but we try using MULTI/EXEC + r multi + for {set i 0} {$i < 20} {incr i} { + r test.createtimer 10 timer-incr-key + } + r exec + + after 500 + assert_equal 20 [r get timer-incr-key] + } + + test {RM_GetTimer: basic sanity} { + # Getting non-existing timer + assert_equal {} [r test.gettimer 0] + + # Getting a real timer + set id [r test.createtimer 10000 timer-incr-key] + set info [r test.gettimer $id] + + assert_equal "timer-incr-key" [lindex $info 0] + set remaining [lindex $info 1] + assert {$remaining < 10000 && $remaining > 1} + # Stop the timer after get timer test + assert_equal 1 [r test.stoptimer $id] + } + + test {RM_StopTimer: basic sanity} { + r set "timer-incr-key" 0 + set id [r test.createtimer 1000 timer-incr-key] + + assert_equal 1 [r test.stoptimer $id] + + # Wait to be sure timer doesn't execute + after 2000 + assert_equal 0 [r get timer-incr-key] + + # Stop non-existing timer + assert_equal 0 [r test.stoptimer $id] + } + + test {Timer appears non-existing after it fires} { + r set "timer-incr-key" 0 + set id [r test.createtimer 10 timer-incr-key] + + # verify timer fired + after 500 + assert_equal 1 [r get timer-incr-key] + + # verify id does not exist + assert_equal {} [r test.gettimer $id] + } + + test "Module can be unloaded when timer was finished" { + r set "timer-incr-key" 0 + r test.createtimer 500 timer-incr-key + + # Make sure the Timer has not been fired + assert_equal 0 [r get timer-incr-key] + # Module can not be unloaded since the timer was ongoing + catch {r module unload timer} err + assert_match {*the module holds timer that is not fired*} $err + + # Wait to be sure timer has been finished + wait_for_condition 10 500 { + [r get timer-incr-key] == 1 + } else { + fail "Timer not fired" + } + + # Timer fired, can be unloaded now. + assert_equal {OK} [r module unload timer] + } + + test "Module can be unloaded when timer was stopped" { + r module load $testmodule + r set "timer-incr-key" 0 + set id [r test.createtimer 5000 timer-incr-key] + + # Module can not be unloaded since the timer was ongoing + catch {r module unload timer} err + assert_match {*the module holds timer that is not fired*} $err + + # Stop the timer + assert_equal 1 [r test.stoptimer $id] + + # Make sure the Timer has not been fired + assert_equal 0 [r get timer-incr-key] + + # Timer has stopped, can be unloaded now. + assert_equal {OK} [r module unload timer] + } +} + diff --git a/tests/unit/moduleapi/usercall.tcl b/tests/unit/moduleapi/usercall.tcl new file mode 100644 index 0000000..51ee1a4 --- /dev/null +++ b/tests/unit/moduleapi/usercall.tcl @@ -0,0 +1,136 @@ +set testmodule [file normalize tests/modules/usercall.so] + +set test_script_set "#!lua +redis.call('set','x',1) +return 1" + +set test_script_get "#!lua +redis.call('get','x') +return 1" + +start_server {tags {"modules usercall"}} { + r module load $testmodule + + # baseline test that module isn't doing anything weird + test {test module check regular redis command without user/acl} { + assert_equal [r usercall.reset_user] OK + assert_equal [r usercall.add_to_acl "~* &* +@all -set"] OK + assert_equal [r usercall.call_without_user set x 5] OK + assert_equal [r usercall.reset_user] OK + } + + # call with user with acl set on it, but without testing the acl + test {test module check regular redis command with user} { + assert_equal [r set x 5] OK + + assert_equal [r usercall.reset_user] OK + assert_equal [r usercall.add_to_acl "~* &* +@all -set"] OK + # off and sanitize-payload because module user / default value + assert_equal [r usercall.get_acl] "off sanitize-payload ~* &* +@all -set" + + # doesn't fail for regular commands as just testing acl here + assert_equal [r usercall.call_with_user_flag {} set x 10] OK + + assert_equal [r get x] 10 + assert_equal [r usercall.reset_user] OK + } + + # call with user with acl set on it, but with testing the acl in rm_call (for cmd itself) + test {test module check regular redis command with user and acl} { + assert_equal [r set x 5] OK + + r ACL LOG RESET + assert_equal [r usercall.reset_user] OK + assert_equal [r usercall.add_to_acl "~* &* +@all -set"] OK + # off and sanitize-payload because module user / default value + assert_equal [r usercall.get_acl] "off sanitize-payload ~* &* +@all -set" + + # fails here as testing acl in rm call + assert_error {*NOPERM User module_user has no permissions*} {r usercall.call_with_user_flag C set x 10} + + assert_equal [r usercall.call_with_user_flag C get x] 5 + + # verify that new log entry added + set entry [lindex [r ACL LOG] 0] + assert_equal [dict get $entry username] {module_user} + assert_equal [dict get $entry context] {module} + assert_equal [dict get $entry object] {set} + assert_equal [dict get $entry reason] {command} + assert_match {*cmd=usercall.call_with_user_flag*} [dict get $entry client-info] + + assert_equal [r usercall.reset_user] OK + } + + # call with user with acl set on it, but with testing the acl in rm_call (for cmd itself) + test {test module check regular redis command with user and acl from blocked background thread} { + assert_equal [r set x 5] OK + + r ACL LOG RESET + assert_equal [r usercall.reset_user] OK + assert_equal [r usercall.add_to_acl "~* &* +@all -set"] OK + + # fails here as testing acl in rm call from a background thread + assert_error {*NOPERM User module_user has no permissions*} {r usercall.call_with_user_bg C set x 10} + + assert_equal [r usercall.call_with_user_bg C get x] 5 + + # verify that new log entry added + set entry [lindex [r ACL LOG] 0] + assert_equal [dict get $entry username] {module_user} + assert_equal [dict get $entry context] {module} + assert_equal [dict get $entry object] {set} + assert_equal [dict get $entry reason] {command} + assert_match {*cmd=NULL*} [dict get $entry client-info] + + assert_equal [r usercall.reset_user] OK + } + + # baseline script test, call without user on script + test {test module check eval script without user} { + set sha_set [r script load $test_script_set] + set sha_get [r script load $test_script_get] + + assert_equal [r usercall.call_without_user evalsha $sha_set 0] 1 + assert_equal [r usercall.call_without_user evalsha $sha_get 0] 1 + } + + # baseline script test, call without user on script + test {test module check eval script with user being set, but not acl testing} { + set sha_set [r script load $test_script_set] + set sha_get [r script load $test_script_get] + + assert_equal [r usercall.reset_user] OK + assert_equal [r usercall.add_to_acl "~* &* +@all -set"] OK + # off and sanitize-payload because module user / default value + assert_equal [r usercall.get_acl] "off sanitize-payload ~* &* +@all -set" + + # passes as not checking ACL + assert_equal [r usercall.call_with_user_flag {} evalsha $sha_set 0] 1 + assert_equal [r usercall.call_with_user_flag {} evalsha $sha_get 0] 1 + } + + # call with user on script (without rm_call acl check) to ensure user carries through to script execution + # we already tested the check in rm_call above, here we are checking the script itself will enforce ACL + test {test module check eval script with user and acl} { + set sha_set [r script load $test_script_set] + set sha_get [r script load $test_script_get] + + r ACL LOG RESET + assert_equal [r usercall.reset_user] OK + assert_equal [r usercall.add_to_acl "~* &* +@all -set"] OK + + # fails here in script, as rm_call will permit the eval call + catch {r usercall.call_with_user_flag C evalsha $sha_set 0} e + assert_match {*ERR ACL failure in script*} $e + + assert_equal [r usercall.call_with_user_flag C evalsha $sha_get 0] 1 + + # verify that new log entry added + set entry [lindex [r ACL LOG] 0] + assert_equal [dict get $entry username] {module_user} + assert_equal [dict get $entry context] {lua} + assert_equal [dict get $entry object] {set} + assert_equal [dict get $entry reason] {command} + assert_match {*cmd=usercall.call_with_user_flag*} [dict get $entry client-info] + } +} diff --git a/tests/unit/moduleapi/zset.tcl b/tests/unit/moduleapi/zset.tcl new file mode 100644 index 0000000..b6ab41d --- /dev/null +++ b/tests/unit/moduleapi/zset.tcl @@ -0,0 +1,40 @@ +set testmodule [file normalize tests/modules/zset.so] + +start_server {tags {"modules"}} { + r module load $testmodule + + test {Module zset rem} { + r del k + r zadd k 100 hello 200 world + assert_equal 1 [r zset.rem k hello] + assert_equal 0 [r zset.rem k hello] + assert_equal 1 [r exists k] + # Check that removing the last element deletes the key + assert_equal 1 [r zset.rem k world] + assert_equal 0 [r exists k] + } + + test {Module zset add} { + r del k + # Check that failure does not create empty key + assert_error "ERR ZsetAdd failed" {r zset.add k nan hello} + assert_equal 0 [r exists k] + + r zset.add k 100 hello + assert_equal {hello 100} [r zrange k 0 -1 withscores] + } + + test {Module zset incrby} { + r del k + # Check that failure does not create empty key + assert_error "ERR ZsetIncrby failed" {r zset.incrby k hello nan} + assert_equal 0 [r exists k] + + r zset.incrby k hello 100 + assert_equal {hello 100} [r zrange k 0 -1 withscores] + } + + test "Unload the module - zset" { + assert_equal {OK} [r module unload zset] + } +} |