From 317c0644ccf108aa23ef3fd8358bd66c2840bfc0 Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Sun, 14 Apr 2024 15:40:54 +0200 Subject: Adding upstream version 5:7.2.4. Signed-off-by: Daniel Baumann --- utils/build-static-symbols.tcl | 22 + utils/cluster_fail_time.tcl | 50 +++ utils/corrupt_rdb.c | 45 ++ utils/create-cluster/.gitignore | 6 + utils/create-cluster/README | 27 ++ utils/create-cluster/create-cluster | 125 ++++++ utils/gen-test-certs.sh | 58 +++ utils/generate-command-code.py | 623 ++++++++++++++++++++++++++ utils/generate-commands-json.py | 136 ++++++ utils/generate-module-api-doc.rb | 205 +++++++++ utils/graphs/commits-over-time/README.md | 16 + utils/graphs/commits-over-time/genhtml.tcl | 96 ++++ utils/hyperloglog/.gitignore | 1 + utils/hyperloglog/hll-err.rb | 27 ++ utils/hyperloglog/hll-gnuplot-graph.rb | 88 ++++ utils/install_server.sh | 291 ++++++++++++ utils/lru/README | 19 + utils/lru/lfu-simulation.c | 158 +++++++ utils/lru/test-lru.rb | 223 +++++++++ utils/redis-copy.rb | 35 ++ utils/redis-sha1.rb | 52 +++ utils/redis_init_script | 50 +++ utils/redis_init_script.tpl | 44 ++ utils/releasetools/01_create_tarball.sh | 14 + utils/releasetools/02_upload_tarball.sh | 23 + utils/releasetools/03_test_release.sh | 28 ++ utils/releasetools/04_release_hash.sh | 13 + utils/releasetools/changelog.tcl | 35 ++ utils/reply_schema_linter.js | 31 ++ utils/req-res-log-validator.py | 350 +++++++++++++++ utils/req-res-validator/requirements.txt | 2 + utils/speed-regression.tcl | 130 ++++++ utils/srandmember/README.md | 14 + utils/srandmember/showdist.rb | 33 ++ utils/srandmember/showfreq.rb | 23 + utils/systemd-redis_multiple_servers@.service | 37 ++ utils/systemd-redis_server.service | 43 ++ utils/tracking_collisions.c | 76 ++++ utils/whatisdoing.sh | 24 + 39 files changed, 3273 insertions(+) create mode 100644 utils/build-static-symbols.tcl create mode 100644 utils/cluster_fail_time.tcl create mode 100644 utils/corrupt_rdb.c create mode 100644 utils/create-cluster/.gitignore create mode 100644 utils/create-cluster/README create mode 100755 utils/create-cluster/create-cluster create mode 100755 utils/gen-test-certs.sh create mode 100755 utils/generate-command-code.py create mode 100755 utils/generate-commands-json.py create mode 100755 utils/generate-module-api-doc.rb create mode 100644 utils/graphs/commits-over-time/README.md create mode 100755 utils/graphs/commits-over-time/genhtml.tcl create mode 100644 utils/hyperloglog/.gitignore create mode 100644 utils/hyperloglog/hll-err.rb create mode 100644 utils/hyperloglog/hll-gnuplot-graph.rb create mode 100755 utils/install_server.sh create mode 100644 utils/lru/README create mode 100644 utils/lru/lfu-simulation.c create mode 100644 utils/lru/test-lru.rb create mode 100644 utils/redis-copy.rb create mode 100644 utils/redis-sha1.rb create mode 100755 utils/redis_init_script create mode 100755 utils/redis_init_script.tpl create mode 100755 utils/releasetools/01_create_tarball.sh create mode 100755 utils/releasetools/02_upload_tarball.sh create mode 100755 utils/releasetools/03_test_release.sh create mode 100755 utils/releasetools/04_release_hash.sh create mode 100755 utils/releasetools/changelog.tcl create mode 100644 utils/reply_schema_linter.js create mode 100755 utils/req-res-log-validator.py create mode 100644 utils/req-res-validator/requirements.txt create mode 100755 utils/speed-regression.tcl create mode 100644 utils/srandmember/README.md create mode 100644 utils/srandmember/showdist.rb create mode 100644 utils/srandmember/showfreq.rb create mode 100644 utils/systemd-redis_multiple_servers@.service create mode 100644 utils/systemd-redis_server.service create mode 100644 utils/tracking_collisions.c create mode 100755 utils/whatisdoing.sh (limited to 'utils') diff --git a/utils/build-static-symbols.tcl b/utils/build-static-symbols.tcl new file mode 100644 index 0000000..e634cbe --- /dev/null +++ b/utils/build-static-symbols.tcl @@ -0,0 +1,22 @@ +# Build a symbol table for static symbols of redis.c +# Useful to get stack traces on segfault without a debugger. See redis.c +# for more information. +# +# Copyright(C) 2009 Salvatore Sanfilippo, under the BSD license. + +set fd [open redis.c] +set symlist {} +while {[gets $fd line] != -1} { + if {[regexp {^static +[A-z0-9]+[ *]+([A-z0-9]*)\(} $line - sym]} { + lappend symlist $sym + } +} +set symlist [lsort -unique $symlist] +puts "static struct redisFunctionSym symsTable\[\] = {" +foreach sym $symlist { + puts "{\"$sym\",(unsigned long)$sym}," +} +puts "{NULL,0}" +puts "};" + +close $fd diff --git a/utils/cluster_fail_time.tcl b/utils/cluster_fail_time.tcl new file mode 100644 index 0000000..8739949 --- /dev/null +++ b/utils/cluster_fail_time.tcl @@ -0,0 +1,50 @@ +# This simple script is used in order to estimate the average PFAIL->FAIL +# state switch after a failure. + +set ::sleep_time 10 ; # How much to sleep to trigger PFAIL. +set ::fail_port 30016 ; # Node to put in sleep. +set ::other_port 30001 ; # Node to use to monitor the flag switch. + +proc avg vector { + set sum 0.0 + foreach x $vector { + set sum [expr {$sum+$x}] + } + expr {$sum/[llength $vector]} +} + +set samples {} +while 1 { + exec redis-cli -p $::fail_port debug sleep $::sleep_time > /dev/null & + + # Wait for fail? to appear. + while 1 { + set output [exec redis-cli -p $::other_port cluster nodes] + if {[string match {*fail\?*} $output]} break + after 100 + } + + puts "FAIL?" + set start [clock milliseconds] + + # Wait for fail? to disappear. + while 1 { + set output [exec redis-cli -p $::other_port cluster nodes] + if {![string match {*fail\?*} $output]} break + after 100 + } + + puts "FAIL" + set now [clock milliseconds] + set elapsed [expr {$now-$start}] + puts $elapsed + lappend samples $elapsed + + puts "AVG([llength $samples]): [avg $samples]" + + # Wait for the instance to be available again. + exec redis-cli -p $::fail_port ping + + # Wait for the fail flag to be cleared. + after 2000 +} diff --git a/utils/corrupt_rdb.c b/utils/corrupt_rdb.c new file mode 100644 index 0000000..df9c93e --- /dev/null +++ b/utils/corrupt_rdb.c @@ -0,0 +1,45 @@ +/* Trivia program to corrupt an RDB file in order to check the RDB check + * program behavior and effectiveness. + * + * Copyright (C) 2016 Salvatore Sanfilippo. + * This software is released in the 3-clause BSD license. */ + +#include +#include +#include +#include +#include +#include + +int main(int argc, char **argv) { + struct stat stat; + int fd, cycles; + + if (argc != 3) { + fprintf(stderr,"Usage: \n"); + exit(1); + } + + srand(time(NULL)); + char *filename = argv[1]; + cycles = atoi(argv[2]); + fd = open(filename,O_RDWR); + if (fd == -1) { + perror("open"); + exit(1); + } + fstat(fd,&stat); + + while(cycles--) { + unsigned char buf[32]; + unsigned long offset = rand()%stat.st_size; + int writelen = 1+rand()%31; + int j; + + for (j = 0; j < writelen; j++) buf[j] = (char)rand(); + lseek(fd,offset,SEEK_SET); + printf("Writing %d bytes at offset %lu\n", writelen, offset); + write(fd,buf,writelen); + } + return 0; +} diff --git a/utils/create-cluster/.gitignore b/utils/create-cluster/.gitignore new file mode 100644 index 0000000..a34b639 --- /dev/null +++ b/utils/create-cluster/.gitignore @@ -0,0 +1,6 @@ +config.sh +*.rdb +*.aof +*.conf +*.log +appendonlydir-* diff --git a/utils/create-cluster/README b/utils/create-cluster/README new file mode 100644 index 0000000..bcd7459 --- /dev/null +++ b/utils/create-cluster/README @@ -0,0 +1,27 @@ +create-cluster is a small script used to easily start a big number of Redis +instances configured to run in cluster mode. Its main goal is to allow manual +testing in a condition which is not easy to replicate with the Redis cluster +unit tests, for example when a lot of instances are needed in order to trigger +a given bug. + +The tool can also be used just to easily create a number of instances in a +Redis Cluster in order to experiment a bit with the system. + +USAGE +--- + +To create a cluster, follow these steps: + +1. Edit create-cluster and change the start / end port, depending on the +number of instances you want to create. +2. Use "./create-cluster start" in order to run the instances. +3. Use "./create-cluster create" in order to execute redis-cli --cluster create, so that +an actual Redis cluster will be created. (If you're accessing your setup via a local container, ensure that the CLUSTER_HOST value is changed to your local IP) +4. Now you are ready to play with the cluster. AOF files and logs for each instances are created in the current directory. + +In order to stop a cluster: + +1. Use "./create-cluster stop" to stop all the instances. After you stopped the instances you can use "./create-cluster start" to restart them if you change your mind. +2. Use "./create-cluster clean" to remove all the AOF / log files to restart with a clean environment. + +Use the command "./create-cluster help" to get the full list of features. diff --git a/utils/create-cluster/create-cluster b/utils/create-cluster/create-cluster new file mode 100755 index 0000000..d97ee2b --- /dev/null +++ b/utils/create-cluster/create-cluster @@ -0,0 +1,125 @@ +#!/bin/bash + +SCRIPT_DIR="$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" + +# Settings +BIN_PATH="$SCRIPT_DIR/../../src/" +CLUSTER_HOST=127.0.0.1 +PORT=30000 +TIMEOUT=2000 +NODES=6 +REPLICAS=1 +PROTECTED_MODE=yes +ADDITIONAL_OPTIONS="" + +# You may want to put the above config parameters into config.sh in order to +# override the defaults without modifying this script. + +if [ -a config.sh ] +then + source "config.sh" +fi + +# Computed vars +ENDPORT=$((PORT+NODES)) + +if [ "$1" == "start" ] +then + while [ $((PORT < ENDPORT)) != "0" ]; do + PORT=$((PORT+1)) + echo "Starting $PORT" + $BIN_PATH/redis-server --port $PORT --protected-mode $PROTECTED_MODE --cluster-enabled yes --cluster-config-file nodes-${PORT}.conf --cluster-node-timeout $TIMEOUT --appendonly yes --appendfilename appendonly-${PORT}.aof --appenddirname appendonlydir-${PORT} --dbfilename dump-${PORT}.rdb --logfile ${PORT}.log --daemonize yes ${ADDITIONAL_OPTIONS} + done + exit 0 +fi + +if [ "$1" == "create" ] +then + HOSTS="" + while [ $((PORT < ENDPORT)) != "0" ]; do + PORT=$((PORT+1)) + HOSTS="$HOSTS $CLUSTER_HOST:$PORT" + done + OPT_ARG="" + if [ "$2" == "-f" ]; then + OPT_ARG="--cluster-yes" + fi + $BIN_PATH/redis-cli --cluster create $HOSTS --cluster-replicas $REPLICAS $OPT_ARG + exit 0 +fi + +if [ "$1" == "stop" ] +then + while [ $((PORT < ENDPORT)) != "0" ]; do + PORT=$((PORT+1)) + echo "Stopping $PORT" + $BIN_PATH/redis-cli -p $PORT shutdown nosave + done + exit 0 +fi + +if [ "$1" == "watch" ] +then + PORT=$((PORT+1)) + while [ 1 ]; do + clear + date + $BIN_PATH/redis-cli -p $PORT cluster nodes | head -30 + sleep 1 + done + exit 0 +fi + +if [ "$1" == "tail" ] +then + INSTANCE=$2 + PORT=$((PORT+INSTANCE)) + tail -f ${PORT}.log + exit 0 +fi + +if [ "$1" == "tailall" ] +then + tail -f *.log + exit 0 +fi + +if [ "$1" == "call" ] +then + while [ $((PORT < ENDPORT)) != "0" ]; do + PORT=$((PORT+1)) + $BIN_PATH/redis-cli -p $PORT $2 $3 $4 $5 $6 $7 $8 $9 + done + exit 0 +fi + +if [ "$1" == "clean" ] +then + echo "Cleaning *.log" + rm -rf *.log + echo "Cleaning appendonlydir-*" + rm -rf appendonlydir-* + echo "Cleaning dump-*.rdb" + rm -rf dump-*.rdb + echo "Cleaning nodes-*.conf" + rm -rf nodes-*.conf + exit 0 +fi + +if [ "$1" == "clean-logs" ] +then + echo "Cleaning *.log" + rm -rf *.log + exit 0 +fi + +echo "Usage: $0 [start|create|stop|watch|tail|tailall|clean|clean-logs|call]" +echo "start -- Launch Redis Cluster instances." +echo "create [-f] -- Create a cluster using redis-cli --cluster create." +echo "stop -- Stop Redis Cluster instances." +echo "watch -- Show CLUSTER NODES output (first 30 lines) of first node." +echo "tail -- Run tail -f of instance at base port + ID." +echo "tailall -- Run tail -f for all the log files at once." +echo "clean -- Remove all instances data, logs, configs." +echo "clean-logs -- Remove just instances logs." +echo "call -- Call a command (up to 7 arguments) on all nodes." diff --git a/utils/gen-test-certs.sh b/utils/gen-test-certs.sh new file mode 100755 index 0000000..6bc9d86 --- /dev/null +++ b/utils/gen-test-certs.sh @@ -0,0 +1,58 @@ +#!/bin/bash + +# Generate some test certificates which are used by the regression test suite: +# +# tests/tls/ca.{crt,key} Self signed CA certificate. +# tests/tls/redis.{crt,key} A certificate with no key usage/policy restrictions. +# tests/tls/client.{crt,key} A certificate restricted for SSL client usage. +# tests/tls/server.{crt,key} A certificate restricted for SSL server usage. +# tests/tls/redis.dh DH Params file. + +generate_cert() { + local name=$1 + local cn="$2" + local opts="$3" + + local keyfile=tests/tls/${name}.key + local certfile=tests/tls/${name}.crt + + [ -f $keyfile ] || openssl genrsa -out $keyfile 2048 + openssl req \ + -new -sha256 \ + -subj "/O=Redis Test/CN=$cn" \ + -key $keyfile | \ + openssl x509 \ + -req -sha256 \ + -CA tests/tls/ca.crt \ + -CAkey tests/tls/ca.key \ + -CAserial tests/tls/ca.txt \ + -CAcreateserial \ + -days 365 \ + $opts \ + -out $certfile +} + +mkdir -p tests/tls +[ -f tests/tls/ca.key ] || openssl genrsa -out tests/tls/ca.key 4096 +openssl req \ + -x509 -new -nodes -sha256 \ + -key tests/tls/ca.key \ + -days 3650 \ + -subj '/O=Redis Test/CN=Certificate Authority' \ + -out tests/tls/ca.crt + +cat > tests/tls/openssl.cnf <<_END_ +[ server_cert ] +keyUsage = digitalSignature, keyEncipherment +nsCertType = server + +[ client_cert ] +keyUsage = digitalSignature, keyEncipherment +nsCertType = client +_END_ + +generate_cert server "Server-only" "-extfile tests/tls/openssl.cnf -extensions server_cert" +generate_cert client "Client-only" "-extfile tests/tls/openssl.cnf -extensions client_cert" +generate_cert redis "Generic-cert" + +[ -f tests/tls/redis.dh ] || openssl dhparam -out tests/tls/redis.dh 2048 diff --git a/utils/generate-command-code.py b/utils/generate-command-code.py new file mode 100755 index 0000000..dc66ce8 --- /dev/null +++ b/utils/generate-command-code.py @@ -0,0 +1,623 @@ +#!/usr/bin/env python3 +import glob +import json +import os +import argparse + +ARG_TYPES = { + "string": "ARG_TYPE_STRING", + "integer": "ARG_TYPE_INTEGER", + "double": "ARG_TYPE_DOUBLE", + "key": "ARG_TYPE_KEY", + "pattern": "ARG_TYPE_PATTERN", + "unix-time": "ARG_TYPE_UNIX_TIME", + "pure-token": "ARG_TYPE_PURE_TOKEN", + "oneof": "ARG_TYPE_ONEOF", + "block": "ARG_TYPE_BLOCK", +} + +GROUPS = { + "generic": "COMMAND_GROUP_GENERIC", + "string": "COMMAND_GROUP_STRING", + "list": "COMMAND_GROUP_LIST", + "set": "COMMAND_GROUP_SET", + "sorted_set": "COMMAND_GROUP_SORTED_SET", + "hash": "COMMAND_GROUP_HASH", + "pubsub": "COMMAND_GROUP_PUBSUB", + "transactions": "COMMAND_GROUP_TRANSACTIONS", + "connection": "COMMAND_GROUP_CONNECTION", + "server": "COMMAND_GROUP_SERVER", + "scripting": "COMMAND_GROUP_SCRIPTING", + "hyperloglog": "COMMAND_GROUP_HYPERLOGLOG", + "cluster": "COMMAND_GROUP_CLUSTER", + "sentinel": "COMMAND_GROUP_SENTINEL", + "geo": "COMMAND_GROUP_GEO", + "stream": "COMMAND_GROUP_STREAM", + "bitmap": "COMMAND_GROUP_BITMAP", +} + + +def get_optional_desc_string(desc, field, force_uppercase=False): + v = desc.get(field, None) + if v and force_uppercase: + v = v.upper() + ret = "\"%s\"" % v if v else "NULL" + return ret.replace("\n", "\\n") + + +def check_command_args_key_specs(args, command_key_specs_index_set, command_arg_key_specs_index_set): + if not args: + return True + + for arg in args: + if arg.key_spec_index is not None: + assert isinstance(arg.key_spec_index, int) + + if arg.key_spec_index not in command_key_specs_index_set: + print("command: %s arg: %s key_spec_index error" % (command.fullname(), arg.name)) + return False + + command_arg_key_specs_index_set.add(arg.key_spec_index) + + if not check_command_args_key_specs(arg.subargs, command_key_specs_index_set, command_arg_key_specs_index_set): + return False + + return True + +def check_command_key_specs(command): + if not command.key_specs: + return True + + assert isinstance(command.key_specs, list) + + for cmd_key_spec in command.key_specs: + if "flags" not in cmd_key_spec: + print("command: %s key_specs missing flags" % command.fullname()) + return False + + if "NOT_KEY" in cmd_key_spec["flags"]: + # Like SUNSUBSCRIBE / SPUBLISH / SSUBSCRIBE + return True + + command_key_specs_index_set = set(range(len(command.key_specs))) + command_arg_key_specs_index_set = set() + + # Collect key_spec used for each arg, including arg.subarg + if not check_command_args_key_specs(command.args, command_key_specs_index_set, command_arg_key_specs_index_set): + return False + + # Check if we have key_specs not used + if command_key_specs_index_set != command_arg_key_specs_index_set: + print("command: %s may have unused key_spec" % command.fullname()) + return False + + return True + + +# Globals +subcommands = {} # container_name -> dict(subcommand_name -> Subcommand) - Only subcommands +commands = {} # command_name -> Command - Only commands + + +class KeySpec(object): + def __init__(self, spec): + self.spec = spec + + def struct_code(self): + def _flags_code(): + s = "" + for flag in self.spec.get("flags", []): + s += "CMD_KEY_%s|" % flag + return s[:-1] if s else 0 + + def _begin_search_code(): + if self.spec["begin_search"].get("index"): + return "KSPEC_BS_INDEX,.bs.index={%d}" % ( + self.spec["begin_search"]["index"]["pos"] + ) + elif self.spec["begin_search"].get("keyword"): + return "KSPEC_BS_KEYWORD,.bs.keyword={\"%s\",%d}" % ( + self.spec["begin_search"]["keyword"]["keyword"], + self.spec["begin_search"]["keyword"]["startfrom"], + ) + elif "unknown" in self.spec["begin_search"]: + return "KSPEC_BS_UNKNOWN,{{0}}" + else: + print("Invalid begin_search! value=%s" % self.spec["begin_search"]) + exit(1) + + def _find_keys_code(): + if self.spec["find_keys"].get("range"): + return "KSPEC_FK_RANGE,.fk.range={%d,%d,%d}" % ( + self.spec["find_keys"]["range"]["lastkey"], + self.spec["find_keys"]["range"]["step"], + self.spec["find_keys"]["range"]["limit"] + ) + elif self.spec["find_keys"].get("keynum"): + return "KSPEC_FK_KEYNUM,.fk.keynum={%d,%d,%d}" % ( + self.spec["find_keys"]["keynum"]["keynumidx"], + self.spec["find_keys"]["keynum"]["firstkey"], + self.spec["find_keys"]["keynum"]["step"] + ) + elif "unknown" in self.spec["find_keys"]: + return "KSPEC_FK_UNKNOWN,{{0}}" + else: + print("Invalid find_keys! value=%s" % self.spec["find_keys"]) + exit(1) + + return "%s,%s,%s,%s" % ( + get_optional_desc_string(self.spec, "notes"), + _flags_code(), + _begin_search_code(), + _find_keys_code() + ) + + +def verify_no_dup_names(container_fullname, args): + name_list = [arg.name for arg in args] + name_set = set(name_list) + if len(name_list) != len(name_set): + print("{}: Dup argument names: {}".format(container_fullname, name_list)) + exit(1) + + +class Argument(object): + def __init__(self, parent_name, desc): + self.parent_name = parent_name + self.desc = desc + self.name = self.desc["name"].lower() + if "_" in self.name: + print("{}: name ({}) should not contain underscores".format(self.fullname(), self.name)) + exit(1) + self.type = self.desc["type"] + self.key_spec_index = self.desc.get("key_spec_index", None) + self.subargs = [] + if self.type in ["oneof", "block"]: + self.display = None + for subdesc in self.desc["arguments"]: + self.subargs.append(Argument(self.fullname(), subdesc)) + if len(self.subargs) < 2: + print("{}: oneof or block arg contains less than two subargs".format(self.fullname())) + exit(1) + verify_no_dup_names(self.fullname(), self.subargs) + else: + self.display = self.desc.get("display") + + def fullname(self): + return ("%s %s" % (self.parent_name, self.name)).replace("-", "_") + + def struct_name(self): + return "%s_Arg" % (self.fullname().replace(" ", "_")) + + def subarg_table_name(self): + assert self.subargs + return "%s_Subargs" % (self.fullname().replace(" ", "_")) + + def struct_code(self): + """ + Output example: + MAKE_ARG("expiration",ARG_TYPE_ONEOF,-1,NULL,NULL,NULL,CMD_ARG_OPTIONAL,5,NULL),.subargs=GETEX_expiration_Subargs + """ + + def _flags_code(): + s = "" + if self.desc.get("optional", False): + s += "CMD_ARG_OPTIONAL|" + if self.desc.get("multiple", False): + s += "CMD_ARG_MULTIPLE|" + if self.desc.get("multiple_token", False): + assert self.desc.get("multiple", False) # Sanity + s += "CMD_ARG_MULTIPLE_TOKEN|" + return s[:-1] if s else "CMD_ARG_NONE" + + s = "MAKE_ARG(\"%s\",%s,%d,%s,%s,%s,%s,%d,%s)" % ( + self.name, + ARG_TYPES[self.type], + self.desc.get("key_spec_index", -1), + get_optional_desc_string(self.desc, "token", force_uppercase=True), + get_optional_desc_string(self.desc, "summary"), + get_optional_desc_string(self.desc, "since"), + _flags_code(), + len(self.subargs), + get_optional_desc_string(self.desc, "deprecated_since"), + ) + if "display" in self.desc: + s += ",.display_text=\"%s\"" % self.desc["display"].lower() + if self.subargs: + s += ",.subargs=%s" % self.subarg_table_name() + + return s + + def write_internal_structs(self, f): + if self.subargs: + for subarg in self.subargs: + subarg.write_internal_structs(f) + + f.write("/* %s argument table */\n" % self.fullname()) + f.write("struct COMMAND_ARG %s[] = {\n" % self.subarg_table_name()) + for subarg in self.subargs: + f.write("{%s},\n" % subarg.struct_code()) + f.write("};\n\n") + + +def to_c_name(str): + return str.replace(":", "").replace(".", "_").replace("$", "_")\ + .replace("^", "_").replace("*", "_").replace("-", "_") + + +class ReplySchema(object): + def __init__(self, name, desc): + self.name = to_c_name(name) + self.schema = {} + if desc.get("type") == "object": + if desc.get("properties") and desc.get("additionalProperties") is None: + print("%s: Any object that has properties should have the additionalProperties field" % self.name) + exit(1) + elif desc.get("type") == "array": + if desc.get("items") and isinstance(desc["items"], list) and any([desc.get(k) is None for k in ["minItems", "maxItems"]]): + print("%s: Any array that has items should have the minItems and maxItems fields" % self.name) + exit(1) + for k, v in desc.items(): + if isinstance(v, dict): + self.schema[k] = ReplySchema("%s_%s" % (self.name, k), v) + elif isinstance(v, list): + self.schema[k] = [] + for i, subdesc in enumerate(v): + self.schema[k].append(ReplySchema("%s_%s_%i" % (self.name, k,i), subdesc)) + else: + self.schema[k] = v + + def write(self, f): + def struct_code(name, k, v): + if isinstance(v, ReplySchema): + t = "JSON_TYPE_OBJECT" + vstr = ".value.object=&%s" % name + elif isinstance(v, list): + t = "JSON_TYPE_ARRAY" + vstr = ".value.array={.objects=%s,.length=%d}" % (name, len(v)) + elif isinstance(v, bool): + t = "JSON_TYPE_BOOLEAN" + vstr = ".value.boolean=%d" % int(v) + elif isinstance(v, str): + t = "JSON_TYPE_STRING" + vstr = ".value.string=\"%s\"" % v + elif isinstance(v, int): + t = "JSON_TYPE_INTEGER" + vstr = ".value.integer=%d" % v + + return "%s,\"%s\",%s" % (t, k, vstr) + + for k, v in self.schema.items(): + if isinstance(v, ReplySchema): + v.write(f) + elif isinstance(v, list): + for i, schema in enumerate(v): + schema.write(f) + name = to_c_name("%s_%s" % (self.name, k)) + f.write("/* %s array reply schema */\n" % name) + f.write("struct jsonObject *%s[] = {\n" % name) + for i, schema in enumerate(v): + f.write("&%s,\n" % schema.name) + f.write("};\n\n") + + f.write("/* %s reply schema */\n" % self.name) + f.write("struct jsonObjectElement %s_elements[] = {\n" % self.name) + for k, v in self.schema.items(): + name = to_c_name("%s_%s" % (self.name, k)) + f.write("{%s},\n" % struct_code(name, k, v)) + f.write("};\n\n") + f.write("struct jsonObject %s = {%s_elements,.length=%d};\n\n" % (self.name, self.name, len(self.schema))) + + +class Command(object): + def __init__(self, name, desc): + self.name = name.upper() + self.desc = desc + self.group = self.desc["group"] + self.key_specs = self.desc.get("key_specs", []) + self.subcommands = [] + self.args = [] + for arg_desc in self.desc.get("arguments", []): + self.args.append(Argument(self.fullname(), arg_desc)) + verify_no_dup_names(self.fullname(), self.args) + self.reply_schema = None + if "reply_schema" in self.desc: + self.reply_schema = ReplySchema(self.reply_schema_name(), self.desc["reply_schema"]) + + def fullname(self): + return self.name.replace("-", "_").replace(":", "") + + def return_types_table_name(self): + return "%s_ReturnInfo" % self.fullname().replace(" ", "_") + + def subcommand_table_name(self): + assert self.subcommands + return "%s_Subcommands" % self.name + + def history_table_name(self): + return "%s_History" % (self.fullname().replace(" ", "_")) + + def tips_table_name(self): + return "%s_Tips" % (self.fullname().replace(" ", "_")) + + def arg_table_name(self): + return "%s_Args" % (self.fullname().replace(" ", "_")) + + def key_specs_table_name(self): + return "%s_Keyspecs" % (self.fullname().replace(" ", "_")) + + def reply_schema_name(self): + return "%s_ReplySchema" % (self.fullname().replace(" ", "_")) + + def struct_name(self): + return "%s_Command" % (self.fullname().replace(" ", "_")) + + def history_code(self): + if not self.desc.get("history"): + return "" + s = "" + for tupl in self.desc["history"]: + s += "{\"%s\",\"%s\"},\n" % (tupl[0], tupl[1]) + return s + + def num_history(self): + if not self.desc.get("history"): + return 0 + return len(self.desc["history"]) + + def tips_code(self): + if not self.desc.get("command_tips"): + return "" + s = "" + for hint in self.desc["command_tips"]: + s += "\"%s\",\n" % hint.lower() + return s + + def num_tips(self): + if not self.desc.get("command_tips"): + return 0 + return len(self.desc["command_tips"]) + + def key_specs_code(self): + s = "" + for spec in self.key_specs: + s += "{%s}," % KeySpec(spec).struct_code() + return s[:-1] + + + def struct_code(self): + """ + Output example: + MAKE_CMD("set","Set the string value of a key","O(1)","1.0.0",CMD_DOC_NONE,NULL,NULL,"string",COMMAND_GROUP_STRING,SET_History,4,SET_Tips,0,setCommand,-3,CMD_WRITE|CMD_DENYOOM,ACL_CATEGORY_STRING,SET_Keyspecs,1,setGetKeys,5),.args=SET_Args + """ + + def _flags_code(): + s = "" + for flag in self.desc.get("command_flags", []): + s += "CMD_%s|" % flag + return s[:-1] if s else 0 + + def _acl_categories_code(): + s = "" + for cat in self.desc.get("acl_categories", []): + s += "ACL_CATEGORY_%s|" % cat + return s[:-1] if s else 0 + + def _doc_flags_code(): + s = "" + for flag in self.desc.get("doc_flags", []): + s += "CMD_DOC_%s|" % flag + return s[:-1] if s else "CMD_DOC_NONE" + + s = "MAKE_CMD(\"%s\",%s,%s,%s,%s,%s,%s,%s,%s,%s,%d,%s,%d,%s,%d,%s,%s,%s,%d,%s,%d)," % ( + self.name.lower(), + get_optional_desc_string(self.desc, "summary"), + get_optional_desc_string(self.desc, "complexity"), + get_optional_desc_string(self.desc, "since"), + _doc_flags_code(), + get_optional_desc_string(self.desc, "replaced_by"), + get_optional_desc_string(self.desc, "deprecated_since"), + "\"%s\"" % self.group, + GROUPS[self.group], + self.history_table_name(), + self.num_history(), + self.tips_table_name(), + self.num_tips(), + self.desc.get("function", "NULL"), + self.desc["arity"], + _flags_code(), + _acl_categories_code(), + self.key_specs_table_name(), + len(self.key_specs), + self.desc.get("get_keys_function", "NULL"), + len(self.args), + ) + + if self.subcommands: + s += ".subcommands=%s," % self.subcommand_table_name() + + if self.args: + s += ".args=%s," % self.arg_table_name() + + if self.reply_schema and args.with_reply_schema: + s += ".reply_schema=&%s," % self.reply_schema_name() + + return s[:-1] + + def write_internal_structs(self, f): + if self.subcommands: + subcommand_list = sorted(self.subcommands, key=lambda cmd: cmd.name) + for subcommand in subcommand_list: + subcommand.write_internal_structs(f) + + f.write("/* %s command table */\n" % self.fullname()) + f.write("struct COMMAND_STRUCT %s[] = {\n" % self.subcommand_table_name()) + for subcommand in subcommand_list: + f.write("{%s},\n" % subcommand.struct_code()) + f.write("{0}\n") + f.write("};\n\n") + + f.write("/********** %s ********************/\n\n" % self.fullname()) + + f.write("#ifndef SKIP_CMD_HISTORY_TABLE\n") + f.write("/* %s history */\n" % self.fullname()) + code = self.history_code() + if code: + f.write("commandHistory %s[] = {\n" % self.history_table_name()) + f.write("%s" % code) + f.write("};\n") + else: + f.write("#define %s NULL\n" % self.history_table_name()) + f.write("#endif\n\n") + + f.write("#ifndef SKIP_CMD_TIPS_TABLE\n") + f.write("/* %s tips */\n" % self.fullname()) + code = self.tips_code() + if code: + f.write("const char *%s[] = {\n" % self.tips_table_name()) + f.write("%s" % code) + f.write("};\n") + else: + f.write("#define %s NULL\n" % self.tips_table_name()) + f.write("#endif\n\n") + + f.write("#ifndef SKIP_CMD_KEY_SPECS_TABLE\n") + f.write("/* %s key specs */\n" % self.fullname()) + code = self.key_specs_code() + if code: + f.write("keySpec %s[%d] = {\n" % (self.key_specs_table_name(), len(self.key_specs))) + f.write("%s\n" % code) + f.write("};\n") + else: + f.write("#define %s NULL\n" % self.key_specs_table_name()) + f.write("#endif\n\n") + + if self.args: + for arg in self.args: + arg.write_internal_structs(f) + + f.write("/* %s argument table */\n" % self.fullname()) + f.write("struct COMMAND_ARG %s[] = {\n" % self.arg_table_name()) + for arg in self.args: + f.write("{%s},\n" % arg.struct_code()) + f.write("};\n\n") + + if self.reply_schema and args.with_reply_schema: + self.reply_schema.write(f) + + +class Subcommand(Command): + def __init__(self, name, desc): + self.container_name = desc["container"].upper() + super(Subcommand, self).__init__(name, desc) + + def fullname(self): + return "%s %s" % (self.container_name, self.name.replace("-", "_").replace(":", "")) + + +def create_command(name, desc): + if desc.get("container"): + cmd = Subcommand(name.upper(), desc) + subcommands.setdefault(desc["container"].upper(), {})[name] = cmd + else: + cmd = Command(name.upper(), desc) + commands[name.upper()] = cmd + + +# MAIN + +# Figure out where the sources are +srcdir = os.path.abspath(os.path.dirname(os.path.abspath(__file__)) + "/../src") + +parser = argparse.ArgumentParser() +parser.add_argument('--with-reply-schema', action='store_true') +args = parser.parse_args() + +# Create all command objects +print("Processing json files...") +for filename in glob.glob('%s/commands/*.json' % srcdir): + with open(filename, "r") as f: + try: + d = json.load(f) + for name, desc in d.items(): + create_command(name, desc) + except json.decoder.JSONDecodeError as err: + print("Error processing %s: %s" % (filename, err)) + exit(1) + +# Link subcommands to containers +print("Linking container command to subcommands...") +for command in commands.values(): + assert command.group + if command.name not in subcommands: + continue + for subcommand in subcommands[command.name].values(): + assert not subcommand.group or subcommand.group == command.group + subcommand.group = command.group + command.subcommands.append(subcommand) + +check_command_error_counter = 0 # An error counter is used to count errors in command checking. + +print("Checking all commands...") +for command in commands.values(): + if not check_command_key_specs(command): + check_command_error_counter += 1 + +if check_command_error_counter != 0: + print("Error: There are errors in the commands check, please check the above logs.") + exit(1) + +commands_filename = "commands_with_reply_schema" if args.with_reply_schema else "commands" +print("Generating %s.def..." % commands_filename) +with open("%s/%s.def" % (srcdir, commands_filename), "w") as f: + f.write("/* Automatically generated by %s, do not edit. */\n\n" % os.path.basename(__file__)) + f.write( +""" +/* We have fabulous commands from + * the fantastic + * Redis Command Table! */ + +/* Must match redisCommandGroup */ +const char *COMMAND_GROUP_STR[] = { + "generic", + "string", + "list", + "set", + "sorted-set", + "hash", + "pubsub", + "transactions", + "connection", + "server", + "scripting", + "hyperloglog", + "cluster", + "sentinel", + "geo", + "stream", + "bitmap", + "module" +}; + +const char *commandGroupStr(int index) { + return COMMAND_GROUP_STR[index]; +} +""" + ) + + command_list = sorted(commands.values(), key=lambda cmd: (cmd.group, cmd.name)) + for command in command_list: + command.write_internal_structs(f) + + f.write("/* Main command table */\n") + f.write("struct COMMAND_STRUCT redisCommandTable[] = {\n") + curr_group = None + for command in command_list: + if curr_group != command.group: + curr_group = command.group + f.write("/* %s */\n" % curr_group) + f.write("{%s},\n" % command.struct_code()) + f.write("{0}\n") + f.write("};\n") + +print("All done, exiting.") diff --git a/utils/generate-commands-json.py b/utils/generate-commands-json.py new file mode 100755 index 0000000..23782ea --- /dev/null +++ b/utils/generate-commands-json.py @@ -0,0 +1,136 @@ +#!/usr/bin/env python3 +import argparse +import json +import os +import subprocess +from collections import OrderedDict +from sys import argv + + +def convert_flags_to_boolean_dict(flags): + """Return a dict with a key set to `True` per element in the flags list.""" + return {f: True for f in flags} + + +def set_if_not_none_or_empty(dst, key, value): + """Set 'key' in 'dst' if 'value' is not `None` or an empty list.""" + if value is not None and (type(value) is not list or len(value)): + dst[key] = value + + +def convert_argument(arg): + """Transform an argument.""" + arg.update(convert_flags_to_boolean_dict(arg.pop('flags', []))) + set_if_not_none_or_empty(arg, 'arguments', + [convert_argument(x) for x in arg.pop('arguments', [])]) + return arg + + +def convert_keyspec(spec): + """Transform a key spec.""" + spec.update(convert_flags_to_boolean_dict(spec.pop('flags', []))) + return spec + + +def convert_entry_to_objects_array(cmd, docs): + """Transform the JSON output of `COMMAND` to a friendlier format. + + cmd is the output of `COMMAND` as follows: + 1. Name (lower case, e.g. "lolwut") + 2. Arity + 3. Flags + 4-6. First/last/step key specification (deprecated as of Redis v7.0) + 7. ACL categories + 8. hints (as of Redis 7.0) + 9. key-specs (as of Redis 7.0) + 10. subcommands (as of Redis 7.0) + + docs is the output of `COMMAND DOCS`, which holds a map of additional metadata + + This returns a list with a dict for the command and per each of its + subcommands. Each dict contains one key, the command's full name, with a + value of a dict that's set with the command's properties and meta + information.""" + assert len(cmd) >= 9 + obj = {} + rep = [obj] + name = cmd[0].upper() + arity = cmd[1] + command_flags = cmd[2] + acl_categories = cmd[6] + hints = cmd[7] + keyspecs = cmd[8] + subcommands = cmd[9] if len(cmd) > 9 else [] + key = name.replace('|', ' ') + + subcommand_docs = docs.pop('subcommands', []) + rep.extend([convert_entry_to_objects_array(x, subcommand_docs[x[0]])[0] for x in subcommands]) + + # The command's value is ordered so the interesting stuff that we care about + # is at the start. Optional `None` and empty list values are filtered out. + value = OrderedDict() + value['summary'] = docs.pop('summary') + value['since'] = docs.pop('since') + value['group'] = docs.pop('group') + set_if_not_none_or_empty(value, 'complexity', docs.pop('complexity', None)) + set_if_not_none_or_empty(value, 'deprecated_since', docs.pop('deprecated_since', None)) + set_if_not_none_or_empty(value, 'replaced_by', docs.pop('replaced_by', None)) + set_if_not_none_or_empty(value, 'history', docs.pop('history', [])) + set_if_not_none_or_empty(value, 'acl_categories', acl_categories) + value['arity'] = arity + set_if_not_none_or_empty(value, 'key_specs', + [convert_keyspec(x) for x in keyspecs]) + set_if_not_none_or_empty(value, 'arguments', + [convert_argument(x) for x in docs.pop('arguments', [])]) + set_if_not_none_or_empty(value, 'command_flags', command_flags) + set_if_not_none_or_empty(value, 'doc_flags', docs.pop('doc_flags', [])) + set_if_not_none_or_empty(value, 'hints', hints) + + # All remaining docs key-value tuples, if any, are appended to the command + # to be future-proof. + while len(docs) > 0: + (k, v) = docs.popitem() + value[k] = v + + obj[key] = value + return rep + + +# Figure out where the sources are +srcdir = os.path.abspath(os.path.dirname(os.path.abspath(__file__)) + "/../src") + +# MAIN +if __name__ == '__main__': + opts = { + 'description': 'Transform the output from `redis-cli --json` using COMMAND and COMMAND DOCS to a single commands.json format.', + 'epilog': f'Usage example: {argv[0]} --cli src/redis-cli --port 6379 > commands.json' + } + parser = argparse.ArgumentParser(**opts) + parser.add_argument('--host', type=str, default='localhost') + parser.add_argument('--port', type=int, default=6379) + parser.add_argument('--cli', type=str, default='%s/redis-cli' % srcdir) + args = parser.parse_args() + + payload = OrderedDict() + cmds = [] + + p = subprocess.Popen([args.cli, '-h', args.host, '-p', str(args.port), '--json', 'command'], stdout=subprocess.PIPE) + stdout, stderr = p.communicate() + commands = json.loads(stdout) + + p = subprocess.Popen([args.cli, '-h', args.host, '-p', str(args.port), '--json', 'command', 'docs'], + stdout=subprocess.PIPE) + stdout, stderr = p.communicate() + docs = json.loads(stdout) + + for entry in commands: + cmd = convert_entry_to_objects_array(entry, docs[entry[0]]) + cmds.extend(cmd) + + # The final output is a dict of all commands, ordered by name. + cmds.sort(key=lambda x: list(x.keys())[0]) + for cmd in cmds: + name = list(cmd.keys())[0] + payload[name] = cmd[name] + + print(json.dumps(payload, indent=4)) diff --git a/utils/generate-module-api-doc.rb b/utils/generate-module-api-doc.rb new file mode 100755 index 0000000..8829434 --- /dev/null +++ b/utils/generate-module-api-doc.rb @@ -0,0 +1,205 @@ +#!/usr/bin/env ruby +# coding: utf-8 +# gendoc.rb -- Converts the top-comments inside module.c to modules API +# reference documentation in markdown format. + +# Convert the C comment to markdown +def markdown(s) + s = s.gsub(/\*\/$/,"") + s = s.gsub(/^ ?\* ?/,"") + s = s.gsub(/^\/\*\*? ?/,"") + s.chop! while s[-1] == "\n" || s[-1] == " " + lines = s.split("\n") + newlines = [] + # Fix some markdown + lines.each{|l| + # Rewrite RM_Xyz() to RedisModule_Xyz(). + l = l.gsub(/(?\n\n" + puts "### `#{name}`\n\n" + puts " #{proto}\n" + puts "**Available since:** #{$since[name] or "unreleased"}\n\n" + comment = "" + while true + i = i-1 + comment = src[i]+comment + break if src[i] =~ /\/\*/ + end + comment = markdown(comment) + puts comment+"\n\n" +end + +# Print a comment from line until */ is found, as markdown. +def section_doc(src, i) + name = get_section_heading(src, i) + comment = "\n\n" + while true + # append line, except if it's a horizontal divider + comment = comment + src[i] if src[i] !~ /^[\/ ]?\*{1,2} ?-{50,}/ + break if src[i] =~ /\*\// + i = i+1 + end + comment = markdown(comment) + puts comment+"\n\n" +end + +# generates an id suitable for links within the page +def section_name_to_id(name) + return "section-" + + name.strip.downcase.gsub(/[^a-z0-9]+/, '-').gsub(/^-+|-+$/, '') +end + +# Returns the name of the first section heading in the comment block for which +# is_section_doc(src, i) is true +def get_section_heading(src, i) + if src[i] =~ /^\/\*\*? \#+ *(.*)/ + heading = $1 + elsif src[i+1] =~ /^ ?\* \#+ *(.*)/ + heading = $1 + end + return heading.gsub(' -- ', ' – ') +end + +# Returns true if the line is the start of a generic documentation section. Such +# section must start with the # symbol, i.e. a markdown heading, on the first or +# the second line. +def is_section_doc(src, i) + return src[i] =~ /^\/\*\*? \#/ || + (src[i] =~ /^\/\*/ && src[i+1] =~ /^ ?\* \#/) +end + +def is_func_line(src, i) + line = src[i] + return line =~ /RM_/ && + line[0] != ' ' && line[0] != '#' && line[0] != '/' && + src[i-1] =~ /\*\// +end + +puts "---\n" +puts "title: \"Modules API reference\"\n" +puts "linkTitle: \"API reference\"\n" +puts "weight: 1\n" +puts "description: >\n" +puts " Reference for the Redis Modules API\n" +puts "aliases:\n" +puts " - /topics/modules-api-ref\n" +puts "---\n" +puts "\n" +puts "\n\n" +src = File.open(File.dirname(__FILE__) ++ "/../src/module.c").to_a + +# Build function index +$index = {} +src.each_with_index do |line,i| + if is_func_line(src, i) + line =~ /RM_([A-z0-9]+)/ + name = "RedisModule_#{$1}" + $index[name] = true + end +end + +# Populate the 'since' map (name => version) if we're in a git repo. +$since = {} +git_dir = File.dirname(__FILE__) ++ "/../.git" +if File.directory?(git_dir) && `which git` != "" + `git --git-dir="#{git_dir}" tag --sort=v:refname`.each_line do |version| + next if version !~ /^(\d+)\.\d+\.\d+?$/ || $1.to_i < 4 + version.chomp! + `git --git-dir="#{git_dir}" cat-file blob "#{version}:src/module.c"`.each_line do |line| + if line =~ /^\w.*[ \*]RM_([A-z0-9]+)/ + name = "RedisModule_#{$1}" + if ! $since[name] + $since[name] = version + end + end + end + end +end + +# Print TOC +puts "## Sections\n\n" +src.each_with_index do |_line,i| + if is_section_doc(src, i) + name = get_section_heading(src, i) + puts "* [#{name}](\##{section_name_to_id(name)})\n" + end +end +puts "* [Function index](#section-function-index)\n\n" + +# Docufy: Print function prototype and markdown docs +src.each_with_index do |_line,i| + if is_func_line(src, i) + docufy(src, i) + elsif is_section_doc(src, i) + section_doc(src, i) + end +end + +# Print function index +puts "\n\n" +puts "## Function index\n\n" +$index.keys.sort.each{|x| puts "* [`#{x}`](\##{x})\n"} +puts "\n" diff --git a/utils/graphs/commits-over-time/README.md b/utils/graphs/commits-over-time/README.md new file mode 100644 index 0000000..b28019e --- /dev/null +++ b/utils/graphs/commits-over-time/README.md @@ -0,0 +1,16 @@ +This Tcl script is what I used in order to generate the graph you +can find at http://antirez.com/news/98. It's really quick & dirty, more +a trow away program than anything else, but probably could be reused or +modified in the future in order to visualize other similar data or an +updated version of the same data. + +The usage is trivial: + + ./genhtml.tcl > output.html + +The generated HTML is quite broken but good enough to grab a screenshot +from the browser. Feel free to improve it if you got time / interest. + +Note that the code filtering the tags, and the hardcoded branch name, does +not make the script, as it is, able to analyze a different repository. +However the changes needed are trivial. diff --git a/utils/graphs/commits-over-time/genhtml.tcl b/utils/graphs/commits-over-time/genhtml.tcl new file mode 100755 index 0000000..c4b4e09 --- /dev/null +++ b/utils/graphs/commits-over-time/genhtml.tcl @@ -0,0 +1,96 @@ +#!/usr/bin/env tclsh + +# Load commits history as "sha1 unixtime". +set commits [exec git log unstable {--pretty="%H %at"}] +set raw_tags [exec git tag] + +# Load all the tags that are about stable releases. +foreach tag $raw_tags { + if {[string match v*-stable $tag]} { + set tag [string range $tag 1 end-7] + puts $tag + } + if {[regexp {^[0-9]+.[0-9]+.[0-9]+$} $tag]} { + lappend tags $tag + } +} + +# For each tag, create a list of "name unixtime" +foreach tag $tags { + set taginfo [exec git log $tag -n 1 "--pretty=\"$tag %at\""] + set taginfo [string trim $taginfo {"}] + lappend labels $taginfo +} + +# For each commit, check the amount of code changed and create an array +# mapping the commit to the number of lines affected. +foreach c $commits { + set stat [exec git show --oneline --numstat [lindex $c 0]] + set linenum 0 + set affected 0 + foreach line [split $stat "\n"] { + incr linenum + if {$linenum == 1 || [string match *deps/* $line]} continue + if {[catch {llength $line} numfields]} continue + if {$numfields == 0} continue + catch { + incr affected [lindex $line 0] + incr affected [lindex $line 1] + } + } + set commit_to_affected([lindex $c 0]) $affected +} + +set base_time [lindex [lindex $commits end] 1] +puts [clock format $base_time] + +# Generate a graph made of HTML DIVs. +puts { + +
+} +foreach c $commits { + set sha [lindex $c 0] + set t [expr {([lindex $c 1]-$base_time)/(3600*24*2)}] + set affected [expr $commit_to_affected($sha)] + set left $t + set height [expr {log($affected)*20}] + puts "
" +} + +set bottom -30 +foreach l $labels { + set name [lindex $l 0] + set t [expr {([lindex $l 1]-$base_time)/(3600*24*2)}] + set left $t + if {$left < 0} continue + incr bottom -20 + if {$bottom == -210} {set bottom -30} + puts "
$name
" +} +puts {
} diff --git a/utils/hyperloglog/.gitignore b/utils/hyperloglog/.gitignore new file mode 100644 index 0000000..2211df6 --- /dev/null +++ b/utils/hyperloglog/.gitignore @@ -0,0 +1 @@ +*.txt diff --git a/utils/hyperloglog/hll-err.rb b/utils/hyperloglog/hll-err.rb new file mode 100644 index 0000000..2c71ac5 --- /dev/null +++ b/utils/hyperloglog/hll-err.rb @@ -0,0 +1,27 @@ +# hll-err.rb - Copyright (C) 2014 Salvatore Sanfilippo +# BSD license, See the COPYING file for more information. +# +# Check error of HyperLogLog Redis implementation for different set sizes. + +require 'rubygems' +require 'redis' +require 'digest/sha1' + +r = Redis.new +r.del('hll') +i = 0 +while true do + 100.times { + elements = [] + 1000.times { + ele = Digest::SHA1.hexdigest(i.to_s) + elements << ele + i += 1 + } + r.pfadd('hll',elements) + } + approx = r.pfcount('hll') + abs_err = (approx-i).abs + rel_err = 100.to_f*abs_err/i + puts "#{i} vs #{approx}: #{rel_err}%" +end diff --git a/utils/hyperloglog/hll-gnuplot-graph.rb b/utils/hyperloglog/hll-gnuplot-graph.rb new file mode 100644 index 0000000..6c7596d --- /dev/null +++ b/utils/hyperloglog/hll-gnuplot-graph.rb @@ -0,0 +1,88 @@ +# hll-err.rb - Copyright (C) 2014 Salvatore Sanfilippo +# BSD license, See the COPYING file for more information. +# +# This program is suited to output average and maximum errors of +# the Redis HyperLogLog implementation in a format suitable to print +# graphs using gnuplot. + +require 'rubygems' +require 'redis' +require 'digest/sha1' + +# Generate an array of [cardinality,relative_error] pairs +# in the 0 - max range, with the specified step. +# +# 'r' is the Redis object used to perform the queries. +# 'seed' must be different every time you want a test performed +# with a different set. The function guarantees that if 'seed' is the +# same, exactly the same dataset is used, and when it is different, +# a totally unrelated different data set is used (without any common +# element in practice). +def run_experiment(r,seed,max,step) + r.del('hll') + i = 0 + samples = [] + step = 1000 if step > 1000 + while i < max do + elements = [] + step.times { + ele = Digest::SHA1.hexdigest(i.to_s+seed.to_s) + elements << ele + i += 1 + } + r.pfadd('hll',elements) + approx = r.pfcount('hll') + err = approx-i + rel_err = 100.to_f*err/i + samples << [i,rel_err] + end + samples +end + +def filter_samples(numsets,max,step,filter) + r = Redis.new + dataset = {} + (0...numsets).each{|i| + dataset[i] = run_experiment(r,i,max,step) + STDERR.puts "Set #{i}" + } + dataset[0].each_with_index{|ele,index| + if filter == :max + card=ele[0] + err=ele[1].abs + (1...numsets).each{|i| + err = dataset[i][index][1] if err < dataset[i][index][1] + } + puts "#{card} #{err}" + elsif filter == :avg + card=ele[0] + err = 0 + (0...numsets).each{|i| + err += dataset[i][index][1] + } + err /= numsets + puts "#{card} #{err}" + elsif filter == :absavg + card=ele[0] + err = 0 + (0...numsets).each{|i| + err += dataset[i][index][1].abs + } + err /= numsets + puts "#{card} #{err}" + elsif filter == :all + (0...numsets).each{|i| + card,err = dataset[i][index] + puts "#{card} #{err}" + } + else + raise "Unknown filter #{filter}" + end + } +end + +if ARGV.length != 4 + puts "Usage: hll-gnuplot-graph (max|avg|absavg|all)" + exit 1 +end +filter_samples(ARGV[0].to_i,ARGV[1].to_i,ARGV[2].to_i,ARGV[3].to_sym) diff --git a/utils/install_server.sh b/utils/install_server.sh new file mode 100755 index 0000000..efda7da --- /dev/null +++ b/utils/install_server.sh @@ -0,0 +1,291 @@ +#!/bin/sh + +# Copyright 2011 Dvir Volk . All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED +# WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO +# EVENT SHALL Dvir Volk OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, +# OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, +# EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +################################################################################ +# +# Service installer for redis server, runs interactively by default. +# +# To run this script non-interactively (for automation/provisioning purposes), +# feed the variables into the script. Any missing variables will be prompted! +# Tip: Environment variables also support command substitution (see REDIS_EXECUTABLE) +# +# Example: +# +# sudo REDIS_PORT=1234 \ +# REDIS_CONFIG_FILE=/etc/redis/1234.conf \ +# REDIS_LOG_FILE=/var/log/redis_1234.log \ +# REDIS_DATA_DIR=/var/lib/redis/1234 \ +# REDIS_EXECUTABLE=`command -v redis-server` ./utils/install_server.sh +# +# This generates a redis config file and an /etc/init.d script, and installs them. +# +# /!\ This script should be run as root +# +# NOTE: This script will not work on Mac OSX. +# It supports Debian and Ubuntu Linux. +# +################################################################################ + +die () { + echo "ERROR: $1. Aborting!" + exit 1 +} + + +#Absolute path to this script +SCRIPT=$(readlink -f $0) +#Absolute path this script is in +SCRIPTPATH=$(dirname $SCRIPT) + +#Initial defaults +_REDIS_PORT=6379 +_MANUAL_EXECUTION=false + +echo "Welcome to the redis service installer" +echo "This script will help you easily set up a running redis server" +echo + +#check for root user +if [ "$(id -u)" -ne 0 ] ; then + echo "You must run this script as root. Sorry!" + exit 1 +fi + +#bail if this system is managed by systemd +_pid_1_exe="$(readlink -f /proc/1/exe)" +if [ "${_pid_1_exe##*/}" = systemd ] +then + echo "This systems seems to use systemd." + echo "Please take a look at the provided example service unit files in this directory, and adapt and install them. Sorry!" + exit 1 +fi +unset _pid_1_exe + +if ! echo $REDIS_PORT | egrep -q '^[0-9]+$' ; then + _MANUAL_EXECUTION=true + #Read the redis port + read -p "Please select the redis port for this instance: [$_REDIS_PORT] " REDIS_PORT + if ! echo $REDIS_PORT | egrep -q '^[0-9]+$' ; then + echo "Selecting default: $_REDIS_PORT" + REDIS_PORT=$_REDIS_PORT + fi +fi + +if [ -z "$REDIS_CONFIG_FILE" ] ; then + _MANUAL_EXECUTION=true + #read the redis config file + _REDIS_CONFIG_FILE="/etc/redis/$REDIS_PORT.conf" + read -p "Please select the redis config file name [$_REDIS_CONFIG_FILE] " REDIS_CONFIG_FILE + if [ -z "$REDIS_CONFIG_FILE" ] ; then + REDIS_CONFIG_FILE=$_REDIS_CONFIG_FILE + echo "Selected default - $REDIS_CONFIG_FILE" + fi +fi + +if [ -z "$REDIS_LOG_FILE" ] ; then + _MANUAL_EXECUTION=true + #read the redis log file path + _REDIS_LOG_FILE="/var/log/redis_$REDIS_PORT.log" + read -p "Please select the redis log file name [$_REDIS_LOG_FILE] " REDIS_LOG_FILE + if [ -z "$REDIS_LOG_FILE" ] ; then + REDIS_LOG_FILE=$_REDIS_LOG_FILE + echo "Selected default - $REDIS_LOG_FILE" + fi +fi + +if [ -z "$REDIS_DATA_DIR" ] ; then + _MANUAL_EXECUTION=true + #get the redis data directory + _REDIS_DATA_DIR="/var/lib/redis/$REDIS_PORT" + read -p "Please select the data directory for this instance [$_REDIS_DATA_DIR] " REDIS_DATA_DIR + if [ -z "$REDIS_DATA_DIR" ] ; then + REDIS_DATA_DIR=$_REDIS_DATA_DIR + echo "Selected default - $REDIS_DATA_DIR" + fi +fi + +if [ ! -x "$REDIS_EXECUTABLE" ] ; then + _MANUAL_EXECUTION=true + #get the redis executable path + _REDIS_EXECUTABLE=`command -v redis-server` + read -p "Please select the redis executable path [$_REDIS_EXECUTABLE] " REDIS_EXECUTABLE + if [ ! -x "$REDIS_EXECUTABLE" ] ; then + REDIS_EXECUTABLE=$_REDIS_EXECUTABLE + + if [ ! -x "$REDIS_EXECUTABLE" ] ; then + echo "Mmmmm... it seems like you don't have a redis executable. Did you run make install yet?" + exit 1 + fi + fi +fi + +#check the default for redis cli +CLI_EXEC=`command -v redis-cli` +if [ -z "$CLI_EXEC" ] ; then + CLI_EXEC=`dirname $REDIS_EXECUTABLE`"/redis-cli" +fi + +echo "Selected config:" + +echo "Port : $REDIS_PORT" +echo "Config file : $REDIS_CONFIG_FILE" +echo "Log file : $REDIS_LOG_FILE" +echo "Data dir : $REDIS_DATA_DIR" +echo "Executable : $REDIS_EXECUTABLE" +echo "Cli Executable : $CLI_EXEC" + +if $_MANUAL_EXECUTION == true ; then + read -p "Is this ok? Then press ENTER to go on or Ctrl-C to abort." _UNUSED_ +fi + +mkdir -p `dirname "$REDIS_CONFIG_FILE"` || die "Could not create redis config directory" +mkdir -p `dirname "$REDIS_LOG_FILE"` || die "Could not create redis log dir" +mkdir -p "$REDIS_DATA_DIR" || die "Could not create redis data directory" + +#render the templates +TMP_FILE="/tmp/${REDIS_PORT}.conf" +DEFAULT_CONFIG="${SCRIPTPATH}/../redis.conf" +INIT_TPL_FILE="${SCRIPTPATH}/redis_init_script.tpl" +INIT_SCRIPT_DEST="/etc/init.d/redis_${REDIS_PORT}" +PIDFILE="/var/run/redis_${REDIS_PORT}.pid" + +if [ ! -f "$DEFAULT_CONFIG" ]; then + echo "Mmmmm... the default config is missing. Did you switch to the utils directory?" + exit 1 +fi + +#Generate config file from the default config file as template +#changing only the stuff we're controlling from this script +echo "## Generated by install_server.sh ##" > $TMP_FILE + +read -r SED_EXPR <<-EOF +s#^port .\+#port ${REDIS_PORT}#; \ +s#^logfile .\+#logfile ${REDIS_LOG_FILE}#; \ +s#^dir .\+#dir ${REDIS_DATA_DIR}#; \ +s#^pidfile .\+#pidfile ${PIDFILE}#; \ +s#^daemonize no#daemonize yes#; +EOF +sed "$SED_EXPR" $DEFAULT_CONFIG >> $TMP_FILE + +#cat $TPL_FILE | while read line; do eval "echo \"$line\"" >> $TMP_FILE; done +cp $TMP_FILE $REDIS_CONFIG_FILE || die "Could not write redis config file $REDIS_CONFIG_FILE" + +#Generate sample script from template file +rm -f $TMP_FILE + +#we hard code the configs here to avoid issues with templates containing env vars +#kinda lame but works! +REDIS_INIT_HEADER=\ +"#!/bin/sh\n +#Configurations injected by install_server below....\n\n +EXEC=$REDIS_EXECUTABLE\n +CLIEXEC=$CLI_EXEC\n +PIDFILE=\"$PIDFILE\"\n +CONF=\"$REDIS_CONFIG_FILE\"\n\n +REDISPORT=\"$REDIS_PORT\"\n\n +###############\n\n" + +REDIS_CHKCONFIG_INFO=\ +"# REDHAT chkconfig header\n\n +# chkconfig: - 58 74\n +# description: redis_${REDIS_PORT} is the redis daemon.\n +### BEGIN INIT INFO\n +# Provides: redis_6379\n +# Required-Start: \$network \$local_fs \$remote_fs\n +# Required-Stop: \$network \$local_fs \$remote_fs\n +# Default-Start: 2 3 4 5\n +# Default-Stop: 0 1 6\n +# Should-Start: \$syslog \$named\n +# Should-Stop: \$syslog \$named\n +# Short-Description: start and stop redis_${REDIS_PORT}\n +# Description: Redis daemon\n +### END INIT INFO\n\n" + +if command -v chkconfig >/dev/null; then + #if we're a box with chkconfig on it we want to include info for chkconfig + echo "$REDIS_INIT_HEADER" "$REDIS_CHKCONFIG_INFO" > $TMP_FILE && cat $INIT_TPL_FILE >> $TMP_FILE || die "Could not write init script to $TMP_FILE" +else + #combine the header and the template (which is actually a static footer) + echo "$REDIS_INIT_HEADER" > $TMP_FILE && cat $INIT_TPL_FILE >> $TMP_FILE || die "Could not write init script to $TMP_FILE" +fi + +### +# Generate sample script from template file +# - No need to check which system we are on. The init info are comments and +# do not interfere with update_rc.d systems. Additionally: +# Ubuntu/debian by default does not come with chkconfig, but does issue a +# warning if init info is not available. + +cat > ${TMP_FILE} <> ${TMP_FILE} + +#copy to /etc/init.d +cp $TMP_FILE $INIT_SCRIPT_DEST && \ + chmod +x $INIT_SCRIPT_DEST || die "Could not copy redis init script to $INIT_SCRIPT_DEST" +echo "Copied $TMP_FILE => $INIT_SCRIPT_DEST" + +#Install the service +echo "Installing service..." +if command -v chkconfig >/dev/null 2>&1; then + # we're chkconfig, so lets add to chkconfig and put in runlevel 345 + chkconfig --add redis_${REDIS_PORT} && echo "Successfully added to chkconfig!" + chkconfig --level 345 redis_${REDIS_PORT} on && echo "Successfully added to runlevels 345!" +elif command -v update-rc.d >/dev/null 2>&1; then + #if we're not a chkconfig box assume we're able to use update-rc.d + update-rc.d redis_${REDIS_PORT} defaults && echo "Success!" +else + echo "No supported init tool found." +fi + +/etc/init.d/redis_$REDIS_PORT start || die "Failed starting service..." + +#tada +echo "Installation successful!" +exit 0 diff --git a/utils/lru/README b/utils/lru/README new file mode 100644 index 0000000..f043b29 --- /dev/null +++ b/utils/lru/README @@ -0,0 +1,19 @@ +The test-lru.rb program can be used in order to check the behavior of the +Redis approximated LRU algorithm against the theoretical output of true +LRU algorithm. + +In order to use the program you need to recompile Redis setting the define +REDIS_LRU_CLOCK_RESOLUTION to 1, by editing the file server.h. +This allows to execute the program in a fast way since the 1 ms resolution +is enough for all the objects to have a different enough time stamp during +the test. + +The program is executed like this: + + ruby test-lru.rb /tmp/lru.html + +You can optionally specify a number of times to run, so that the program +will output averages of different runs, by adding an additional argument. +For instance in order to run the test 10 times use: + + ruby test-lru.rb /tmp/lru.html 10 diff --git a/utils/lru/lfu-simulation.c b/utils/lru/lfu-simulation.c new file mode 100644 index 0000000..60105e5 --- /dev/null +++ b/utils/lru/lfu-simulation.c @@ -0,0 +1,158 @@ +#include +#include +#include +#include + +int decr_every = 1; +int keyspace_size = 1000000; +time_t switch_after = 30; /* Switch access pattern after N seconds. */ + +struct entry { + /* Field that the LFU Redis implementation will have (we have + * 24 bits of total space in the object->lru field). */ + uint8_t counter; /* Logarithmic counter. */ + uint16_t decrtime; /* (Reduced precision) time of last decrement. */ + + /* Fields only useful for visualization. */ + uint64_t hits; /* Number of real accesses. */ + time_t ctime; /* Key creation time. */ +}; + +#define to_16bit_minutes(x) ((x/60) & 65535) +#define LFU_INIT_VAL 5 + +/* Compute the difference in minutes between two 16 bit minutes times + * obtained with to_16bit_minutes(). Since they can wrap around if + * we detect the overflow we account for it as if the counter wrapped + * a single time. */ +uint16_t minutes_diff(uint16_t now, uint16_t prev) { + if (now >= prev) return now-prev; + return 65535-prev+now; +} + +/* Increment a counter logarithmically: the greatest is its value, the + * less likely is that the counter is really incremented. + * The maximum value of the counter is saturated at 255. */ +uint8_t log_incr(uint8_t counter) { + if (counter == 255) return counter; + double r = (double)rand()/RAND_MAX; + double baseval = counter-LFU_INIT_VAL; + if (baseval < 0) baseval = 0; + double limit = 1.0/(baseval*10+1); + if (r < limit) counter++; + return counter; +} + +/* Simulate an access to an entry. */ +void access_entry(struct entry *e) { + e->counter = log_incr(e->counter); + e->hits++; +} + +/* Return the entry LFU value and as a side effect decrement the + * entry value if the decrement time was reached. */ +uint8_t scan_entry(struct entry *e) { + if (minutes_diff(to_16bit_minutes(time(NULL)),e->decrtime) + >= decr_every) + { + if (e->counter) { + if (e->counter > LFU_INIT_VAL*2) { + e->counter /= 2; + } else { + e->counter--; + } + } + e->decrtime = to_16bit_minutes(time(NULL)); + } + return e->counter; +} + +/* Print the entry info. */ +void show_entry(long pos, struct entry *e) { + char *tag = "normal "; + + if (pos >= 10 && pos <= 14) tag = "new no access"; + if (pos >= 15 && pos <= 19) tag = "new accessed "; + if (pos >= keyspace_size -5) tag= "old no access"; + + printf("%ld] <%s> frequency:%d decrtime:%d [%lu hits | age:%ld sec]\n", + pos, tag, e->counter, e->decrtime, (unsigned long)e->hits, + time(NULL) - e->ctime); +} + +int main(void) { + time_t start = time(NULL); + time_t new_entry_time = start; + time_t display_time = start; + struct entry *entries = malloc(sizeof(*entries)*keyspace_size); + long j; + + /* Initialize. */ + for (j = 0; j < keyspace_size; j++) { + entries[j].counter = LFU_INIT_VAL; + entries[j].decrtime = to_16bit_minutes(start); + entries[j].hits = 0; + entries[j].ctime = time(NULL); + } + + while(1) { + time_t now = time(NULL); + long idx; + + /* Scan N random entries (simulates the eviction under maxmemory). */ + for (j = 0; j < 3; j++) { + scan_entry(entries+(rand()%keyspace_size)); + } + + /* Access a random entry: use a power-law access pattern up to + * 'switch_after' seconds. Then revert to flat access pattern. */ + if (now-start < switch_after) { + /* Power law. */ + idx = 1; + while((rand() % 21) != 0 && idx < keyspace_size) idx *= 2; + if (idx > keyspace_size) idx = keyspace_size; + idx = rand() % idx; + } else { + /* Flat. */ + idx = rand() % keyspace_size; + } + + /* Never access entries between position 10 and 14, so that + * we simulate what happens to new entries that are never + * accessed VS new entries which are accessed in positions + * 15-19. + * + * Also never access last 5 entry, so that we have keys which + * are never recreated (old), and never accessed. */ + if ((idx < 10 || idx > 14) && (idx < keyspace_size-5)) + access_entry(entries+idx); + + /* Simulate the addition of new entries at positions between + * 10 and 19, a random one every 10 seconds. */ + if (new_entry_time <= now) { + idx = 10+(rand()%10); + entries[idx].counter = LFU_INIT_VAL; + entries[idx].decrtime = to_16bit_minutes(time(NULL)); + entries[idx].hits = 0; + entries[idx].ctime = time(NULL); + new_entry_time = now+10; + } + + /* Show the first 20 entries and the last 20 entries. */ + if (display_time != now) { + printf("=============================\n"); + printf("Current minutes time: %d\n", (int)to_16bit_minutes(now)); + printf("Access method: %s\n", + (now-start < switch_after) ? "power-law" : "flat"); + + for (j = 0; j < 20; j++) + show_entry(j,entries+j); + + for (j = keyspace_size-20; j < keyspace_size; j++) + show_entry(j,entries+j); + display_time = now; + } + } + return 0; +} + diff --git a/utils/lru/test-lru.rb b/utils/lru/test-lru.rb new file mode 100644 index 0000000..d511e20 --- /dev/null +++ b/utils/lru/test-lru.rb @@ -0,0 +1,223 @@ +require 'rubygems' +require 'redis' + +$runs = []; # Remember the error rate of each run for average purposes. +$o = {}; # Options set parsing arguments + +def testit(filename) + r = Redis.new + r.config("SET","maxmemory","2000000") + if $o[:ttl] + r.config("SET","maxmemory-policy","volatile-ttl") + else + r.config("SET","maxmemory-policy","allkeys-lru") + end + r.config("SET","maxmemory-samples",5) + r.config("RESETSTAT") + r.flushall + + html = "" + html << < + + + 
+EOF
+
+    # Fill the DB up to the first eviction.
+    oldsize = r.dbsize
+    id = 0
+    while true
+        id += 1
+        begin
+            r.set(id,"foo")
+        rescue
+            break
+        end
+        newsize = r.dbsize
+        break if newsize == oldsize # A key was evicted? Stop.
+        oldsize = newsize
+    end
+
+    inserted = r.dbsize
+    first_set_max_id = id
+    html << "#{r.dbsize} keys inserted.\n"
+
+    # Access keys sequentially, so that in theory the first part will be expired
+    # and the latter part will not, according to perfect LRU.
+
+    if $o[:ttl]
+        STDERR.puts "Set increasing expire value"
+        (1..first_set_max_id).each{|id|
+            r.expire(id,1000+id)
+            STDERR.print(".") if (id % 150) == 0
+        }
+    else
+        STDERR.puts "Access keys sequentially"
+        (1..first_set_max_id).each{|id|
+            r.get(id)
+            sleep 0.001
+            STDERR.print(".") if (id % 150) == 0
+        }
+    end
+    STDERR.puts
+
+    # Insert more 50% keys. We expect that the new keys will rarely be expired
+    # since their last access time is recent compared to the others.
+    #
+    # Note that we insert the first 100 keys of the new set into DB1 instead
+    # of DB0, so that we can try how cross-DB eviction works.
+    half = inserted/2
+    html << "Insert enough keys to evict half the keys we inserted.\n"
+    add = 0
+
+    otherdb_start_idx = id+1
+    otherdb_end_idx = id+100
+    while true
+        add += 1
+        id += 1
+        if id >= otherdb_start_idx && id <= otherdb_end_idx
+            r.select(1)
+            r.set(id,"foo")
+            r.select(0)
+        else
+            r.set(id,"foo")
+        end
+        break if r.info['evicted_keys'].to_i >= half
+    end
+
+    html << "#{add} additional keys added.\n"
+    html << "#{r.dbsize} keys in DB.\n"
+
+    # Check if evicted keys respect LRU
+    # We consider errors from 1 to N progressively more serious as they violate
+    # more the access pattern.
+
+    errors = 0
+    e = 1
+    error_per_key = 100000.0/first_set_max_id
+    half_set_size = first_set_max_id/2
+    maxerr = 0
+    (1..(first_set_max_id/2)).each{|id|
+        if id >= otherdb_start_idx && id <= otherdb_end_idx
+            r.select(1)
+            exists = r.exists(id)
+            r.select(0)
+        else
+            exists = r.exists(id)
+        end
+        if id < first_set_max_id/2
+            thiserr = error_per_key * ((half_set_size-id).to_f/half_set_size)
+            maxerr += thiserr
+            errors += thiserr if exists
+        elsif id >= first_set_max_id/2
+            thiserr = error_per_key * ((id-half_set_size).to_f/half_set_size)
+            maxerr += thiserr
+            errors += thiserr if !exists
+        end
+    }
+    errors = errors*100/maxerr
+
+    STDERR.puts "Test finished with #{errors}% error! Generating HTML on stdout."
+
+    html << "#{errors}% error!\n"
+    html << "
" + $runs << errors + + # Generate the graphical representation + (1..id).each{|id| + # Mark first set and added items in a different way. + c = "box" + if id >= otherdb_start_idx && id <= otherdb_end_idx + c << " otherdb" + elsif id <= first_set_max_id + c << " old" + else + c << " new" + end + + # Add class if exists + if id >= otherdb_start_idx && id <= otherdb_end_idx + r.select(1) + exists = r.exists(id) + r.select(0) + else + exists = r.exists(id) + end + + c << " ex" if exists + html << "
" + } + + # Close HTML page + + html << < + +EOF + + f = File.open(filename,"w") + f.write(html) + f.close +end + +def print_avg + avg = ($runs.reduce {|a,b| a+b}) / $runs.length + puts "#{$runs.length} runs, AVG is #{avg}" +end + +if ARGV.length < 1 + STDERR.puts "Usage: ruby test-lru.rb [--runs ] [--ttl]" + STDERR.puts "Options:" + STDERR.puts " --runs Execute the test times." + STDERR.puts " --ttl Set keys with increasing TTL values" + STDERR.puts " (starting from 1000 seconds) in order to" + STDERR.puts " test the volatile-lru policy." + exit 1 +end + +filename = ARGV[0] +$o[:numruns] = 1 + +# Options parsing +i = 1 +while i < ARGV.length + if ARGV[i] == '--runs' + $o[:numruns] = ARGV[i+1].to_i + i+= 1 + elsif ARGV[i] == '--ttl' + $o[:ttl] = true + else + STDERR.puts "Unknown option #{ARGV[i]}" + exit 1 + end + i+= 1 +end + +$o[:numruns].times { + testit(filename) + print_avg if $o[:numruns] != 1 +} diff --git a/utils/redis-copy.rb b/utils/redis-copy.rb new file mode 100644 index 0000000..7c5c52d --- /dev/null +++ b/utils/redis-copy.rb @@ -0,0 +1,35 @@ +# redis-copy.rb - Copyright (C) 2009-2010 Salvatore Sanfilippo +# BSD license, See the COPYING file for more information. +# +# Copy the whole dataset from one Redis instance to another one +# +# WARNING: this utility is deprecated and serves as a legacy adapter +# for the more-robust redis-copy gem. + +require 'shellwords' + +def redisCopy(opts={}) + src = "#{opts[:srchost]}:#{opts[:srcport]}" + dst = "#{opts[:dsthost]}:#{opts[:dstport]}" + `redis-copy #{src.shellescape} #{dst.shellescape}` +rescue Errno::ENOENT + $stderr.puts 'This utility requires the redis-copy executable', + 'from the redis-copy gem on https://rubygems.org', + 'To install it, run `gem install redis-copy`.' + exit 1 +end + +$stderr.puts "This utility is deprecated. Use the redis-copy gem instead." +if ARGV.length != 4 + puts "Usage: redis-copy.rb " + exit 1 +end +puts "WARNING: it's up to you to FLUSHDB the destination host before to continue, press any key when ready." +STDIN.gets +srchost = ARGV[0] +srcport = ARGV[1] +dsthost = ARGV[2] +dstport = ARGV[3] +puts "Copying #{srchost}:#{srcport} into #{dsthost}:#{dstport}" +redisCopy(:srchost => srchost, :srcport => srcport.to_i, + :dsthost => dsthost, :dstport => dstport.to_i) diff --git a/utils/redis-sha1.rb b/utils/redis-sha1.rb new file mode 100644 index 0000000..6a8b4f3 --- /dev/null +++ b/utils/redis-sha1.rb @@ -0,0 +1,52 @@ +# redis-sha1.rb - Copyright (C) 2009 Salvatore Sanfilippo +# BSD license, See the COPYING file for more information. +# +# Performs the SHA1 sum of the whole dataset. +# This is useful to spot bugs in persistence related code and to make sure +# Slaves and Masters are in SYNC. +# +# If you hack this code make sure to sort keys and set elements as this are +# unsorted elements. Otherwise the sum may differ with equal dataset. + +require 'rubygems' +require 'redis' +require 'digest/sha1' + +def redisSha1(opts={}) + sha1="" + r = Redis.new(opts) + r.keys('*').sort.each{|k| + vtype = r.type?(k) + if vtype == "string" + len = 1 + sha1 = Digest::SHA1.hexdigest(sha1+k) + sha1 = Digest::SHA1.hexdigest(sha1+r.get(k)) + elsif vtype == "list" + len = r.llen(k) + if len != 0 + sha1 = Digest::SHA1.hexdigest(sha1+k) + sha1 = Digest::SHA1.hexdigest(sha1+r.list_range(k,0,-1).join("\x01")) + end + elsif vtype == "set" + len = r.scard(k) + if len != 0 + sha1 = Digest::SHA1.hexdigest(sha1+k) + sha1 = Digest::SHA1.hexdigest(sha1+r.set_members(k).to_a.sort.join("\x02")) + end + elsif vtype == "zset" + len = r.zcard(k) + if len != 0 + sha1 = Digest::SHA1.hexdigest(sha1+k) + sha1 = Digest::SHA1.hexdigest(sha1+r.zrange(k,0,-1).join("\x01")) + end + end + # puts "#{k} => #{sha1}" if len != 0 + } + sha1 +end + +host = ARGV[0] || "127.0.0.1" +port = ARGV[1] || "6379" +db = ARGV[2] || "0" +puts "Performing SHA1 of Redis server #{host} #{port} DB: #{db}" +p "Dataset SHA1: #{redisSha1(:host => host, :port => port.to_i, :db => db)}" diff --git a/utils/redis_init_script b/utils/redis_init_script new file mode 100755 index 0000000..006db87 --- /dev/null +++ b/utils/redis_init_script @@ -0,0 +1,50 @@ +#!/bin/sh +# +# Simple Redis init.d script conceived to work on Linux systems +# as it does use of the /proc filesystem. + +### BEGIN INIT INFO +# Provides: redis_6379 +# Default-Start: 2 3 4 5 +# Default-Stop: 0 1 6 +# Short-Description: Redis data structure server +# Description: Redis data structure server. See https://redis.io +### END INIT INFO + +REDISPORT=6379 +EXEC=/usr/local/bin/redis-server +CLIEXEC=/usr/local/bin/redis-cli + +PIDFILE=/var/run/redis_${REDISPORT}.pid +CONF="/etc/redis/${REDISPORT}.conf" + +case "$1" in + start) + if [ -f $PIDFILE ] + then + echo "$PIDFILE exists, process is already running or crashed" + else + echo "Starting Redis server..." + $EXEC $CONF + fi + ;; + stop) + if [ ! -f $PIDFILE ] + then + echo "$PIDFILE does not exist, process is not running" + else + PID=$(cat $PIDFILE) + echo "Stopping ..." + $CLIEXEC -p $REDISPORT shutdown + while [ -x /proc/${PID} ] + do + echo "Waiting for Redis to shutdown ..." + sleep 1 + done + echo "Redis stopped" + fi + ;; + *) + echo "Please use start or stop as first argument" + ;; +esac diff --git a/utils/redis_init_script.tpl b/utils/redis_init_script.tpl new file mode 100755 index 0000000..2e5b613 --- /dev/null +++ b/utils/redis_init_script.tpl @@ -0,0 +1,44 @@ + +case "$1" in + start) + if [ -f $PIDFILE ] + then + echo "$PIDFILE exists, process is already running or crashed" + else + echo "Starting Redis server..." + $EXEC $CONF + fi + ;; + stop) + if [ ! -f $PIDFILE ] + then + echo "$PIDFILE does not exist, process is not running" + else + PID=$(cat $PIDFILE) + echo "Stopping ..." + $CLIEXEC -p $REDISPORT shutdown + while [ -x /proc/${PID} ] + do + echo "Waiting for Redis to shutdown ..." + sleep 1 + done + echo "Redis stopped" + fi + ;; + status) + PID=$(cat $PIDFILE) + if [ ! -x /proc/${PID} ] + then + echo 'Redis is not running' + else + echo "Redis is running ($PID)" + fi + ;; + restart) + $0 stop + $0 start + ;; + *) + echo "Please use start, stop, restart or status as first argument" + ;; +esac diff --git a/utils/releasetools/01_create_tarball.sh b/utils/releasetools/01_create_tarball.sh new file mode 100755 index 0000000..366a61e --- /dev/null +++ b/utils/releasetools/01_create_tarball.sh @@ -0,0 +1,14 @@ +#!/bin/sh +if [ $# != "1" ] +then + echo "Usage: ./utils/releasetools/01_create_tarball.sh " + exit 1 +fi + +TAG=$1 +TARNAME="redis-${TAG}.tar" +echo "Generating /tmp/${TARNAME}" +git archive $TAG --prefix redis-${TAG}/ > /tmp/$TARNAME || exit 1 +echo "Gizipping the archive" +rm -f /tmp/$TARNAME.gz +gzip -9 /tmp/$TARNAME diff --git a/utils/releasetools/02_upload_tarball.sh b/utils/releasetools/02_upload_tarball.sh new file mode 100755 index 0000000..ef1e777 --- /dev/null +++ b/utils/releasetools/02_upload_tarball.sh @@ -0,0 +1,23 @@ +#!/bin/bash +if [ $# != "1" ] +then + echo "Usage: ./utils/releasetools/02_upload_tarball.sh " + exit 1 +fi + +echo "Uploading..." +scp /tmp/redis-${1}.tar.gz ubuntu@host.redis.io:/var/www/download/releases/ +echo "Updating web site... " +echo "Please check the github action tests for the release." +echo "Press any key if it is a stable release, or Ctrl+C to abort" +read x +ssh ubuntu@host.redis.io "cd /var/www/download; + rm -rf redis-${1}.tar.gz; + wget http://download.redis.io/releases/redis-${1}.tar.gz; + tar xvzf redis-${1}.tar.gz; + rm -rf redis-stable; + mv redis-${1} redis-stable; + tar cvzf redis-stable.tar.gz redis-stable; + rm -rf redis-${1}.tar.gz; + shasum -a 256 redis-stable.tar.gz > redis-stable.tar.gz.SHA256SUM; + " diff --git a/utils/releasetools/03_test_release.sh b/utils/releasetools/03_test_release.sh new file mode 100755 index 0000000..493d0b7 --- /dev/null +++ b/utils/releasetools/03_test_release.sh @@ -0,0 +1,28 @@ +#!/bin/sh +set -e +if [ $# != "1" ] +then + echo "Usage: ./utils/releasetools/03_test_release.sh " + exit 1 +fi + +TAG=$1 +TARNAME="redis-${TAG}.tar.gz" +DOWNLOADURL="http://download.redis.io/releases/${TARNAME}" + +echo "Doing sanity test on the actual tarball" + +cd /tmp +rm -rf test_release_tmp_dir +mkdir test_release_tmp_dir +cd test_release_tmp_dir +rm -f $TARNAME +rm -rf redis-${TAG} +wget $DOWNLOADURL +tar xvzf $TARNAME +cd redis-${TAG} +make +./runtest +./runtest-sentinel +./runtest-cluster +./runtest-moduleapi diff --git a/utils/releasetools/04_release_hash.sh b/utils/releasetools/04_release_hash.sh new file mode 100755 index 0000000..d932928 --- /dev/null +++ b/utils/releasetools/04_release_hash.sh @@ -0,0 +1,13 @@ +#!/bin/bash +if [ $# != "1" ] +then + echo "Usage: ./utils/releasetools/04_release_hash.sh " + exit 1 +fi + +SHA=$(curl -s http://download.redis.io/releases/redis-${1}.tar.gz | shasum -a 256 | cut -f 1 -d' ') +ENTRY="hash redis-${1}.tar.gz sha256 $SHA http://download.redis.io/releases/redis-${1}.tar.gz" +echo $ENTRY >> ../redis-hashes/README +echo "Press any key to commit, Ctrl-C to abort)." +read yes +(cd ../redis-hashes; git commit -a -m "${1} hash."; git push) diff --git a/utils/releasetools/changelog.tcl b/utils/releasetools/changelog.tcl new file mode 100755 index 0000000..2288794 --- /dev/null +++ b/utils/releasetools/changelog.tcl @@ -0,0 +1,35 @@ +#!/usr/bin/env tclsh + +if {[llength $::argv] != 2 && [llength $::argv] != 3} { + puts "Usage: $::argv0 \[\]" + exit 1 +} + +set branch [lindex $::argv 0] +set ver [lindex $::argv 1] +if {[llength $::argv] == 3} { + set count [lindex ::$argv 2] +} else { + set count 100 +} + +set template { +================================================================================ +Redis %ver% Released %date% +================================================================================ + +Upgrade urgency : +} + +set template [string trim $template] +append template "\n\n" +set date [clock format [clock seconds]] +set template [string map [list %ver% $ver %date% $date] $template] + +append template [exec git log $branch~$count..$branch "--format=format:%an in commit %h:%n %s" --shortstat] + +#Older, more verbose version. +# +#append template [exec git log $branch~30..$branch "--format=format:+-------------------------------------------------------------------------------%n| %s%n| By %an, %ai%n+--------------------------------------------------------------------------------%nhttps://github.com/redis/redis/commit/%H%n%n%b" --stat] + +puts $template diff --git a/utils/reply_schema_linter.js b/utils/reply_schema_linter.js new file mode 100644 index 0000000..e2358d4 --- /dev/null +++ b/utils/reply_schema_linter.js @@ -0,0 +1,31 @@ +function validate_schema(command_schema) { + var error_status = false + const Ajv = require("ajv/dist/2019") + const ajv = new Ajv({strict: true, strictTuples: false}) + let json = require('../src/commands/'+ command_schema); + for (var item in json) { + const schema = json[item].reply_schema + if (schema == undefined) + continue; + try { + ajv.compile(schema) + } catch (error) { + console.error(command_schema + " : " + error.toString()) + error_status = true + } + } + return error_status +} + +const schema_directory_path = './src/commands' +const path = require('path') +var fs = require('fs'); +var files = fs.readdirSync(schema_directory_path); +jsonFiles = files.filter(el => path.extname(el) === '.json') +var error_status = false +jsonFiles.forEach(function(file){ + if (validate_schema(file)) + error_status = true +}) +if (error_status) + process.exit(1) diff --git a/utils/req-res-log-validator.py b/utils/req-res-log-validator.py new file mode 100755 index 0000000..46c1100 --- /dev/null +++ b/utils/req-res-log-validator.py @@ -0,0 +1,350 @@ +#!/usr/bin/env python3 +import os +import glob +import json +import sys + +import jsonschema +import subprocess +import redis +import time +import argparse +import multiprocessing +import collections +import io +import traceback +from datetime import timedelta +from functools import partial +try: + from jsonschema import Draft201909Validator as schema_validator +except ImportError: + from jsonschema import Draft7Validator as schema_validator + +""" +The purpose of this file is to validate the reply_schema values of COMMAND DOCS. +Basically, this is what it does: +1. Goes over req-res files, generated by redis-servers, spawned by the testsuite (see logreqres.c) +2. For each request-response pair, it validates the response against the request's reply_schema (obtained from COMMAND DOCS) + +This script spins up a redis-server and a redis-cli in order to obtain COMMAND DOCS. + +In order to use this file you must run the redis testsuite with the following flags: +./runtest --dont-clean --force-resp3 --log-req-res + +And then: +./utils/req-res-log-validator.py + +The script will fail only if: +1. One or more of the replies doesn't comply with its schema. +2. One or more of the commands in COMMANDS DOCS doesn't have the reply_schema field (with --fail-missing-reply-schemas) +3. The testsuite didn't execute all of the commands (with --fail-commands-not-all-hit) + +Future validations: +1. Fail the script if one or more of the branches of the reply schema (e.g. oneOf, anyOf) was not hit. +""" + +IGNORED_COMMANDS = { + # Commands that don't work in a req-res manner (see logreqres.c) + "debug", # because of DEBUG SEGFAULT + "sync", + "psync", + "monitor", + "subscribe", + "unsubscribe", + "ssubscribe", + "sunsubscribe", + "psubscribe", + "punsubscribe", + # Commands to which we decided not write a reply schema + "pfdebug", + "lolwut", +} + +class Request(object): + """ + This class represents a Redis request (AKA command, argv) + """ + def __init__(self, f, docs, line_counter): + """ + Read lines from `f` (generated by logreqres.c) and populates the argv array + """ + self.command = None + self.schema = None + self.argv = [] + + while True: + line = f.readline() + line_counter[0] += 1 + if not line: + break + length = int(line) + arg = str(f.read(length)) + f.read(2) # read \r\n + line_counter[0] += 1 + if arg == "__argv_end__": + break + self.argv.append(arg) + + if not self.argv: + return + + self.command = self.argv[0].lower() + doc = docs.get(self.command, {}) + if not doc and len(self.argv) > 1: + self.command = f"{self.argv[0].lower()}|{self.argv[1].lower()}" + doc = docs.get(self.command, {}) + + if not doc: + self.command = None + return + + self.schema = doc.get("reply_schema") + + def __str__(self): + return json.dumps(self.argv) + + +class Response(object): + """ + This class represents a Redis response in RESP3 + """ + def __init__(self, f, line_counter): + """ + Read lines from `f` (generated by logreqres.c) and build the JSON representing the response in RESP3 + """ + self.error = False + self.queued = False + self.json = None + + line = f.readline()[:-2] + line_counter[0] += 1 + if line[0] == '+': + self.json = line[1:] + if self.json == "QUEUED": + self.queued = True + elif line[0] == '-': + self.json = line[1:] + self.error = True + elif line[0] == '$': + self.json = str(f.read(int(line[1:]))) + f.read(2) # read \r\n + line_counter[0] += 1 + elif line[0] == ':': + self.json = int(line[1:]) + elif line[0] == ',': + self.json = float(line[1:]) + elif line[0] == '_': + self.json = None + elif line[0] == '#': + self.json = line[1] == 't' + elif line[0] == '!': + self.json = str(f.read(int(line[1:]))) + f.read(2) # read \r\n + line_counter[0] += 1 + self.error = True + elif line[0] == '=': + self.json = str(f.read(int(line[1:])))[4:] # skip "txt:" or "mkd:" + f.read(2) # read \r\n + line_counter[0] += 1 + self.json.count("\r\n") + elif line[0] == '(': + self.json = line[1:] # big-number is actually a string + elif line[0] in ['*', '~', '>']: # unfortunately JSON doesn't tell the difference between a list and a set + self.json = [] + count = int(line[1:]) + for i in range(count): + ele = Response(f, line_counter) + self.json.append(ele.json) + elif line[0] in ['%', '|']: + self.json = {} + count = int(line[1:]) + for i in range(count): + field = Response(f, line_counter) + # Redis allows fields to be non-strings but JSON doesn't. + # Luckily, for any kind of response we can validate, the fields are + # always strings (example: XINFO STREAM) + # The reason we can't always convert to string is because of DEBUG PROTOCOL MAP + # which anyway doesn't have a schema + if isinstance(field.json, str): + field = field.json + value = Response(f, line_counter) + self.json[field] = value.json + if line[0] == '|': + # We don't care about the attributes, read the real response + real_res = Response(f, line_counter) + self.__dict__.update(real_res.__dict__) + + + def __str__(self): + return json.dumps(self.json) + + +def process_file(docs, path): + """ + This function processes a single file generated by logreqres.c + """ + line_counter = [0] # A list with one integer: to force python to pass it by reference + command_counter = dict() + + print(f"Processing {path} ...") + + # Convert file to StringIO in order to minimize IO operations + with open(path, "r", newline="\r\n", encoding="latin-1") as f: + content = f.read() + + with io.StringIO(content) as fakefile: + while True: + try: + req = Request(fakefile, docs, line_counter) + if not req.argv: + # EOF + break + res = Response(fakefile, line_counter) + except json.decoder.JSONDecodeError as err: + print(f"JSON decoder error while processing {path}:{line_counter[0]}: {err}") + print(traceback.format_exc()) + raise + except Exception as err: + print(f"General error while processing {path}:{line_counter[0]}: {err}") + print(traceback.format_exc()) + raise + + if not req.command: + # Unknown command + continue + + command_counter[req.command] = command_counter.get(req.command, 0) + 1 + + if res.error or res.queued: + continue + + if req.command in IGNORED_COMMANDS: + continue + + try: + jsonschema.validate(instance=res.json, schema=req.schema, cls=schema_validator) + except (jsonschema.ValidationError, jsonschema.exceptions.SchemaError) as err: + print(f"JSON schema validation error on {path}: {err}") + print(f"argv: {req.argv}") + try: + print(f"Response: {res}") + except UnicodeDecodeError as err: + print("Response: (unprintable)") + print(f"Schema: {json.dumps(req.schema, indent=2)}") + print(traceback.format_exc()) + raise + + return command_counter + + +def fetch_schemas(cli, port, args, docs): + redis_proc = subprocess.Popen(args, stdout=subprocess.PIPE) + + while True: + try: + print('Connecting to Redis...') + r = redis.Redis(port=port) + r.ping() + break + except Exception as e: + time.sleep(0.1) + + print('Connected') + + cli_proc = subprocess.Popen([cli, '-p', str(port), '--json', 'command', 'docs'], stdout=subprocess.PIPE) + stdout, stderr = cli_proc.communicate() + docs_response = json.loads(stdout) + + for name, doc in docs_response.items(): + if "subcommands" in doc: + for subname, subdoc in doc["subcommands"].items(): + docs[subname] = subdoc + else: + docs[name] = doc + + redis_proc.terminate() + redis_proc.wait() + + +if __name__ == '__main__': + # Figure out where the sources are + srcdir = os.path.abspath(os.path.dirname(os.path.abspath(__file__)) + "/../src") + testdir = os.path.abspath(os.path.dirname(os.path.abspath(__file__)) + "/../tests") + + parser = argparse.ArgumentParser() + parser.add_argument('--server', type=str, default='%s/redis-server' % srcdir) + parser.add_argument('--port', type=int, default=6534) + parser.add_argument('--cli', type=str, default='%s/redis-cli' % srcdir) + parser.add_argument('--module', type=str, action='append', default=[]) + parser.add_argument('--verbose', action='store_true') + parser.add_argument('--fail-commands-not-all-hit', action='store_true') + parser.add_argument('--fail-missing-reply-schemas', action='store_true') + args = parser.parse_args() + + docs = dict() + + # Fetch schemas from a Redis instance + print('Starting Redis server') + redis_args = [args.server, '--port', str(args.port)] + for module in args.module: + redis_args += ['--loadmodule', 'tests/modules/%s.so' % module] + + fetch_schemas(args.cli, args.port, redis_args, docs) + + # Fetch schemas from a sentinel + print('Starting Redis sentinel') + + # Sentinel needs a config file to start + config_file = "tmpsentinel.conf" + open(config_file, 'a').close() + + sentinel_args = [args.server, config_file, '--port', str(args.port), "--sentinel"] + fetch_schemas(args.cli, args.port, sentinel_args, docs) + os.unlink(config_file) + + missing_schema = [k for k, v in docs.items() + if "reply_schema" not in v and k not in IGNORED_COMMANDS] + if missing_schema: + print("WARNING! The following commands are missing a reply_schema:") + for k in sorted(missing_schema): + print(f" {k}") + if args.fail_missing_reply_schemas: + print("ERROR! at least one command does not have a reply_schema") + sys.exit(1) + + start = time.time() + + # Obtain all the files to processes + paths = [] + for path in glob.glob('%s/tmp/*/*.reqres' % testdir): + paths.append(path) + + for path in glob.glob('%s/cluster/tmp/*/*.reqres' % testdir): + paths.append(path) + + for path in glob.glob('%s/sentinel/tmp/*/*.reqres' % testdir): + paths.append(path) + + counter = collections.Counter() + # Spin several processes to handle the files in parallel + with multiprocessing.Pool(multiprocessing.cpu_count()) as pool: + func = partial(process_file, docs) + # pool.map blocks until all the files have been processed + for result in pool.map(func, paths): + counter.update(result) + command_counter = dict(counter) + + elapsed = time.time() - start + print(f"Done. ({timedelta(seconds=elapsed)})") + print("Hits per command:") + for k, v in sorted(command_counter.items()): + print(f" {k}: {v}") + not_hit = set(set(docs.keys()) - set(command_counter.keys()) - set(IGNORED_COMMANDS)) + if not_hit: + if args.verbose: + print("WARNING! The following commands were not hit at all:") + for k in sorted(not_hit): + print(f" {k}") + if args.fail_commands_not_all_hit: + print("ERROR! at least one command was not hit by the tests") + sys.exit(1) + diff --git a/utils/req-res-validator/requirements.txt b/utils/req-res-validator/requirements.txt new file mode 100644 index 0000000..0e3024b --- /dev/null +++ b/utils/req-res-validator/requirements.txt @@ -0,0 +1,2 @@ +jsonschema==4.17.3 +redis==4.5.1 \ No newline at end of file diff --git a/utils/speed-regression.tcl b/utils/speed-regression.tcl new file mode 100755 index 0000000..bf35c7d --- /dev/null +++ b/utils/speed-regression.tcl @@ -0,0 +1,130 @@ +#!/usr/bin/env tclsh8.5 +# Copyright (C) 2011 Salvatore Sanfilippo +# Released under the BSD license like Redis itself + +source ../tests/support/redis.tcl +set ::port 12123 +set ::tests {PING,SET,GET,INCR,LPUSH,LPOP,SADD,SPOP,LRANGE_100,LRANGE_600,MSET} +set ::datasize 16 +set ::requests 100000 + +proc run-tests branches { + set runs {} + set branch_id 0 + foreach b $branches { + cd ../src + puts "Benchmarking $b" + exec -ignorestderr git checkout $b 2> /dev/null + exec -ignorestderr make clean 2> /dev/null + puts " compiling..." + exec -ignorestderr make 2> /dev/null + + if {$branch_id == 0} { + puts " copy redis-benchmark from unstable to /tmp..." + exec -ignorestderr cp ./redis-benchmark /tmp + incr branch_id + continue + } + + # Start the Redis server + puts " starting the server... [exec ./redis-server -v]" + set pids [exec echo "port $::port\nloglevel warning\n" | ./redis-server - > /dev/null 2> /dev/null &] + puts " pids: $pids" + after 1000 + puts " running the benchmark" + + set r [redis 127.0.0.1 $::port] + set i [$r info] + puts " redis INFO shows version: [lindex [split $i] 0]" + $r close + + set output [exec /tmp/redis-benchmark -n $::requests -t $::tests -d $::datasize --csv -p $::port] + lappend runs $b $output + puts " killing server..." + catch {exec kill -9 [lindex $pids 0]} + catch {exec kill -9 [lindex $pids 1]} + incr branch_id + } + return $runs +} + +proc get-result-with-name {output name} { + foreach line [split $output "\n"] { + lassign [split $line ","] key value + set key [string tolower [string range $key 1 end-1]] + set value [string range $value 1 end-1] + if {$key eq [string tolower $name]} { + return $value + } + } + return "n/a" +} + +proc get-test-names output { + set names {} + foreach line [split $output "\n"] { + lassign [split $line ","] key value + set key [string tolower [string range $key 1 end-1]] + lappend names $key + } + return $names +} + +proc combine-results {results} { + set tests [get-test-names [lindex $results 1]] + foreach test $tests { + puts $test + foreach {branch output} $results { + puts [format "%-20s %s" \ + $branch [get-result-with-name $output $test]] + } + puts {} + } +} + +proc main {} { + # Note: the first branch is only used in order to get the redis-benchmark + # executable. Tests are performed starting from the second branch. + set branches { + slowset 2.2.0 2.4.0 unstable slowset + } + set results [run-tests $branches] + puts "\n" + puts "# Test results: datasize=$::datasize requests=$::requests" + puts [combine-results $results] +} + +# Force the user to run the script from the 'utils' directory. +if {![file exists speed-regression.tcl]} { + puts "Please make sure to run speed-regression.tcl while inside /utils." + puts "Example: cd utils; ./speed-regression.tcl" + exit 1 +} + +# Make sure there is not already a server running on port 12123 +set is_not_running [catch {set r [redis 127.0.0.1 $::port]}] +if {!$is_not_running} { + puts "Sorry, you have a running server on port $::port" + exit 1 +} + +# parse arguments +for {set j 0} {$j < [llength $argv]} {incr j} { + set opt [lindex $argv $j] + set arg [lindex $argv [expr $j+1]] + if {$opt eq {--tests}} { + set ::tests $arg + incr j + } elseif {$opt eq {--datasize}} { + set ::datasize $arg + incr j + } elseif {$opt eq {--requests}} { + set ::requests $arg + incr j + } else { + puts "Wrong argument: $opt" + exit 1 + } +} + +main diff --git a/utils/srandmember/README.md b/utils/srandmember/README.md new file mode 100644 index 0000000..d3da1e8 --- /dev/null +++ b/utils/srandmember/README.md @@ -0,0 +1,14 @@ +The utilities in this directory plot the distribution of SRANDMEMBER to +evaluate how fair it is. + +See http://theshfl.com/redis_sets for more information on the topic that lead +to such investigation fix. + +showdist.rb -- shows the distribution of the frequency elements are returned. + The x axis is the number of times elements were returned, and + the y axis is how many elements were returned with such + frequency. + +showfreq.rb -- shows the frequency each element was returned. + The x axis is the element number. + The y axis is the times it was returned. diff --git a/utils/srandmember/showdist.rb b/utils/srandmember/showdist.rb new file mode 100644 index 0000000..2435857 --- /dev/null +++ b/utils/srandmember/showdist.rb @@ -0,0 +1,33 @@ +require 'redis' + +r = Redis.new +r.select(9) +r.del("myset"); +r.sadd("myset",(0..999).to_a) +freq = {} +100.times { + res = r.pipelined { + 1000.times { + r.srandmember("myset") + } + } + res.each{|ele| + freq[ele] = 0 if freq[ele] == nil + freq[ele] += 1 + } +} + +# Convert into frequency distribution +dist = {} +freq.each{|item,count| + dist[count] = 0 if dist[count] == nil + dist[count] += 1 +} + +min = dist.keys.min +max = dist.keys.max +(min..max).each{|x| + count = dist[x] + count = 0 if count == nil + puts "#{x} -> #{"*"*count}" +} diff --git a/utils/srandmember/showfreq.rb b/utils/srandmember/showfreq.rb new file mode 100644 index 0000000..625519c --- /dev/null +++ b/utils/srandmember/showfreq.rb @@ -0,0 +1,23 @@ +require 'redis' + +r = Redis.new +r.select(9) +r.del("myset"); +r.sadd("myset",(0..999).to_a) +freq = {} +500.times { + res = r.pipelined { + 1000.times { + r.srandmember("myset") + } + } + res.each{|ele| + freq[ele] = 0 if freq[ele] == nil + freq[ele] += 1 + } +} + +# Print the frequency each element was yield to process it with gnuplot +freq.each{|item,count| + puts "#{item} #{count}" +} diff --git a/utils/systemd-redis_multiple_servers@.service b/utils/systemd-redis_multiple_servers@.service new file mode 100644 index 0000000..108ccfc --- /dev/null +++ b/utils/systemd-redis_multiple_servers@.service @@ -0,0 +1,37 @@ +# example systemd template service unit file for multiple redis-servers +# +# You can use this file as a blueprint for your actual template service unit +# file, if you intend to run multiple independent redis-server instances in +# parallel using systemd's "template unit files" feature. If you do, you will +# want to choose a better basename for your service unit by renaming this file +# when copying it. +# +# Please take a look at the provided "systemd-redis_server.service" example +# service unit file, too, if you choose to use this approach at managing +# multiple redis-server instances via systemd. + +[Unit] +Description=Redis data structure server - instance %i +Documentation=https://redis.io/documentation +# This template unit assumes your redis-server configuration file(s) +# to live at /etc/redis/redis_server_.conf +AssertPathExists=/etc/redis/redis_server_%i.conf +#Before=your_application.service another_example_application.service +#AssertPathExists=/var/lib/redis + +[Service] +ExecStart=/usr/local/bin/redis-server /etc/redis/redis_server_%i.conf +LimitNOFILE=10032 +NoNewPrivileges=yes +#OOMScoreAdjust=-900 +#PrivateTmp=yes +Type=notify +TimeoutStartSec=infinity +TimeoutStopSec=infinity +UMask=0077 +#User=redis +#Group=redis +#WorkingDirectory=/var/lib/redis + +[Install] +WantedBy=multi-user.target diff --git a/utils/systemd-redis_server.service b/utils/systemd-redis_server.service new file mode 100644 index 0000000..cf15864 --- /dev/null +++ b/utils/systemd-redis_server.service @@ -0,0 +1,43 @@ +# example systemd service unit file for redis-server +# +# In order to use this as a template for providing a redis service in your +# environment, _at the very least_ make sure to adapt the redis configuration +# file you intend to use as needed (make sure to set "supervised systemd"), and +# to set sane TimeoutStartSec and TimeoutStopSec property values in the unit's +# "[Service]" section to fit your needs. +# +# Some properties, such as User= and Group=, are highly desirable for virtually +# all deployments of redis, but cannot be provided in a manner that fits all +# expectable environments. Some of these properties have been commented out in +# this example service unit file, but you are highly encouraged to set them to +# fit your needs. +# +# Please refer to systemd.unit(5), systemd.service(5), and systemd.exec(5) for +# more information. + +[Unit] +Description=Redis data structure server +Documentation=https://redis.io/documentation +#Before=your_application.service another_example_application.service +#AssertPathExists=/var/lib/redis +Wants=network-online.target +After=network-online.target + +[Service] +ExecStart=/usr/local/bin/redis-server --supervised systemd --daemonize no +## Alternatively, have redis-server load a configuration file: +#ExecStart=/usr/local/bin/redis-server /path/to/your/redis.conf +LimitNOFILE=10032 +NoNewPrivileges=yes +#OOMScoreAdjust=-900 +#PrivateTmp=yes +Type=notify +TimeoutStartSec=infinity +TimeoutStopSec=infinity +UMask=0077 +#User=redis +#Group=redis +#WorkingDirectory=/var/lib/redis + +[Install] +WantedBy=multi-user.target diff --git a/utils/tracking_collisions.c b/utils/tracking_collisions.c new file mode 100644 index 0000000..f521111 --- /dev/null +++ b/utils/tracking_collisions.c @@ -0,0 +1,76 @@ +/* This is a small program used in order to understand the collision rate + * of CRC64 (ISO version) VS other stronger hashing functions in the context + * of hashing keys for the Redis "tracking" feature (client side caching + * assisted by the server). + * + * The program attempts to hash keys with common names in the form of + * + * prefix: + * + * And counts the resulting collisions generated in the 24 bits of output + * needed for the tracking feature invalidation table (16 millions + entries) + * + * Compile with: + * + * cc -O2 ./tracking_collisions.c ../src/crc64.c ../src/sha1.c + * ./a.out + * + * -------------------------------------------------------------------------- + * + * Copyright (C) 2019 Salvatore Sanfilippo + * This code is released under the BSD 2 clause license. + */ + +#include +#include +#include +#include +#include "../src/crc64.h" +#include "../src/sha1.h" + +#define TABLE_SIZE (1<<24) +int Table[TABLE_SIZE]; + +uint64_t crc64Hash(char *key, size_t len) { + return crc64(0,(unsigned char*)key,len); +} + +uint64_t sha1Hash(char *key, size_t len) { + SHA1_CTX ctx; + unsigned char hash[20]; + + SHA1Init(&ctx); + SHA1Update(&ctx,(unsigned char*)key,len); + SHA1Final(hash,&ctx); + uint64_t hash64; + memcpy(&hash64,hash,sizeof(hash64)); + return hash64; +} + +/* Test the hashing function provided as callback and return the + * number of collisions found. */ +unsigned long testHashingFunction(uint64_t (*hash)(char *, size_t)) { + unsigned long collisions = 0; + memset(Table,0,sizeof(Table)); + char *prefixes[] = {"object", "message", "user", NULL}; + for (int i = 0; prefixes[i] != NULL; i++) { + for (int j = 0; j < TABLE_SIZE/2; j++) { + char keyname[128]; + size_t keylen = snprintf(keyname,sizeof(keyname),"%s:%d", + prefixes[i],j); + uint64_t bucket = hash(keyname,keylen) % TABLE_SIZE; + if (Table[bucket]) { + collisions++; + } else { + Table[bucket] = 1; + } + } + } + return collisions; +} + +int main(void) { + printf("SHA1 : %lu\n", testHashingFunction(sha1Hash)); + printf("CRC64: %lu\n", testHashingFunction(crc64Hash)); + return 0; +} diff --git a/utils/whatisdoing.sh b/utils/whatisdoing.sh new file mode 100755 index 0000000..68d7f7c --- /dev/null +++ b/utils/whatisdoing.sh @@ -0,0 +1,24 @@ +# This script is from http://poormansprofiler.org/ +# +# NOTE: Instead of using this script, you should use the Redis +# Software Watchdog, which provides a similar functionality but in +# a more reliable / easy to use way. +# +# Check https://redis.io/topics/latency for more information. + +#!/bin/bash +nsamples=1 +sleeptime=0 +pid=$(ps auxww | grep '[r]edis-server' | awk '{print $2}') + +for x in $(seq 1 $nsamples) + do + gdb -ex "set pagination 0" -ex "thread apply all bt" -batch -p $pid + sleep $sleeptime + done | \ +awk ' + BEGIN { s = ""; } + /Thread/ { print s; s = ""; } + /^\#/ { if (s != "" ) { s = s "," $4} else { s = $4 } } + END { print s }' | \ +sort | uniq -c | sort -r -n -k 1,1 -- cgit v1.2.3