From 317c0644ccf108aa23ef3fd8358bd66c2840bfc0 Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Sun, 14 Apr 2024 15:40:54 +0200 Subject: Adding upstream version 5:7.2.4. Signed-off-by: Daniel Baumann --- src/eval.c | 1667 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 1667 insertions(+) create mode 100644 src/eval.c (limited to 'src/eval.c') diff --git a/src/eval.c b/src/eval.c new file mode 100644 index 0000000..eb4b529 --- /dev/null +++ b/src/eval.c @@ -0,0 +1,1667 @@ +/* + * Copyright (c) 2009-2012, Salvatore Sanfilippo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include "server.h" +#include "sha1.h" +#include "rand.h" +#include "cluster.h" +#include "monotonic.h" +#include "resp_parser.h" +#include "script_lua.h" + +#include +#include +#include +#include +#include + +void ldbInit(void); +void ldbDisable(client *c); +void ldbEnable(client *c); +void evalGenericCommandWithDebugging(client *c, int evalsha); +sds ldbCatStackValue(sds s, lua_State *lua, int idx); + +static void dictLuaScriptDestructor(dict *d, void *val) { + UNUSED(d); + if (val == NULL) return; /* Lazy freeing will set value to NULL. */ + decrRefCount(((luaScript*)val)->body); + zfree(val); +} + +static uint64_t dictStrCaseHash(const void *key) { + return dictGenCaseHashFunction((unsigned char*)key, strlen((char*)key)); +} + +/* server.lua_scripts sha (as sds string) -> scripts (as luaScript) cache. */ +dictType shaScriptObjectDictType = { + dictStrCaseHash, /* hash function */ + NULL, /* key dup */ + NULL, /* val dup */ + dictSdsKeyCaseCompare, /* key compare */ + dictSdsDestructor, /* key destructor */ + dictLuaScriptDestructor, /* val destructor */ + NULL /* allow to expand */ +}; + +/* Lua context */ +struct luaCtx { + lua_State *lua; /* The Lua interpreter. We use just one for all clients */ + client *lua_client; /* The "fake client" to query Redis from Lua */ + dict *lua_scripts; /* A dictionary of SHA1 -> Lua scripts */ + unsigned long long lua_scripts_mem; /* Cached scripts' memory + oh */ +} lctx; + +/* Debugger shared state is stored inside this global structure. */ +#define LDB_BREAKPOINTS_MAX 64 /* Max number of breakpoints. */ +#define LDB_MAX_LEN_DEFAULT 256 /* Default len limit for replies / var dumps. */ +struct ldbState { + connection *conn; /* Connection of the debugging client. */ + int active; /* Are we debugging EVAL right now? */ + int forked; /* Is this a fork()ed debugging session? */ + list *logs; /* List of messages to send to the client. */ + list *traces; /* Messages about Redis commands executed since last stop.*/ + list *children; /* All forked debugging sessions pids. */ + int bp[LDB_BREAKPOINTS_MAX]; /* An array of breakpoints line numbers. */ + int bpcount; /* Number of valid entries inside bp. */ + int step; /* Stop at next line regardless of breakpoints. */ + int luabp; /* Stop at next line because redis.breakpoint() was called. */ + sds *src; /* Lua script source code split by line. */ + int lines; /* Number of lines in 'src'. */ + int currentline; /* Current line number. */ + sds cbuf; /* Debugger client command buffer. */ + size_t maxlen; /* Max var dump / reply length. */ + int maxlen_hint_sent; /* Did we already hint about "set maxlen"? */ +} ldb; + +/* --------------------------------------------------------------------------- + * Utility functions. + * ------------------------------------------------------------------------- */ + +/* Perform the SHA1 of the input string. We use this both for hashing script + * bodies in order to obtain the Lua function name, and in the implementation + * of redis.sha1(). + * + * 'digest' should point to a 41 bytes buffer: 40 for SHA1 converted into an + * hexadecimal number, plus 1 byte for null term. */ +void sha1hex(char *digest, char *script, size_t len) { + SHA1_CTX ctx; + unsigned char hash[20]; + char *cset = "0123456789abcdef"; + int j; + + SHA1Init(&ctx); + SHA1Update(&ctx,(unsigned char*)script,len); + SHA1Final(hash,&ctx); + + for (j = 0; j < 20; j++) { + digest[j*2] = cset[((hash[j]&0xF0)>>4)]; + digest[j*2+1] = cset[(hash[j]&0xF)]; + } + digest[40] = '\0'; +} + +/* redis.breakpoint() + * + * Allows to stop execution during a debugging session from within + * the Lua code implementation, like if a breakpoint was set in the code + * immediately after the function. */ +int luaRedisBreakpointCommand(lua_State *lua) { + if (ldb.active) { + ldb.luabp = 1; + lua_pushboolean(lua,1); + } else { + lua_pushboolean(lua,0); + } + return 1; +} + +/* redis.debug() + * + * Log a string message into the output console. + * Can take multiple arguments that will be separated by commas. + * Nothing is returned to the caller. */ +int luaRedisDebugCommand(lua_State *lua) { + if (!ldb.active) return 0; + int argc = lua_gettop(lua); + sds log = sdscatprintf(sdsempty()," line %d: ", ldb.currentline); + while(argc--) { + log = ldbCatStackValue(log,lua,-1 - argc); + if (argc != 0) log = sdscatlen(log,", ",2); + } + ldbLog(log); + return 0; +} + +/* redis.replicate_commands() + * + * DEPRECATED: Now do nothing and always return true. + * Turn on single commands replication if the script never called + * a write command so far, and returns true. Otherwise if the script + * already started to write, returns false and stick to whole scripts + * replication, which is our default. */ +int luaRedisReplicateCommandsCommand(lua_State *lua) { + lua_pushboolean(lua,1); + return 1; +} + +/* Initialize the scripting environment. + * + * This function is called the first time at server startup with + * the 'setup' argument set to 1. + * + * It can be called again multiple times during the lifetime of the Redis + * process, with 'setup' set to 0, and following a scriptingRelease() call, + * in order to reset the Lua scripting environment. + * + * However it is simpler to just call scriptingReset() that does just that. */ +void scriptingInit(int setup) { + lua_State *lua = lua_open(); + + if (setup) { + lctx.lua_client = NULL; + server.script_disable_deny_script = 0; + ldbInit(); + } + + /* Initialize a dictionary we use to map SHAs to scripts. + * This is useful for replication, as we need to replicate EVALSHA + * as EVAL, so we need to remember the associated script. */ + lctx.lua_scripts = dictCreate(&shaScriptObjectDictType); + lctx.lua_scripts_mem = 0; + + luaRegisterRedisAPI(lua); + + /* register debug commands */ + lua_getglobal(lua,"redis"); + + /* redis.breakpoint */ + lua_pushstring(lua,"breakpoint"); + lua_pushcfunction(lua,luaRedisBreakpointCommand); + lua_settable(lua,-3); + + /* redis.debug */ + lua_pushstring(lua,"debug"); + lua_pushcfunction(lua,luaRedisDebugCommand); + lua_settable(lua,-3); + + /* redis.replicate_commands */ + lua_pushstring(lua, "replicate_commands"); + lua_pushcfunction(lua, luaRedisReplicateCommandsCommand); + lua_settable(lua, -3); + + lua_setglobal(lua,"redis"); + + /* Add a helper function we use for pcall error reporting. + * Note that when the error is in the C function we want to report the + * information about the caller, that's what makes sense from the point + * of view of the user debugging a script. */ + { + char *errh_func = "local dbg = debug\n" + "debug = nil\n" + "function __redis__err__handler(err)\n" + " local i = dbg.getinfo(2,'nSl')\n" + " if i and i.what == 'C' then\n" + " i = dbg.getinfo(3,'nSl')\n" + " end\n" + " if type(err) ~= 'table' then\n" + " err = {err='ERR ' .. tostring(err)}" + " end" + " if i then\n" + " err['source'] = i.source\n" + " err['line'] = i.currentline\n" + " end" + " return err\n" + "end\n"; + luaL_loadbuffer(lua,errh_func,strlen(errh_func),"@err_handler_def"); + lua_pcall(lua,0,0,0); + } + + /* Create the (non connected) client that we use to execute Redis commands + * inside the Lua interpreter. + * Note: there is no need to create it again when this function is called + * by scriptingReset(). */ + if (lctx.lua_client == NULL) { + lctx.lua_client = createClient(NULL); + lctx.lua_client->flags |= CLIENT_SCRIPT; + + /* We do not want to allow blocking commands inside Lua */ + lctx.lua_client->flags |= CLIENT_DENY_BLOCKING; + } + + /* Lock the global table from any changes */ + lua_pushvalue(lua, LUA_GLOBALSINDEX); + luaSetErrorMetatable(lua); + /* Recursively lock all tables that can be reached from the global table */ + luaSetTableProtectionRecursively(lua); + lua_pop(lua, 1); + + lctx.lua = lua; +} + +/* Release resources related to Lua scripting. + * This function is used in order to reset the scripting environment. */ +void scriptingRelease(int async) { + if (async) + freeLuaScriptsAsync(lctx.lua_scripts); + else + dictRelease(lctx.lua_scripts); + lctx.lua_scripts_mem = 0; + lua_close(lctx.lua); +} + +void scriptingReset(int async) { + scriptingRelease(async); + scriptingInit(0); +} + +/* --------------------------------------------------------------------------- + * EVAL and SCRIPT commands implementation + * ------------------------------------------------------------------------- */ + +static void evalCalcFunctionName(int evalsha, sds script, char *out_funcname) { + /* We obtain the script SHA1, then check if this function is already + * defined into the Lua state */ + out_funcname[0] = 'f'; + out_funcname[1] = '_'; + if (!evalsha) { + /* Hash the code if this is an EVAL call */ + sha1hex(out_funcname+2,script,sdslen(script)); + } else { + /* We already have the SHA if it is an EVALSHA */ + int j; + char *sha = script; + + /* Convert to lowercase. We don't use tolower since the function + * managed to always show up in the profiler output consuming + * a non trivial amount of time. */ + for (j = 0; j < 40; j++) + out_funcname[j+2] = (sha[j] >= 'A' && sha[j] <= 'Z') ? + sha[j]+('a'-'A') : sha[j]; + out_funcname[42] = '\0'; + } +} + +/* Helper function to try and extract shebang flags from the script body. + * If no shebang is found, return with success and COMPAT mode flag. + * The err arg is optional, can be used to get a detailed error string. + * The out_shebang_len arg is optional, can be used to trim the shebang from the script. + * Returns C_OK on success, and C_ERR on error. */ +int evalExtractShebangFlags(sds body, uint64_t *out_flags, ssize_t *out_shebang_len, sds *err) { + ssize_t shebang_len = 0; + uint64_t script_flags = SCRIPT_FLAG_EVAL_COMPAT_MODE; + if (!strncmp(body, "#!", 2)) { + int numparts,j; + char *shebang_end = strchr(body, '\n'); + if (shebang_end == NULL) { + if (err) + *err = sdsnew("Invalid script shebang"); + return C_ERR; + } + shebang_len = shebang_end - body; + sds shebang = sdsnewlen(body, shebang_len); + sds *parts = sdssplitargs(shebang, &numparts); + sdsfree(shebang); + if (!parts || numparts == 0) { + if (err) + *err = sdsnew("Invalid engine in script shebang"); + sdsfreesplitres(parts, numparts); + return C_ERR; + } + /* Verify lua interpreter was specified */ + if (strcmp(parts[0], "#!lua")) { + if (err) + *err = sdscatfmt(sdsempty(), "Unexpected engine in script shebang: %s", parts[0]); + sdsfreesplitres(parts, numparts); + return C_ERR; + } + script_flags &= ~SCRIPT_FLAG_EVAL_COMPAT_MODE; + for (j = 1; j < numparts; j++) { + if (!strncmp(parts[j], "flags=", 6)) { + sdsrange(parts[j], 6, -1); + int numflags, jj; + sds *flags = sdssplitlen(parts[j], sdslen(parts[j]), ",", 1, &numflags); + for (jj = 0; jj < numflags; jj++) { + scriptFlag *sf; + for (sf = scripts_flags_def; sf->flag; sf++) { + if (!strcmp(flags[jj], sf->str)) break; + } + if (!sf->flag) { + if (err) + *err = sdscatfmt(sdsempty(), "Unexpected flag in script shebang: %s", flags[jj]); + sdsfreesplitres(flags, numflags); + sdsfreesplitres(parts, numparts); + return C_ERR; + } + script_flags |= sf->flag; + } + sdsfreesplitres(flags, numflags); + } else { + /* We only support function flags options for lua scripts */ + if (err) + *err = sdscatfmt(sdsempty(), "Unknown lua shebang option: %s", parts[j]); + sdsfreesplitres(parts, numparts); + return C_ERR; + } + } + sdsfreesplitres(parts, numparts); + } + if (out_shebang_len) + *out_shebang_len = shebang_len; + *out_flags = script_flags; + return C_OK; +} + +/* Try to extract command flags if we can, returns the modified flags. + * Note that it does not guarantee the command arguments are right. */ +uint64_t evalGetCommandFlags(client *c, uint64_t cmd_flags) { + char funcname[43]; + int evalsha = c->cmd->proc == evalShaCommand || c->cmd->proc == evalShaRoCommand; + if (evalsha && sdslen(c->argv[1]->ptr) != 40) + return cmd_flags; + uint64_t script_flags; + evalCalcFunctionName(evalsha, c->argv[1]->ptr, funcname); + char *lua_cur_script = funcname + 2; + c->cur_script = dictFind(lctx.lua_scripts, lua_cur_script); + if (!c->cur_script) { + if (evalsha) + return cmd_flags; + if (evalExtractShebangFlags(c->argv[1]->ptr, &script_flags, NULL, NULL) == C_ERR) + return cmd_flags; + } else { + luaScript *l = dictGetVal(c->cur_script); + script_flags = l->flags; + } + if (script_flags & SCRIPT_FLAG_EVAL_COMPAT_MODE) + return cmd_flags; + return scriptFlagsToCmdFlags(cmd_flags, script_flags); +} + +/* Define a Lua function with the specified body. + * The function name will be generated in the following form: + * + * f_ + * + * The function increments the reference count of the 'body' object as a + * side effect of a successful call. + * + * On success a pointer to an SDS string representing the function SHA1 of the + * just added function is returned (and will be valid until the next call + * to scriptingReset() function), otherwise NULL is returned. + * + * The function handles the fact of being called with a script that already + * exists, and in such a case, it behaves like in the success case. + * + * If 'c' is not NULL, on error the client is informed with an appropriate + * error describing the nature of the problem and the Lua interpreter error. */ +sds luaCreateFunction(client *c, robj *body) { + char funcname[43]; + dictEntry *de; + uint64_t script_flags; + + funcname[0] = 'f'; + funcname[1] = '_'; + sha1hex(funcname+2,body->ptr,sdslen(body->ptr)); + + if ((de = dictFind(lctx.lua_scripts,funcname+2)) != NULL) { + return dictGetKey(de); + } + + /* Handle shebang header in script code */ + ssize_t shebang_len = 0; + sds err = NULL; + if (evalExtractShebangFlags(body->ptr, &script_flags, &shebang_len, &err) == C_ERR) { + addReplyErrorSds(c, err); + return NULL; + } + + /* Note that in case of a shebang line we skip it but keep the line feed to conserve the user's line numbers */ + if (luaL_loadbuffer(lctx.lua,(char*)body->ptr + shebang_len,sdslen(body->ptr) - shebang_len,"@user_script")) { + if (c != NULL) { + addReplyErrorFormat(c, + "Error compiling script (new function): %s", + lua_tostring(lctx.lua,-1)); + } + lua_pop(lctx.lua,1); + return NULL; + } + + serverAssert(lua_isfunction(lctx.lua, -1)); + + lua_setfield(lctx.lua, LUA_REGISTRYINDEX, funcname); + + /* We also save a SHA1 -> Original script map in a dictionary + * so that we can replicate / write in the AOF all the + * EVALSHA commands as EVAL using the original script. */ + luaScript *l = zcalloc(sizeof(luaScript)); + l->body = body; + l->flags = script_flags; + sds sha = sdsnewlen(funcname+2,40); + int retval = dictAdd(lctx.lua_scripts,sha,l); + serverAssertWithInfo(c ? c : lctx.lua_client,NULL,retval == DICT_OK); + lctx.lua_scripts_mem += sdsZmallocSize(sha) + getStringObjectSdsUsedMemory(body); + incrRefCount(body); + return sha; +} + +void evalGenericCommand(client *c, int evalsha) { + lua_State *lua = lctx.lua; + char funcname[43]; + long long numkeys; + + /* Get the number of arguments that are keys */ + if (getLongLongFromObjectOrReply(c,c->argv[2],&numkeys,NULL) != C_OK) + return; + if (numkeys > (c->argc - 3)) { + addReplyError(c,"Number of keys can't be greater than number of args"); + return; + } else if (numkeys < 0) { + addReplyError(c,"Number of keys can't be negative"); + return; + } + + if (c->cur_script) { + funcname[0] = 'f', funcname[1] = '_'; + memcpy(funcname+2, dictGetKey(c->cur_script), 40); + funcname[42] = '\0'; + } else + evalCalcFunctionName(evalsha, c->argv[1]->ptr, funcname); + + /* Push the pcall error handler function on the stack. */ + lua_getglobal(lua, "__redis__err__handler"); + + /* Try to lookup the Lua function */ + lua_getfield(lua, LUA_REGISTRYINDEX, funcname); + if (lua_isnil(lua,-1)) { + lua_pop(lua,1); /* remove the nil from the stack */ + /* Function not defined... let's define it if we have the + * body of the function. If this is an EVALSHA call we can just + * return an error. */ + if (evalsha) { + lua_pop(lua,1); /* remove the error handler from the stack. */ + addReplyErrorObject(c, shared.noscripterr); + return; + } + if (luaCreateFunction(c,c->argv[1]) == NULL) { + lua_pop(lua,1); /* remove the error handler from the stack. */ + /* The error is sent to the client by luaCreateFunction() + * itself when it returns NULL. */ + return; + } + /* Now the following is guaranteed to return non nil */ + lua_getfield(lua, LUA_REGISTRYINDEX, funcname); + serverAssert(!lua_isnil(lua,-1)); + } + + char *lua_cur_script = funcname + 2; + dictEntry *de = c->cur_script; + if (!de) + de = dictFind(lctx.lua_scripts, lua_cur_script); + luaScript *l = dictGetVal(de); + int ro = c->cmd->proc == evalRoCommand || c->cmd->proc == evalShaRoCommand; + + scriptRunCtx rctx; + if (scriptPrepareForRun(&rctx, lctx.lua_client, c, lua_cur_script, l->flags, ro) != C_OK) { + lua_pop(lua,2); /* Remove the function and error handler. */ + return; + } + rctx.flags |= SCRIPT_EVAL_MODE; /* mark the current run as EVAL (as opposed to FCALL) so we'll + get appropriate error messages and logs */ + + luaCallFunction(&rctx, lua, c->argv+3, numkeys, c->argv+3+numkeys, c->argc-3-numkeys, ldb.active); + lua_pop(lua,1); /* Remove the error handler. */ + scriptResetRun(&rctx); +} + +void evalCommand(client *c) { + /* Explicitly feed monitor here so that lua commands appear after their + * script command. */ + replicationFeedMonitors(c,server.monitors,c->db->id,c->argv,c->argc); + if (!(c->flags & CLIENT_LUA_DEBUG)) + evalGenericCommand(c,0); + else + evalGenericCommandWithDebugging(c,0); +} + +void evalRoCommand(client *c) { + evalCommand(c); +} + +void evalShaCommand(client *c) { + /* Explicitly feed monitor here so that lua commands appear after their + * script command. */ + replicationFeedMonitors(c,server.monitors,c->db->id,c->argv,c->argc); + if (sdslen(c->argv[1]->ptr) != 40) { + /* We know that a match is not possible if the provided SHA is + * not the right length. So we return an error ASAP, this way + * evalGenericCommand() can be implemented without string length + * sanity check */ + addReplyErrorObject(c, shared.noscripterr); + return; + } + if (!(c->flags & CLIENT_LUA_DEBUG)) + evalGenericCommand(c,1); + else { + addReplyError(c,"Please use EVAL instead of EVALSHA for debugging"); + return; + } +} + +void evalShaRoCommand(client *c) { + evalShaCommand(c); +} + +void scriptCommand(client *c) { + if (c->argc == 2 && !strcasecmp(c->argv[1]->ptr,"help")) { + const char *help[] = { +"DEBUG (YES|SYNC|NO)", +" Set the debug mode for subsequent scripts executed.", +"EXISTS [ ...]", +" Return information about the existence of the scripts in the script cache.", +"FLUSH [ASYNC|SYNC]", +" Flush the Lua scripts cache. Very dangerous on replicas.", +" When called without the optional mode argument, the behavior is determined by the", +" lazyfree-lazy-user-flush configuration directive. Valid modes are:", +" * ASYNC: Asynchronously flush the scripts cache.", +" * SYNC: Synchronously flush the scripts cache.", +"KILL", +" Kill the currently executing Lua script.", +"LOAD