/* * Copyright (c) 2009-2012, Salvatore Sanfilippo * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * * Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * * Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * Neither the name of Redis nor the names of its contributors may be used * to endorse or promote products derived from this software without * specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ #include "server.h" #include "sha1.h" #include "rand.h" #include "cluster.h" #include "monotonic.h" #include "resp_parser.h" #include "script_lua.h" #include #include #include #include #include void ldbInit(void); void ldbDisable(client *c); void ldbEnable(client *c); void evalGenericCommandWithDebugging(client *c, int evalsha); sds ldbCatStackValue(sds s, lua_State *lua, int idx); static void dictLuaScriptDestructor(dict *d, void *val) { UNUSED(d); if (val == NULL) return; /* Lazy freeing will set value to NULL. */ decrRefCount(((luaScript*)val)->body); zfree(val); } static uint64_t dictStrCaseHash(const void *key) { return dictGenCaseHashFunction((unsigned char*)key, strlen((char*)key)); } /* server.lua_scripts sha (as sds string) -> scripts (as luaScript) cache. */ dictType shaScriptObjectDictType = { dictStrCaseHash, /* hash function */ NULL, /* key dup */ NULL, /* val dup */ dictSdsKeyCaseCompare, /* key compare */ dictSdsDestructor, /* key destructor */ dictLuaScriptDestructor, /* val destructor */ NULL /* allow to expand */ }; /* Lua context */ struct luaCtx { lua_State *lua; /* The Lua interpreter. We use just one for all clients */ client *lua_client; /* The "fake client" to query Redis from Lua */ dict *lua_scripts; /* A dictionary of SHA1 -> Lua scripts */ unsigned long long lua_scripts_mem; /* Cached scripts' memory + oh */ } lctx; /* Debugger shared state is stored inside this global structure. */ #define LDB_BREAKPOINTS_MAX 64 /* Max number of breakpoints. */ #define LDB_MAX_LEN_DEFAULT 256 /* Default len limit for replies / var dumps. */ struct ldbState { connection *conn; /* Connection of the debugging client. */ int active; /* Are we debugging EVAL right now? */ int forked; /* Is this a fork()ed debugging session? */ list *logs; /* List of messages to send to the client. */ list *traces; /* Messages about Redis commands executed since last stop.*/ list *children; /* All forked debugging sessions pids. */ int bp[LDB_BREAKPOINTS_MAX]; /* An array of breakpoints line numbers. */ int bpcount; /* Number of valid entries inside bp. */ int step; /* Stop at next line regardless of breakpoints. */ int luabp; /* Stop at next line because redis.breakpoint() was called. */ sds *src; /* Lua script source code split by line. */ int lines; /* Number of lines in 'src'. */ int currentline; /* Current line number. */ sds cbuf; /* Debugger client command buffer. */ size_t maxlen; /* Max var dump / reply length. */ int maxlen_hint_sent; /* Did we already hint about "set maxlen"? */ } ldb; /* --------------------------------------------------------------------------- * Utility functions. * ------------------------------------------------------------------------- */ /* Perform the SHA1 of the input string. We use this both for hashing script * bodies in order to obtain the Lua function name, and in the implementation * of redis.sha1(). * * 'digest' should point to a 41 bytes buffer: 40 for SHA1 converted into an * hexadecimal number, plus 1 byte for null term. */ void sha1hex(char *digest, char *script, size_t len) { SHA1_CTX ctx; unsigned char hash[20]; char *cset = "0123456789abcdef"; int j; SHA1Init(&ctx); SHA1Update(&ctx,(unsigned char*)script,len); SHA1Final(hash,&ctx); for (j = 0; j < 20; j++) { digest[j*2] = cset[((hash[j]&0xF0)>>4)]; digest[j*2+1] = cset[(hash[j]&0xF)]; } digest[40] = '\0'; } /* redis.breakpoint() * * Allows to stop execution during a debugging session from within * the Lua code implementation, like if a breakpoint was set in the code * immediately after the function. */ int luaRedisBreakpointCommand(lua_State *lua) { if (ldb.active) { ldb.luabp = 1; lua_pushboolean(lua,1); } else { lua_pushboolean(lua,0); } return 1; } /* redis.debug() * * Log a string message into the output console. * Can take multiple arguments that will be separated by commas. * Nothing is returned to the caller. */ int luaRedisDebugCommand(lua_State *lua) { if (!ldb.active) return 0; int argc = lua_gettop(lua); sds log = sdscatprintf(sdsempty()," line %d: ", ldb.currentline); while(argc--) { log = ldbCatStackValue(log,lua,-1 - argc); if (argc != 0) log = sdscatlen(log,", ",2); } ldbLog(log); return 0; } /* redis.replicate_commands() * * DEPRECATED: Now do nothing and always return true. * Turn on single commands replication if the script never called * a write command so far, and returns true. Otherwise if the script * already started to write, returns false and stick to whole scripts * replication, which is our default. */ int luaRedisReplicateCommandsCommand(lua_State *lua) { lua_pushboolean(lua,1); return 1; } /* Initialize the scripting environment. * * This function is called the first time at server startup with * the 'setup' argument set to 1. * * It can be called again multiple times during the lifetime of the Redis * process, with 'setup' set to 0, and following a scriptingRelease() call, * in order to reset the Lua scripting environment. * * However it is simpler to just call scriptingReset() that does just that. */ void scriptingInit(int setup) { lua_State *lua = lua_open(); if (setup) { lctx.lua_client = NULL; server.script_caller = NULL; server.script_disable_deny_script = 0; ldbInit(); } /* Initialize a dictionary we use to map SHAs to scripts. * This is useful for replication, as we need to replicate EVALSHA * as EVAL, so we need to remember the associated script. */ lctx.lua_scripts = dictCreate(&shaScriptObjectDictType); lctx.lua_scripts_mem = 0; luaRegisterRedisAPI(lua); /* register debug commands */ lua_getglobal(lua,"redis"); /* redis.breakpoint */ lua_pushstring(lua,"breakpoint"); lua_pushcfunction(lua,luaRedisBreakpointCommand); lua_settable(lua,-3); /* redis.debug */ lua_pushstring(lua,"debug"); lua_pushcfunction(lua,luaRedisDebugCommand); lua_settable(lua,-3); /* redis.replicate_commands */ lua_pushstring(lua, "replicate_commands"); lua_pushcfunction(lua, luaRedisReplicateCommandsCommand); lua_settable(lua, -3); lua_setglobal(lua,"redis"); /* Add a helper function we use for pcall error reporting. * Note that when the error is in the C function we want to report the * information about the caller, that's what makes sense from the point * of view of the user debugging a script. */ { char *errh_func = "local dbg = debug\n" "debug = nil\n" "function __redis__err__handler(err)\n" " local i = dbg.getinfo(2,'nSl')\n" " if i and i.what == 'C' then\n" " i = dbg.getinfo(3,'nSl')\n" " end\n" " if type(err) ~= 'table' then\n" " err = {err='ERR ' .. tostring(err)}" " end" " if i then\n" " err['source'] = i.source\n" " err['line'] = i.currentline\n" " end" " return err\n" "end\n"; luaL_loadbuffer(lua,errh_func,strlen(errh_func),"@err_handler_def"); lua_pcall(lua,0,0,0); } /* Create the (non connected) client that we use to execute Redis commands * inside the Lua interpreter. * Note: there is no need to create it again when this function is called * by scriptingReset(). */ if (lctx.lua_client == NULL) { lctx.lua_client = createClient(NULL); lctx.lua_client->flags |= CLIENT_SCRIPT; /* We do not want to allow blocking commands inside Lua */ lctx.lua_client->flags |= CLIENT_DENY_BLOCKING; } /* Lock the global table from any changes */ lua_pushvalue(lua, LUA_GLOBALSINDEX); luaSetErrorMetatable(lua); /* Recursively lock all tables that can be reached from the global table */ luaSetTableProtectionRecursively(lua); lua_pop(lua, 1); lctx.lua = lua; } /* Release resources related to Lua scripting. * This function is used in order to reset the scripting environment. */ void scriptingRelease(int async) { if (async) freeLuaScriptsAsync(lctx.lua_scripts); else dictRelease(lctx.lua_scripts); lctx.lua_scripts_mem = 0; lua_close(lctx.lua); } void scriptingReset(int async) { scriptingRelease(async); scriptingInit(0); } /* --------------------------------------------------------------------------- * EVAL and SCRIPT commands implementation * ------------------------------------------------------------------------- */ static void evalCalcFunctionName(int evalsha, sds script, char *out_funcname) { /* We obtain the script SHA1, then check if this function is already * defined into the Lua state */ out_funcname[0] = 'f'; out_funcname[1] = '_'; if (!evalsha) { /* Hash the code if this is an EVAL call */ sha1hex(out_funcname+2,script,sdslen(script)); } else { /* We already have the SHA if it is an EVALSHA */ int j; char *sha = script; /* Convert to lowercase. We don't use tolower since the function * managed to always show up in the profiler output consuming * a non trivial amount of time. */ for (j = 0; j < 40; j++) out_funcname[j+2] = (sha[j] >= 'A' && sha[j] <= 'Z') ? sha[j]+('a'-'A') : sha[j]; out_funcname[42] = '\0'; } } /* Helper function to try and extract shebang flags from the script body. * If no shebang is found, return with success and COMPAT mode flag. * The err arg is optional, can be used to get a detailed error string. * The out_shebang_len arg is optional, can be used to trim the shebang from the script. * Returns C_OK on success, and C_ERR on error. */ int evalExtractShebangFlags(sds body, uint64_t *out_flags, ssize_t *out_shebang_len, sds *err) { ssize_t shebang_len = 0; uint64_t script_flags = SCRIPT_FLAG_EVAL_COMPAT_MODE; if (!strncmp(body, "#!", 2)) { int numparts,j; char *shebang_end = strchr(body, '\n'); if (shebang_end == NULL) { if (err) *err = sdsnew("Invalid script shebang"); return C_ERR; } shebang_len = shebang_end - body; sds shebang = sdsnewlen(body, shebang_len); sds *parts = sdssplitargs(shebang, &numparts); sdsfree(shebang); if (!parts || numparts == 0) { if (err) *err = sdsnew("Invalid engine in script shebang"); sdsfreesplitres(parts, numparts); return C_ERR; } /* Verify lua interpreter was specified */ if (strcmp(parts[0], "#!lua")) { if (err) *err = sdscatfmt(sdsempty(), "Unexpected engine in script shebang: %s", parts[0]); sdsfreesplitres(parts, numparts); return C_ERR; } script_flags &= ~SCRIPT_FLAG_EVAL_COMPAT_MODE; for (j = 1; j < numparts; j++) { if (!strncmp(parts[j], "flags=", 6)) { sdsrange(parts[j], 6, -1); int numflags, jj; sds *flags = sdssplitlen(parts[j], sdslen(parts[j]), ",", 1, &numflags); for (jj = 0; jj < numflags; jj++) { scriptFlag *sf; for (sf = scripts_flags_def; sf->flag; sf++) { if (!strcmp(flags[jj], sf->str)) break; } if (!sf->flag) { if (err) *err = sdscatfmt(sdsempty(), "Unexpected flag in script shebang: %s", flags[jj]); sdsfreesplitres(flags, numflags); sdsfreesplitres(parts, numparts); return C_ERR; } script_flags |= sf->flag; } sdsfreesplitres(flags, numflags); } else { /* We only support function flags options for lua scripts */ if (err) *err = sdscatfmt(sdsempty(), "Unknown lua shebang option: %s", parts[j]); sdsfreesplitres(parts, numparts); return C_ERR; } } sdsfreesplitres(parts, numparts); } if (out_shebang_len) *out_shebang_len = shebang_len; *out_flags = script_flags; return C_OK; } /* Try to extract command flags if we can, returns the modified flags. * Note that it does not guarantee the command arguments are right. */ uint64_t evalGetCommandFlags(client *c, uint64_t cmd_flags) { char funcname[43]; int evalsha = c->cmd->proc == evalShaCommand || c->cmd->proc == evalShaRoCommand; if (evalsha && sdslen(c->argv[1]->ptr) != 40) return cmd_flags; uint64_t script_flags; evalCalcFunctionName(evalsha, c->argv[1]->ptr, funcname); char *lua_cur_script = funcname + 2; c->cur_script = dictFind(lctx.lua_scripts, lua_cur_script); if (!c->cur_script) { if (evalsha) return cmd_flags; if (evalExtractShebangFlags(c->argv[1]->ptr, &script_flags, NULL, NULL) == C_ERR) return cmd_flags; } else { luaScript *l = dictGetVal(c->cur_script); script_flags = l->flags; } if (script_flags & SCRIPT_FLAG_EVAL_COMPAT_MODE) return cmd_flags; return scriptFlagsToCmdFlags(cmd_flags, script_flags); } /* Define a Lua function with the specified body. * The function name will be generated in the following form: * * f_ * * The function increments the reference count of the 'body' object as a * side effect of a successful call. * * On success a pointer to an SDS string representing the function SHA1 of the * just added function is returned (and will be valid until the next call * to scriptingReset() function), otherwise NULL is returned. * * The function handles the fact of being called with a script that already * exists, and in such a case, it behaves like in the success case. * * If 'c' is not NULL, on error the client is informed with an appropriate * error describing the nature of the problem and the Lua interpreter error. */ sds luaCreateFunction(client *c, robj *body) { char funcname[43]; dictEntry *de; uint64_t script_flags; funcname[0] = 'f'; funcname[1] = '_'; sha1hex(funcname+2,body->ptr,sdslen(body->ptr)); if ((de = dictFind(lctx.lua_scripts,funcname+2)) != NULL) { return dictGetKey(de); } /* Handle shebang header in script code */ ssize_t shebang_len = 0; sds err = NULL; if (evalExtractShebangFlags(body->ptr, &script_flags, &shebang_len, &err) == C_ERR) { addReplyErrorSds(c, err); return NULL; } /* Note that in case of a shebang line we skip it but keep the line feed to conserve the user's line numbers */ if (luaL_loadbuffer(lctx.lua,(char*)body->ptr + shebang_len,sdslen(body->ptr) - shebang_len,"@user_script")) { if (c != NULL) { addReplyErrorFormat(c, "Error compiling script (new function): %s", lua_tostring(lctx.lua,-1)); } lua_pop(lctx.lua,1); return NULL; } serverAssert(lua_isfunction(lctx.lua, -1)); lua_setfield(lctx.lua, LUA_REGISTRYINDEX, funcname); /* We also save a SHA1 -> Original script map in a dictionary * so that we can replicate / write in the AOF all the * EVALSHA commands as EVAL using the original script. */ luaScript *l = zcalloc(sizeof(luaScript)); l->body = body; l->flags = script_flags; sds sha = sdsnewlen(funcname+2,40); int retval = dictAdd(lctx.lua_scripts,sha,l); serverAssertWithInfo(c ? c : lctx.lua_client,NULL,retval == DICT_OK); lctx.lua_scripts_mem += sdsZmallocSize(sha) + getStringObjectSdsUsedMemory(body); incrRefCount(body); return sha; } void prepareLuaClient(void) { /* Select the right DB in the context of the Lua client */ selectDb(lctx.lua_client,server.script_caller->db->id); lctx.lua_client->resp = 2; /* Default is RESP2, scripts can change it. */ /* If we are in MULTI context, flag Lua client as CLIENT_MULTI. */ if (server.script_caller->flags & CLIENT_MULTI) { lctx.lua_client->flags |= CLIENT_MULTI; } } void resetLuaClient(void) { /* After the script done, remove the MULTI state. */ lctx.lua_client->flags &= ~CLIENT_MULTI; } void evalGenericCommand(client *c, int evalsha) { lua_State *lua = lctx.lua; char funcname[43]; long long numkeys; /* Get the number of arguments that are keys */ if (getLongLongFromObjectOrReply(c,c->argv[2],&numkeys,NULL) != C_OK) return; if (numkeys > (c->argc - 3)) { addReplyError(c,"Number of keys can't be greater than number of args"); return; } else if (numkeys < 0) { addReplyError(c,"Number of keys can't be negative"); return; } if (c->cur_script) { funcname[0] = 'f', funcname[1] = '_'; memcpy(funcname+2, dictGetKey(c->cur_script), 40); funcname[42] = '\0'; } else evalCalcFunctionName(evalsha, c->argv[1]->ptr, funcname); /* Push the pcall error handler function on the stack. */ lua_getglobal(lua, "__redis__err__handler"); /* Try to lookup the Lua function */ lua_getfield(lua, LUA_REGISTRYINDEX, funcname); if (lua_isnil(lua,-1)) { lua_pop(lua,1); /* remove the nil from the stack */ /* Function not defined... let's define it if we have the * body of the function. If this is an EVALSHA call we can just * return an error. */ if (evalsha) { lua_pop(lua,1); /* remove the error handler from the stack. */ addReplyErrorObject(c, shared.noscripterr); return; } if (luaCreateFunction(c,c->argv[1]) == NULL) { lua_pop(lua,1); /* remove the error handler from the stack. */ /* The error is sent to the client by luaCreateFunction() * itself when it returns NULL. */ return; } /* Now the following is guaranteed to return non nil */ lua_getfield(lua, LUA_REGISTRYINDEX, funcname); serverAssert(!lua_isnil(lua,-1)); } char *lua_cur_script = funcname + 2; dictEntry *de = c->cur_script; if (!de) de = dictFind(lctx.lua_scripts, lua_cur_script); luaScript *l = dictGetVal(de); int ro = c->cmd->proc == evalRoCommand || c->cmd->proc == evalShaRoCommand; scriptRunCtx rctx; if (scriptPrepareForRun(&rctx, lctx.lua_client, c, lua_cur_script, l->flags, ro) != C_OK) { lua_pop(lua,2); /* Remove the function and error handler. */ return; } rctx.flags |= SCRIPT_EVAL_MODE; /* mark the current run as EVAL (as opposed to FCALL) so we'll get appropriate error messages and logs */ luaCallFunction(&rctx, lua, c->argv+3, numkeys, c->argv+3+numkeys, c->argc-3-numkeys, ldb.active); lua_pop(lua,1); /* Remove the error handler. */ scriptResetRun(&rctx); } void evalCommand(client *c) { /* Explicitly feed monitor here so that lua commands appear after their * script command. */ replicationFeedMonitors(c,server.monitors,c->db->id,c->argv,c->argc); if (!(c->flags & CLIENT_LUA_DEBUG)) evalGenericCommand(c,0); else evalGenericCommandWithDebugging(c,0); } void evalRoCommand(client *c) { evalCommand(c); } void evalShaCommand(client *c) { /* Explicitly feed monitor here so that lua commands appear after their * script command. */ replicationFeedMonitors(c,server.monitors,c->db->id,c->argv,c->argc); if (sdslen(c->argv[1]->ptr) != 40) { /* We know that a match is not possible if the provided SHA is * not the right length. So we return an error ASAP, this way * evalGenericCommand() can be implemented without string length * sanity check */ addReplyErrorObject(c, shared.noscripterr); return; } if (!(c->flags & CLIENT_LUA_DEBUG)) evalGenericCommand(c,1); else { addReplyError(c,"Please use EVAL instead of EVALSHA for debugging"); return; } } void evalShaRoCommand(client *c) { evalShaCommand(c); } void scriptCommand(client *c) { if (c->argc == 2 && !strcasecmp(c->argv[1]->ptr,"help")) { const char *help[] = { "DEBUG (YES|SYNC|NO)", " Set the debug mode for subsequent scripts executed.", "EXISTS [ ...]", " Return information about the existence of the scripts in the script cache.", "FLUSH [ASYNC|SYNC]", " Flush the Lua scripts cache. Very dangerous on replicas.", " When called without the optional mode argument, the behavior is determined by the", " lazyfree-lazy-user-flush configuration directive. Valid modes are:", " * ASYNC: Asynchronously flush the scripts cache.", " * SYNC: Synchronously flush the scripts cache.", "KILL", " Kill the currently executing Lua script.", "LOAD