1 files changed, 89 insertions, 220 deletions

diff --git a/web/server/web_client_cache.c b/web/server/web_client_cache.c
index 4344209c..b410ba7f 100644
--- a/web/server/web_client_cache.c
+++ b/web/server/web_client_cache.c
@@ -6,77 +6,6 @@
 // ----------------------------------------------------------------------------
 // allocate and free web_clients
 
-#ifdef ENABLE_HTTPS
-
-static void web_client_reuse_ssl(struct web_client *w) {
-    if (netdata_ssl_srv_ctx) {
-        if (w->ssl.conn) {
-            SSL_SESSION *session = SSL_get_session(w->ssl.conn);
-            SSL *old = w->ssl.conn;
-            w->ssl.conn = SSL_new(netdata_ssl_srv_ctx);
-            if (session) {
-#if OPENSSL_VERSION_NUMBER >= OPENSSL_VERSION_111
-                if (SSL_SESSION_is_resumable(session))
-#endif
-                    SSL_set_session(w->ssl.conn, session);
-            }
-            SSL_free(old);
-        }
-    }
-}
-#endif
-
-
-static void web_client_zero(struct web_client *w) {
-    // zero everything about it - but keep the buffers
-
-    // remember the pointers to the buffers
-    BUFFER *b1 = w->response.data;
-    BUFFER *b2 = w->response.header;
-    BUFFER *b3 = w->response.header_output;
-
-    // empty the buffers
-    buffer_flush(b1);
-    buffer_flush(b2);
-    buffer_flush(b3);
-
-    freez(w->user_agent);
-
-    // zero everything
-    memset(w, 0, sizeof(struct web_client));
-
-    // restore the pointers of the buffers
-    w->response.data = b1;
-    w->response.header = b2;
-    w->response.header_output = b3;
-}
-
-static void web_client_free(struct web_client *w) {
-    buffer_free(w->response.header_output);
-    buffer_free(w->response.header);
-    buffer_free(w->response.data);
-    freez(w->user_agent);
-#ifdef ENABLE_HTTPS
-    if ((!web_client_check_unix(w)) && (netdata_ssl_srv_ctx)) {
-        if (w->ssl.conn) {
-            SSL_free(w->ssl.conn);
-            w->ssl.conn = NULL;
-        }
-    }
-#endif
-    freez(w);
-    __atomic_sub_fetch(&netdata_buffers_statistics.buffers_web, sizeof(struct web_client), __ATOMIC_RELAXED);
-}
-
-static struct web_client *web_client_alloc(void) {
-    struct web_client *w = callocz(1, sizeof(struct web_client));
-    __atomic_add_fetch(&netdata_buffers_statistics.buffers_web, sizeof(struct web_client), __ATOMIC_RELAXED);
-    w->response.data = buffer_create(NETDATA_WEB_RESPONSE_INITIAL_SIZE, &netdata_buffers_statistics.buffers_web);
-    w->response.header = buffer_create(NETDATA_WEB_RESPONSE_HEADER_SIZE, &netdata_buffers_statistics.buffers_web);
-    w->response.header_output = buffer_create(NETDATA_WEB_RESPONSE_HEADER_SIZE, &netdata_buffers_statistics.buffers_web);
-    return w;
-}
-
 // ----------------------------------------------------------------------------
 // web clients caching
 
@@ -87,194 +16,134 @@ static struct web_client *web_client_alloc(void) {
 // The size of the cache is adaptive. It caches the structures of 2x
 // the number of currently connected clients.
 
-// Comments per server:
-// SINGLE-THREADED : 1 cache is maintained
-// MULTI-THREADED  : 1 cache is maintained
-// STATIC-THREADED : 1 cache for each thread of the web server
-
-__thread struct clients_cache web_clients_cache = {
-        .pid = 0,
-        .used = NULL,
-        .used_count = 0,
-        .avail = NULL,
-        .avail_count = 0,
-        .allocated = 0,
-        .reused = 0
+static struct clients_cache {
+    struct {
+        SPINLOCK spinlock;
+        struct web_client *head;    // the structures of the currently connected clients
+        size_t count;               // the count the currently connected clients
+
+        size_t allocated;           // the number of allocations
+        size_t reused;              // the number of re-uses
+    } used;
+
+    struct {
+        SPINLOCK spinlock;
+        struct web_client *head;    // the cached structures, available for future clients
+        size_t count;               // the number of cached structures
+    } avail;
+} web_clients_cache = {
+        .used = {
+                .spinlock = NETDATA_SPINLOCK_INITIALIZER,
+                .head = NULL,
+                .count = 0,
+                .reused = 0,
+                .allocated = 0,
+        },
+        .avail = {
+                .spinlock = NETDATA_SPINLOCK_INITIALIZER,
+                .head = NULL,
+                .count = 0,
+        },
 };
 
-inline void web_client_cache_verify(int force) {
-#ifdef NETDATA_INTERNAL_CHECKS
-    static __thread size_t count = 0;
-    count++;
-
-    if(unlikely(force || count > 1000)) {
-        count = 0;
-
-        struct web_client *w;
-        size_t used = 0, avail = 0;
-        for(w = web_clients_cache.used; w ; w = w->next) used++;
-        for(w = web_clients_cache.avail; w ; w = w->next) avail++;
-
-        info("web_client_cache has %zu (%zu) used and %zu (%zu) available clients, allocated %zu, reused %zu (hit %zu%%)."
-             , used, web_clients_cache.used_count
-             , avail, web_clients_cache.avail_count
-             , web_clients_cache.allocated
-             , web_clients_cache.reused
-             , (web_clients_cache.allocated + web_clients_cache.reused)?(web_clients_cache.reused * 100 / (web_clients_cache.allocated + web_clients_cache.reused)):0
-        );
-    }
-#else
-    if(unlikely(force)) {
-        info("web_client_cache has %zu used and %zu available clients, allocated %zu, reused %zu (hit %zu%%)."
-             , web_clients_cache.used_count
-             , web_clients_cache.avail_count
-             , web_clients_cache.allocated
-             , web_clients_cache.reused
-             , (web_clients_cache.allocated + web_clients_cache.reused)?(web_clients_cache.reused * 100 / (web_clients_cache.allocated + web_clients_cache.reused)):0
-            );
-    }
-#endif
-}
-
 // destroy the cache and free all the memory it uses
 void web_client_cache_destroy(void) {
-#ifdef NETDATA_INTERNAL_CHECKS
-    if(unlikely(web_clients_cache.pid != 0 && web_clients_cache.pid != gettid()))
-        error("Oops! wrong thread accessing the cache. Expected %d, found %d", (int)web_clients_cache.pid, (int)gettid());
-
-    web_client_cache_verify(1);
-#endif
-
-    netdata_thread_disable_cancelability();
+    internal_error(true, "web_client_cache has %zu used and %zu available clients, allocated %zu, reused %zu (hit %zu%%)."
+        , web_clients_cache.used.count
+        , web_clients_cache.avail.count
+        , web_clients_cache.used.allocated
+        , web_clients_cache.used.reused
+        , (web_clients_cache.used.allocated + web_clients_cache.used.reused)?(web_clients_cache.used.reused * 100 / (web_clients_cache.used.allocated + web_clients_cache.used.reused)):0
+        );
 
     struct web_client *w, *t;
 
-    w = web_clients_cache.used;
+    netdata_spinlock_lock(&web_clients_cache.avail.spinlock);
+    w = web_clients_cache.avail.head;
     while(w) {
         t = w;
-        w = w->next;
+        w = w->cache.next;
         web_client_free(t);
     }
-    web_clients_cache.used = NULL;
-    web_clients_cache.used_count = 0;
-
-    w = web_clients_cache.avail;
-    while(w) {
-        t = w;
-        w = w->next;
-        web_client_free(t);
-    }
-    web_clients_cache.avail = NULL;
-    web_clients_cache.avail_count = 0;
-
-    netdata_thread_enable_cancelability();
+    web_clients_cache.avail.head = NULL;
+    web_clients_cache.avail.count = 0;
+    netdata_spinlock_unlock(&web_clients_cache.avail.spinlock);
+
+// DO NOT FREE THEM IF THEY ARE USED
+//    netdata_spinlock_lock(&web_clients_cache.used.spinlock);
+//    w = web_clients_cache.used.head;
+//    while(w) {
+//        t = w;
+//        w = w->next;
+//        web_client_free(t);
+//    }
+//    web_clients_cache.used.head = NULL;
+//    web_clients_cache.used.count = 0;
+//    web_clients_cache.used.reused = 0;
+//    web_clients_cache.used.allocated = 0;
+//    netdata_spinlock_unlock(&web_clients_cache.used.spinlock);
 }
 
-struct web_client *web_client_get_from_cache_or_allocate() {
-
-#ifdef NETDATA_INTERNAL_CHECKS
-    if(unlikely(web_clients_cache.pid == 0))
-        web_clients_cache.pid = gettid();
-
-    if(unlikely(web_clients_cache.pid != 0 && web_clients_cache.pid != gettid()))
-        error("Oops! wrong thread accessing the cache. Expected %d, found %d", (int)web_clients_cache.pid, (int)gettid());
-#endif
-
-    netdata_thread_disable_cancelability();
-
-    struct web_client *w = web_clients_cache.avail;
-
+struct web_client *web_client_get_from_cache(void) {
+    netdata_spinlock_lock(&web_clients_cache.avail.spinlock);
+    struct web_client *w = web_clients_cache.avail.head;
     if(w) {
         // get it from avail
-        if (w == web_clients_cache.avail) web_clients_cache.avail = w->next;
-        if(w->prev) w->prev->next = w->next;
-        if(w->next) w->next->prev = w->prev;
-        web_clients_cache.avail_count--;
-#ifdef ENABLE_HTTPS
-        web_client_reuse_ssl(w);
-        SSL *ssl = w->ssl.conn;
-#endif
+        DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(web_clients_cache.avail.head, w, cache.prev, cache.next);
+        web_clients_cache.avail.count--;
+        netdata_spinlock_unlock(&web_clients_cache.avail.spinlock);
+
         web_client_zero(w);
-        web_clients_cache.reused++;
-#ifdef ENABLE_HTTPS
-        w->ssl.conn = ssl;
-        w->ssl.flags = NETDATA_SSL_START;
-        debug(D_WEB_CLIENT_ACCESS,"Reusing SSL structure with (w->ssl = NULL, w->accepted = %u)", w->ssl.flags);
-#endif
+
+        netdata_spinlock_lock(&web_clients_cache.used.spinlock);
+        web_clients_cache.used.reused++;
     }
     else {
+        netdata_spinlock_unlock(&web_clients_cache.avail.spinlock);
+
         // allocate it
-        w = web_client_alloc();
+        w = web_client_create(&netdata_buffers_statistics.buffers_web);
+
 #ifdef ENABLE_HTTPS
         w->ssl.flags = NETDATA_SSL_START;
         debug(D_WEB_CLIENT_ACCESS,"Starting SSL structure with (w->ssl = NULL, w->accepted = %u)", w->ssl.flags);
 #endif
-        web_clients_cache.allocated++;
+
+        netdata_spinlock_lock(&web_clients_cache.used.spinlock);
+        web_clients_cache.used.allocated++;
     }
 
     // link it to used web clients
-    if (web_clients_cache.used) web_clients_cache.used->prev = w;
-    w->next = web_clients_cache.used;
-    w->prev = NULL;
-    web_clients_cache.used = w;
-    web_clients_cache.used_count++;
+    DOUBLE_LINKED_LIST_PREPEND_ITEM_UNSAFE(web_clients_cache.used.head, w, cache.prev, cache.next);
+    web_clients_cache.used.count++;
+    netdata_spinlock_unlock(&web_clients_cache.used.spinlock);
 
     // initialize it
+    w->use_count++;
     w->id = global_statistics_web_client_connected();
-    w->mode = WEB_CLIENT_MODE_NORMAL;
-
-    netdata_thread_enable_cancelability();
+    w->mode = WEB_CLIENT_MODE_GET;
 
     return w;
 }
 
-void web_client_release(struct web_client *w) {
-#ifdef NETDATA_INTERNAL_CHECKS
-    if(unlikely(web_clients_cache.pid != 0 && web_clients_cache.pid != gettid()))
-        error("Oops! wrong thread accessing the cache. Expected %d, found %d", (int)web_clients_cache.pid, (int)gettid());
-
-    if(unlikely(w->running))
-        error("%llu: releasing web client from %s port %s, but it still running.", w->id, w->client_ip, w->client_port);
-#endif
-
-    debug(D_WEB_CLIENT_ACCESS, "%llu: Closing web client from %s port %s.", w->id, w->client_ip, w->client_port);
-
-    web_server_log_connection(w, "DISCONNECTED");
-    web_client_request_done(w);
-    global_statistics_web_client_disconnected();
-
-    netdata_thread_disable_cancelability();
-
-    if(web_server_mode != WEB_SERVER_MODE_STATIC_THREADED) {
-        if (w->ifd != -1) close(w->ifd);
-        if (w->ofd != -1 && w->ofd != w->ifd) close(w->ofd);
-        w->ifd = w->ofd = -1;
-#ifdef ENABLE_HTTPS
-        web_client_reuse_ssl(w);
-        w->ssl.flags = NETDATA_SSL_START;
-#endif
-
-    }
-
+void web_client_release_to_cache(struct web_client *w) {
     // unlink it from the used
-    if (w == web_clients_cache.used) web_clients_cache.used = w->next;
-    if(w->prev) w->prev->next = w->next;
-    if(w->next) w->next->prev = w->prev;
-    web_clients_cache.used_count--;
+    netdata_spinlock_lock(&web_clients_cache.used.spinlock);
+    DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(web_clients_cache.used.head, w, cache.prev, cache.next);
+    ssize_t used_count = (ssize_t)--web_clients_cache.used.count;
+    netdata_spinlock_unlock(&web_clients_cache.used.spinlock);
+
+    netdata_spinlock_lock(&web_clients_cache.avail.spinlock);
+    if(w->use_count > 100 || (used_count > 0 && web_clients_cache.avail.count >= 2 * (size_t)used_count) || (used_count <= 10 && web_clients_cache.avail.count >= 20)) {
+        netdata_spinlock_unlock(&web_clients_cache.avail.spinlock);
 
-    if(web_clients_cache.avail_count >= 2 * web_clients_cache.used_count) {
         // we have too many of them - free it
         web_client_free(w);
     }
     else {
         // link it to the avail
-        if (web_clients_cache.avail) web_clients_cache.avail->prev = w;
-        w->next = web_clients_cache.avail;
-        w->prev = NULL;
-        web_clients_cache.avail = w;
-        web_clients_cache.avail_count++;
+        DOUBLE_LINKED_LIST_PREPEND_ITEM_UNSAFE(web_clients_cache.avail.head, w, cache.prev, cache.next);
+        web_clients_cache.avail.count++;
+        netdata_spinlock_unlock(&web_clients_cache.avail.spinlock);
     }
-
-    netdata_thread_enable_cancelability();
 }
-