Imported Upstream version 1.2.0upstream/1.2.0

1 files changed, 1838 insertions, 0 deletions

diff --git a/src/registry.c b/src/registry.c
new file mode 100644
index 000000000..f39ce3e2e
--- /dev/null
+++ b/src/registry.c
@@ -0,0 +1,1838 @@
+#ifdef HAVE_CONFIG_H
+#include <config.h>
+#endif
+
+#include <uuid/uuid.h>
+#include <inttypes.h>
+#include <stdlib.h>
+#include <string.h>
+#include <ctype.h>
+#include <unistd.h>
+#include <sys/stat.h>
+#include <sys/types.h>
+#include <errno.h>
+#include <fcntl.h>
+
+#include "log.h"
+#include "common.h"
+#include "dictionary.h"
+#include "appconfig.h"
+
+#include "web_client.h"
+#include "rrd.h"
+#include "rrd2json.h"
+#include "registry.h"
+
+
+// ----------------------------------------------------------------------------
+// TODO
+//
+// 1. the default tracking cookie expires in 1 year, but the persons are not
+//    removed from the db - this means the database only grows - ideally the
+//    database should be cleaned in registry_save() for both on-disk and
+//    on-memory entries.
+//
+//    Cleanup:
+//    i. Find all the PERSONs that have expired cookie
+//    ii. For each of their PERSON_URLs:
+//     - decrement the linked MACHINE links
+//     - if the linked MACHINE has no other links, remove the linked MACHINE too
+//     - remove the PERSON_URL
+//
+// 2. add protection to prevent abusing the registry by flooding it with
+//    requests to fill the memory and crash it.
+//
+//    Possible protections:
+//    - limit the number of URLs per person
+//    - limit the number of URLs per machine
+//    - limit the number of persons
+//    - limit the number of machines
+//    - [DONE] limit the size of URLs
+//    - [DONE] limit the size of PERSON_URL names
+//    - limit the number of requests that add data to the registry,
+//      per client IP per hour
+
+
+
+#define REGISTRY_URL_FLAGS_DEFAULT 0x00
+#define REGISTRY_URL_FLAGS_EXPIRED 0x01
+
+#define DICTIONARY_FLAGS DICTIONARY_FLAG_VALUE_LINK_DONT_CLONE | DICTIONARY_FLAG_NAME_LINK_DONT_CLONE
+
+// ----------------------------------------------------------------------------
+// COMMON structures
+
+struct registry {
+	int enabled;
+
+	char machine_guid[36 + 1];
+
+	// entries counters / statistics
+	unsigned long long persons_count;
+	unsigned long long machines_count;
+	unsigned long long usages_count;
+	unsigned long long urls_count;
+	unsigned long long persons_urls_count;
+	unsigned long long machines_urls_count;
+	unsigned long long log_count;
+
+	// memory counters / statistics
+	unsigned long long persons_memory;
+	unsigned long long machines_memory;
+	unsigned long long urls_memory;
+	unsigned long long persons_urls_memory;
+	unsigned long long machines_urls_memory;
+
+	// configuration
+	unsigned long long save_registry_every_entries;
+	char *registry_domain;
+	char *hostname;
+	char *registry_to_announce;
+	time_t persons_expiration; // seconds to expire idle persons
+
+	size_t max_url_length;
+	size_t max_name_length;
+
+	// file/path names
+	char *pathname;
+	char *db_filename;
+	char *log_filename;
+	char *machine_guid_filename;
+
+	// open files
+	FILE *log_fp;
+
+	// the database
+	DICTIONARY *persons; 	// dictionary of PERSON *, with key the PERSON.guid
+	DICTIONARY *machines; 	// dictionary of MACHINE *, with key the MACHINE.guid
+	DICTIONARY *urls; 		// dictionary of URL *, with key the URL.url
+
+	// concurrency locking
+	// we keep different locks for different things
+	// so that many tasks can be completed in parallel
+	pthread_mutex_t persons_lock;
+	pthread_mutex_t machines_lock;
+	pthread_mutex_t urls_lock;
+	pthread_mutex_t person_urls_lock;
+	pthread_mutex_t machine_urls_lock;
+	pthread_mutex_t log_lock;
+} registry;
+
+
+// ----------------------------------------------------------------------------
+// URL structures
+// Save memory by de-duplicating URLs
+// so instead of storing URLs all over the place
+// we store them here and we keep pointers elsewhere
+
+struct url {
+	uint32_t links;	// the number of links to this URL - when none is left, we free it
+	uint16_t len;	// the length of the URL in bytes
+	char url[1];	// the URL - dynamically allocated to more size
+};
+typedef struct url URL;
+
+
+// ----------------------------------------------------------------------------
+// MACHINE structures
+
+// For each MACHINE-URL pair we keep this
+struct machine_url {
+	URL *url;					// de-duplicated URL
+//	DICTIONARY *persons;		// dictionary of PERSON *
+
+	uint8_t flags;
+	uint32_t first_t;			// the first time we saw this
+	uint32_t last_t;			// the last time we saw this
+	uint32_t usages;			// how many times this has been accessed
+};
+typedef struct machine_url MACHINE_URL;
+
+// A machine
+struct machine {
+	char guid[36 + 1];			// the GUID
+
+	uint32_t links;				// the number of PERSON_URLs linked to this machine
+
+	DICTIONARY *urls; 			// MACHINE_URL *
+
+	uint32_t first_t;			// the first time we saw this
+	uint32_t last_t;			// the last time we saw this
+	uint32_t usages;			// how many times this has been accessed
+};
+typedef struct machine MACHINE;
+
+
+// ----------------------------------------------------------------------------
+// PERSON structures
+
+// for each PERSON-URL pair we keep this
+struct person_url {
+	URL *url;					// de-duplicated URL
+	MACHINE *machine;			// link the MACHINE of this URL
+
+	uint8_t flags;
+	uint32_t first_t;			// the first time we saw this
+	uint32_t last_t;			// the last time we saw this
+	uint32_t usages;			// how many times this has been accessed
+
+	char name[1];				// the name of the URL, as known by the user
+								// dynamically allocated to fit properly
+};
+typedef struct person_url PERSON_URL;
+
+// A person
+struct person {
+	char guid[36 + 1];			// the person GUID
+
+	DICTIONARY *urls;			// dictionary of PERSON_URL *
+
+	uint32_t first_t;			// the first time we saw this
+	uint32_t last_t;			// the last time we saw this
+	uint32_t usages;			// how many times this has been accessed
+};
+typedef struct person PERSON;
+
+
+// ----------------------------------------------------------------------------
+// REGISTRY concurrency locking
+
+static inline void registry_persons_lock(void) {
+	pthread_mutex_lock(&registry.persons_lock);
+}
+
+static inline void registry_persons_unlock(void) {
+	pthread_mutex_unlock(&registry.persons_lock);
+}
+
+static inline void registry_machines_lock(void) {
+	pthread_mutex_lock(&registry.machines_lock);
+}
+
+static inline void registry_machines_unlock(void) {
+	pthread_mutex_unlock(&registry.machines_lock);
+}
+
+static inline void registry_urls_lock(void) {
+	pthread_mutex_lock(&registry.urls_lock);
+}
+
+static inline void registry_urls_unlock(void) {
+	pthread_mutex_unlock(&registry.urls_lock);
+}
+
+// ideally, we should not lock the whole registry for
+// updating a person's urls.
+// however, to save the memory required for keeping a
+// mutex (40 bytes) per person, we do...
+static inline void registry_person_urls_lock(PERSON *p) {
+	(void)p;
+	pthread_mutex_lock(&registry.person_urls_lock);
+}
+
+static inline void registry_person_urls_unlock(PERSON *p) {
+	(void)p;
+	pthread_mutex_unlock(&registry.person_urls_lock);
+}
+
+// ideally, we should not lock the whole registry for
+// updating a machine's urls.
+// however, to save the memory required for keeping a
+// mutex (40 bytes) per machine, we do...
+static inline void registry_machine_urls_lock(MACHINE *m) {
+	(void)m;
+	pthread_mutex_lock(&registry.machine_urls_lock);
+}
+
+static inline void registry_machine_urls_unlock(MACHINE *m) {
+	(void)m;
+	pthread_mutex_unlock(&registry.machine_urls_lock);
+}
+
+static inline void registry_log_lock(void) {
+	pthread_mutex_lock(&registry.log_lock);
+}
+
+static inline void registry_log_unlock(void) {
+	pthread_mutex_unlock(&registry.log_lock);
+}
+
+
+// ----------------------------------------------------------------------------
+// common functions
+
+// parse a GUID and re-generated to be always lower case
+// this is used as a protection against the variations of GUIDs
+static inline int registry_regenerate_guid(const char *guid, char *result) {
+	uuid_t uuid;
+	if(unlikely(uuid_parse(guid, uuid) == -1)) {
+		info("Registry: GUID '%s' is not a valid GUID.", guid);
+		return -1;
+	}
+	else {
+		uuid_unparse_lower(uuid, result);
+
+#ifdef NETDATA_INTERNAL_CHECKS
+		if(strcmp(guid, result))
+			info("Registry: source GUID '%s' and re-generated GUID '%s' differ!", guid, result);
+#endif /* NETDATA_INTERNAL_CHECKS */
+	}
+
+	return 0;
+}
+
+// make sure the names of the machines / URLs do not contain any tabs
+// (which are used as our separator in the database files)
+// and are properly trimmed (before and after)
+static inline char *registry_fix_machine_name(char *name, size_t *len) {
+	char *s = name?name:"";
+
+	// skip leading spaces
+	while(*s && isspace(*s)) s++;
+
+	// make sure all spaces are a SPACE
+	char *t = s;
+	while(*t) {
+		if(unlikely(isspace(*t)))
+			*t = ' ';
+
+		t++;
+	}
+
+	// remove trailing spaces
+	while(--t >= s) {
+		if(*t == ' ')
+			*t = '\0';
+		else
+			break;
+	}
+	t++;
+
+	if(likely(len))
+		*len = (t - s);
+
+	return s;
+}
+
+static inline char *registry_fix_url(char *url, size_t *len) {
+	return registry_fix_machine_name(url, len);
+}
+
+
+// ----------------------------------------------------------------------------
+// forward definition of functions
+
+extern PERSON *registry_request_access(char *person_guid, char *machine_guid, char *url, char *name, time_t when);
+extern PERSON *registry_request_delete(char *person_guid, char *machine_guid, char *url, char *delete_url, time_t when);
+
+
+// ----------------------------------------------------------------------------
+// URL
+
+static inline URL *registry_url_allocate_nolock(const char *url, size_t urllen) {
+	// protection from too big URLs
+	if(urllen > registry.max_url_length)
+		urllen = registry.max_url_length;
+
+	debug(D_REGISTRY, "Registry: registry_url_allocate_nolock('%s'): allocating %zu bytes", url, sizeof(URL) + urllen);
+	URL *u = malloc(sizeof(URL) + urllen);
+	if(!u) fatal("Cannot allocate %zu bytes for URL '%s'", sizeof(URL) + urllen);
+
+	// a simple strcpy() should do the job
+	// but I prefer to be safe, since the caller specified urllen
+	strncpyz(u->url, url, urllen);
+
+	u->len = urllen;
+	u->links = 0;
+
+	registry.urls_memory += sizeof(URL) + urllen;
+
+	debug(D_REGISTRY, "Registry: registry_url_allocate_nolock('%s'): indexing it", url);
+	dictionary_set(registry.urls, u->url, u, sizeof(URL));
+
+	return u;
+}
+
+static inline URL *registry_url_get(const char *url, size_t urllen) {
+	debug(D_REGISTRY, "Registry: registry_url_get('%s')", url);
+
+	registry_urls_lock();
+
+	URL *u = dictionary_get(registry.urls, url);
+	if(!u) {
+		u = registry_url_allocate_nolock(url, urllen);
+		registry.urls_count++;
+	}
+
+	registry_urls_unlock();
+
+	return u;
+}
+
+static inline void registry_url_link_nolock(URL *u) {
+	u->links++;
+	debug(D_REGISTRY, "Registry: registry_url_link_nolock('%s'): URL has now %u links", u->url, u->links);
+}
+
+static inline void registry_url_unlink_nolock(URL *u) {
+	u->links--;
+	if(!u->links) {
+		debug(D_REGISTRY, "Registry: registry_url_unlink_nolock('%s'): No more links for this URL", u->url);
+		dictionary_del(registry.urls, u->url);
+		free(u);
+	}
+	else
+		debug(D_REGISTRY, "Registry: registry_url_unlink_nolock('%s'): URL has %u links left", u->url, u->links);
+}
+
+
+// ----------------------------------------------------------------------------
+// MACHINE
+
+static inline MACHINE *registry_machine_find(const char *machine_guid) {
+	debug(D_REGISTRY, "Registry: registry_machine_find('%s')", machine_guid);
+	return dictionary_get(registry.machines, machine_guid);
+}
+
+static inline MACHINE_URL *registry_machine_url_allocate(MACHINE *m, URL *u, time_t when) {
+	debug(D_REGISTRY, "registry_machine_link_to_url('%s', '%s'): allocating %zu bytes", m->guid, u->url, sizeof(MACHINE_URL));
+
+	MACHINE_URL *mu = malloc(sizeof(MACHINE_URL));
+	if(!mu) fatal("registry_machine_link_to_url('%s', '%s'): cannot allocate %zu bytes.", m->guid, u->url, sizeof(MACHINE_URL));
+
+	// mu->persons = dictionary_create(DICTIONARY_FLAGS);
+	// dictionary_set(mu->persons, p->guid, p, sizeof(PERSON));
+
+	mu->first_t = mu->last_t = when;
+	mu->usages = 1;
+	mu->url = u;
+	mu->flags = REGISTRY_URL_FLAGS_DEFAULT;
+
+	registry.machines_urls_memory += sizeof(MACHINE_URL);
+
+	debug(D_REGISTRY, "registry_machine_link_to_url('%s', '%s'): indexing URL in machine", m->guid, u->url);
+	dictionary_set(m->urls, u->url, mu, sizeof(MACHINE_URL));
+	registry_url_link_nolock(u);
+
+	return mu;
+}
+
+static inline MACHINE *registry_machine_allocate(const char *machine_guid, time_t when) {
+	debug(D_REGISTRY, "Registry: registry_machine_allocate('%s'): creating new machine, sizeof(MACHINE)=%zu", machine_guid, sizeof(MACHINE));
+
+	MACHINE *m = malloc(sizeof(MACHINE));
+	if(!m) fatal("Registry: cannot allocate memory for new machine '%s'", machine_guid);
+
+	strncpyz(m->guid, machine_guid, 36);
+
+	debug(D_REGISTRY, "Registry: registry_machine_allocate('%s'): creating dictionary of urls", machine_guid);
+	m->urls = dictionary_create(DICTIONARY_FLAGS);
+
+	m->first_t = m->last_t = when;
+	m->usages = 0;
+
+	registry.machines_memory += sizeof(MACHINE);
+
+	registry.machines_count++;
+	dictionary_set(registry.machines, m->guid, m, sizeof(MACHINE));
+
+	return m;
+}
+
+// 1. validate machine GUID
+// 2. if it is valid, find it or create it and return it
+// 3. if it is not valid, return NULL
+static inline MACHINE *registry_machine_get(const char *machine_guid, time_t when) {
+	MACHINE *m = NULL;
+
+	registry_machines_lock();
+
+	if(likely(machine_guid && *machine_guid)) {
+		// validate it is a GUID
+		char buf[36 + 1];
+		if(unlikely(registry_regenerate_guid(machine_guid, buf) == -1))
+			info("Registry: machine guid '%s' is not a valid guid. Ignoring it.", machine_guid);
+		else {
+			machine_guid = buf;
+			m = registry_machine_find(machine_guid);
+			if(!m) m = registry_machine_allocate(machine_guid, when);
+		}
+	}
+
+	registry_machines_unlock();
+
+	return m;
+}
+
+
+// ----------------------------------------------------------------------------
+// PERSON
+
+static inline PERSON *registry_person_find(const char *person_guid) {
+	debug(D_REGISTRY, "Registry: registry_person_find('%s')", person_guid);
+	return dictionary_get(registry.persons, person_guid);
+}
+
+static inline PERSON_URL *registry_person_url_allocate(PERSON *p, MACHINE *m, URL *u, char *name, size_t namelen, time_t when) {
+	// protection from too big names
+	if(namelen > registry.max_name_length)
+		namelen = registry.max_name_length;
+
+	debug(D_REGISTRY, "registry_person_url_allocate('%s', '%s', '%s'): allocating %zu bytes", p->guid, m->guid, u->url,
+		  sizeof(PERSON_URL) + namelen);
+
+	PERSON_URL *pu = malloc(sizeof(PERSON_URL) + namelen);
+	if(!pu) fatal("registry_person_url_allocate('%s', '%s', '%s'): cannot allocate %zu bytes.", p->guid, m->guid, u->url, sizeof(PERSON_URL) + namelen);
+
+	// a simple strcpy() should do the job
+	// but I prefer to be safe, since the caller specified urllen
+	strncpyz(pu->name, name, namelen);
+
+	pu->machine = m;
+	pu->first_t = pu->last_t = when;
+	pu->usages = 1;
+	pu->url = u;
+	pu->flags = REGISTRY_URL_FLAGS_DEFAULT;
+	m->links++;
+
+	registry.persons_urls_memory += sizeof(PERSON_URL) + namelen;
+
+	debug(D_REGISTRY, "registry_person_url_allocate('%s', '%s', '%s'): indexing URL in person", p->guid, m->guid, u->url);
+	dictionary_set(p->urls, u->url, pu, sizeof(PERSON_URL));
+	registry_url_link_nolock(u);
+
+	return pu;
+}
+
+static inline PERSON_URL *registry_person_url_reallocate(PERSON *p, MACHINE *m, URL *u, char *name, size_t namelen, time_t when, PERSON_URL *pu) {
+	// this function is needed to change the name of a PERSON_URL
+
+	debug(D_REGISTRY, "registry_person_url_reallocate('%s', '%s', '%s'): allocating %zu bytes", p->guid, m->guid, u->url,
+		  sizeof(PERSON_URL) + namelen);
+
+	PERSON_URL *tpu = registry_person_url_allocate(p, m, u, name, namelen, when);
+	tpu->first_t = pu->first_t;
+	tpu->last_t = pu->last_t;
+	tpu->usages = pu->usages;
+
+	// ok, these are a hack - since the registry_person_url_allocate() is
+	// adding these, we have to subtract them
+	tpu->machine->links--;
+	registry.persons_urls_memory -= sizeof(PERSON_URL) + strlen(pu->name);
+	registry_url_unlink_nolock(u);
+
+	free(pu);
+
+	return tpu;
+}
+
+static inline PERSON *registry_person_allocate(const char *person_guid, time_t when) {
+	PERSON *p = NULL;
+
+	debug(D_REGISTRY, "Registry: registry_person_allocate('%s'): allocating new person, sizeof(PERSON)=%zu", (person_guid)?person_guid:"", sizeof(PERSON));
+
+	p = malloc(sizeof(PERSON));
+	if(!p) fatal("Registry: cannot allocate memory for new person.");
+
+	if(!person_guid) {
+		for (; ;) {
+			uuid_t uuid;
+			uuid_generate(uuid);
+			uuid_unparse_lower(uuid, p->guid);
+
+			debug(D_REGISTRY, "Registry: Checking if the generated person guid '%s' is unique", p->guid);
+			if (!dictionary_get(registry.persons, p->guid)) {
+				debug(D_REGISTRY, "Registry: generated person guid '%s' is unique", p->guid);
+				break;
+			}
+			else
+				info("Registry: generated person guid '%s' found in the registry. Retrying...", p->guid);
+		}
+	}
+	else
+		strncpyz(p->guid, person_guid, 36);
+
+	debug(D_REGISTRY, "Registry: registry_person_allocate('%s'): creating dictionary of urls", p->guid);
+	p->urls = dictionary_create(DICTIONARY_FLAGS);
+
+	p->first_t = p->last_t = when;
+	p->usages = 0;
+
+	registry.persons_memory += sizeof(PERSON);
+
+	registry.persons_count++;
+	dictionary_set(registry.persons, p->guid, p, sizeof(PERSON));
+
+	return p;
+}
+
+
+// 1. validate person GUID
+// 2. if it is valid, find it
+// 3. if it is not valid, create a new one
+// 4. return it
+static inline PERSON *registry_person_get(const char *person_guid, time_t when) {
+	PERSON *p = NULL;
+
+	registry_persons_lock();
+
+	if(person_guid && *person_guid) {
+		char buf[36 + 1];
+		// validate it is a GUID
+		if(unlikely(registry_regenerate_guid(person_guid, buf) == -1))
+			info("Registry: person guid '%s' is not a valid guid. Ignoring it.", person_guid);
+		else {
+			person_guid = buf;
+			p = registry_person_find(person_guid);
+			if(!p) person_guid = NULL;
+		}
+	}
+
+	if(!p) p = registry_person_allocate(NULL, when);
+
+	registry_persons_unlock();
+
+	return p;
+}
+
+// ----------------------------------------------------------------------------
+// LINKING OF OBJECTS
+
+static inline PERSON_URL *registry_person_link_to_url(PERSON *p, MACHINE *m, URL *u, char *name, size_t namelen, time_t when) {
+	debug(D_REGISTRY, "registry_person_link_to_url('%s', '%s', '%s'): searching for URL in person", p->guid, m->guid, u->url);
+
+	registry_person_urls_lock(p);
+
+	PERSON_URL *pu = dictionary_get(p->urls, u->url);
+	if(!pu) {
+		debug(D_REGISTRY, "registry_person_link_to_url('%s', '%s', '%s'): not found", p->guid, m->guid, u->url);
+		pu = registry_person_url_allocate(p, m, u, name, namelen, when);
+		registry.persons_urls_count++;
+	}
+	else {
+		debug(D_REGISTRY, "registry_person_link_to_url('%s', '%s', '%s'): found", p->guid, m->guid, u->url);
+		pu->usages++;
+		if(likely(pu->last_t < when)) pu->last_t = when;
+
+		if(pu->machine != m) {
+			MACHINE_URL *mu = dictionary_get(pu->machine->urls, u->url);
+			if(mu) {
+				info("registry_person_link_to_url('%s', '%s', '%s'): URL switched machines (old was '%s') - expiring it from previous machine.",
+					 p->guid, m->guid, u->url, pu->machine->guid);
+				mu->flags |= REGISTRY_URL_FLAGS_EXPIRED;
+			}
+			else {
+				info("registry_person_link_to_url('%s', '%s', '%s'): URL switched machines (old was '%s') - but the URL is not linked to the old machine.",
+					 p->guid, m->guid, u->url, pu->machine->guid);
+			}
+
+			pu->machine->links--;
+			pu->machine = m;
+		}
+
+		if(strcmp(pu->name, name)) {
+			// the name of the PERSON_URL has changed !
+			pu = registry_person_url_reallocate(p, m, u, name, namelen, when, pu);
+		}
+	}
+
+	p->usages++;
+	if(likely(p->last_t < when)) p->last_t = when;
+
+	if(pu->flags & REGISTRY_URL_FLAGS_EXPIRED) {
+		info("registry_person_link_to_url('%s', '%s', '%s'): accessing an expired URL. Re-enabling URL.", p->guid, m->guid, u->url);
+		pu->flags &= ~REGISTRY_URL_FLAGS_EXPIRED;
+	}
+
+	registry_person_urls_unlock(p);
+
+	return pu;
+}
+
+static inline MACHINE_URL *registry_machine_link_to_url(PERSON *p, MACHINE *m, URL *u, time_t when) {
+	debug(D_REGISTRY, "registry_machine_link_to_url('%s', '%s', '%s'): searching for URL in machine", p->guid, m->guid, u->url);
+
+	registry_machine_urls_lock(m);
+
+	MACHINE_URL *mu = dictionary_get(m->urls, u->url);
+	if(!mu) {
+		debug(D_REGISTRY, "registry_machine_link_to_url('%s', '%s', '%s'): not found", p->guid, m->guid, u->url);
+		mu = registry_machine_url_allocate(m, u, when);
+		registry.machines_urls_count++;
+	}
+	else {
+		debug(D_REGISTRY, "registry_machine_link_to_url('%s', '%s', '%s'): found", p->guid, m->guid, u->url);
+		mu->usages++;
+		if(likely(mu->last_t < when)) mu->last_t = when;
+	}
+
+	//debug(D_REGISTRY, "registry_machine_link_to_url('%s', '%s', '%s'): indexing person in machine", p->guid, m->guid, u->url);
+	//dictionary_set(mu->persons, p->guid, p, sizeof(PERSON));
+
+	m->usages++;
+	if(likely(m->last_t < when)) m->last_t = when;
+
+	if(mu->flags & REGISTRY_URL_FLAGS_EXPIRED) {
+		info("registry_machine_link_to_url('%s', '%s', '%s'): accessing an expired URL.", p->guid, m->guid, u->url);
+		mu->flags &= ~REGISTRY_URL_FLAGS_EXPIRED;
+	}
+
+	registry_machine_urls_unlock(m);
+
+	return mu;
+}
+
+// ----------------------------------------------------------------------------
+// REGISTRY LOG LOAD/SAVE
+
+static inline int registry_should_save_db(void) {
+	debug(D_REGISTRY, "log entries %llu, max %llu", registry.log_count, registry.save_registry_every_entries);
+	return registry.log_count > registry.save_registry_every_entries;
+}
+
+static inline void registry_log(const char action, PERSON *p, MACHINE *m, URL *u, char *name) {
+	if(likely(registry.log_fp)) {
+		// we lock only if the file is open
+		// to allow replaying the log at registry_log_load()
+		registry_log_lock();
+
+		if(unlikely(fprintf(registry.log_fp, "%c\t%08x\t%s\t%s\t%s\t%s\n",
+				action,
+				p->last_t,
+				p->guid,
+				m->guid,
+				name,
+				u->url) < 0))
+			error("Registry: failed to save log. Registry data may be lost in case of abnormal restart.");
+
+		// we increase the counter even on failures
+		// so that the registry will be saved periodically
+		registry.log_count++;
+
+		registry_log_unlock();
+
+		// this must be outside the log_lock(), or a deadlock will happen.
+		// registry_save() checks the same inside the log_lock, so only
+		// one thread will save the db
+		if(unlikely(registry_should_save_db()))
+			registry_save();
+	}
+}
+
+static inline int registry_log_open_nolock(void) {
+	if(registry.log_fp)
+		fclose(registry.log_fp);
+
+	registry.log_fp = fopen(registry.log_filename, "a");
+
+	if(registry.log_fp) {
+		if (setvbuf(registry.log_fp, NULL, _IOLBF, 0) != 0)
+			error("Cannot set line buffering on registry log file.");
+		return 0;
+	}
+
+	error("Cannot open registry log file '%s'. Registry data will be lost in case of netdata or server crash.", registry.log_filename);
+	return -1;
+}
+
+static inline void registry_log_close_nolock(void) {
+	if(registry.log_fp) {
+		fclose(registry.log_fp);
+		registry.log_fp = NULL;
+	}
+}
+
+static inline void registry_log_recreate_nolock(void) {
+	if(registry.log_fp != NULL) {
+		registry_log_close_nolock();
+
+		// open it with truncate
+		registry.log_fp = fopen(registry.log_filename, "w");
+		if(registry.log_fp) fclose(registry.log_fp);
+		else error("Cannot truncate registry log '%s'", registry.log_filename);
+
+		registry.log_fp = NULL;
+
+		registry_log_open_nolock();
+	}
+}
+
+int registry_log_load(void) {
+	char *s, buf[4096 + 1];
+	size_t line = -1;
+
+	// closing the log is required here
+	// otherwise we will append to it the values we read
+	registry_log_close_nolock();
+
+	debug(D_REGISTRY, "Registry: loading active db from: %s", registry.log_filename);
+	FILE *fp = fopen(registry.log_filename, "r");
+	if(!fp)
+		error("Registry: cannot open registry file: %s", registry.log_filename);
+	else {
+		line = 0;
+		size_t len = 0;
+		while ((s = fgets_trim_len(buf, 4096, fp, &len))) {
+			line++;
+
+			switch (s[0]) {
+				case 'A': // accesses
+				case 'D': // deletes
+
+					// verify it is valid
+					if (unlikely(len < 85 || s[1] != '\t' || s[10] != '\t' || s[47] != '\t' || s[84] != '\t')) {
+						error("Registry: log line %u is wrong (len = %zu).", line, len);
+						continue;
+					}
+					s[1] = s[10] = s[47] = s[84] = '\0';
+
+					// get the variables
+					time_t when = strtoul(&s[2], NULL, 16);
+					char *person_guid = &s[11];
+					char *machine_guid = &s[48];
+					char *name = &s[85];
+
+					// skip the name to find the url
+					char *url = name;
+					while(*url && *url != '\t') url++;
+					if(!*url) {
+						error("Registry: log line %u does not have a url.", line);
+						continue;
+					}
+					*url++ = '\0';
+
+					// make sure the person exists
+					// without this, a new person guid will be created
+					PERSON *p = registry_person_find(person_guid);
+					if(!p) p = registry_person_allocate(person_guid, when);
+
+					if(s[0] == 'A')
+						registry_request_access(p->guid, machine_guid, url, name, when);
+					else
+						registry_request_delete(p->guid, machine_guid, url, name, when);
+
+					break;
+
+				default:
+					error("Registry: ignoring line %zu of filename '%s': %s.", line, registry.log_filename, s);
+					break;
+			}
+		}
+	}
+
+	// open the log again
+	registry_log_open_nolock();
+
+	return line;
+}
+
+
+// ----------------------------------------------------------------------------
+// REGISTRY REQUESTS
+
+PERSON *registry_request_access(char *person_guid, char *machine_guid, char *url, char *name, time_t when) {
+	debug(D_REGISTRY, "registry_request_access('%s', '%s', '%s'): NEW REQUEST", (person_guid)?person_guid:"", machine_guid, url);
+
+	MACHINE *m = registry_machine_get(machine_guid, when);
+	if(!m) return NULL;
+
+	// make sure the name is valid
+	size_t namelen;
+	name = registry_fix_machine_name(name, &namelen);
+
+	size_t urllen;
+	url = registry_fix_url(url, &urllen);
+
+	URL *u = registry_url_get(url, urllen);
+	PERSON *p = registry_person_get(person_guid, when);
+
+	registry_person_link_to_url(p, m, u, name, namelen, when);
+	registry_machine_link_to_url(p, m, u, when);
+
+	registry_log('A', p, m, u, name);
+
+	registry.usages_count++;
+	return p;
+}
+
+// verify the person, the machine and the URL exist in our DB
+PERSON_URL *registry_verify_request(char *person_guid, char *machine_guid, char *url, PERSON **pp, MACHINE **mm) {
+	char pbuf[36 + 1], mbuf[36 + 1];
+
+	if(!person_guid || !*person_guid || !machine_guid || !*machine_guid || !url || !*url) {
+		info("Registry Request Verification: invalid request! person: '%s', machine '%s', url '%s'", person_guid?person_guid:"UNSET", machine_guid?machine_guid:"UNSET", url?url:"UNSET");
+		return NULL;
+	}
+
+	// normalize the url
+	url = registry_fix_url(url, NULL);
+
+	// make sure the person GUID is valid
+	if(registry_regenerate_guid(person_guid, pbuf) == -1) {
+		info("Registry Request Verification: invalid person GUID, person: '%s', machine '%s', url '%s'", person_guid, machine_guid, url);
+		return NULL;
+	}
+	person_guid = pbuf;
+
+	// make sure the machine GUID is valid
+	if(registry_regenerate_guid(machine_guid, mbuf) == -1) {
+		info("Registry Request Verification: invalid machine GUID, person: '%s', machine '%s', url '%s'", person_guid, machine_guid, url);
+		return NULL;
+	}
+	machine_guid = mbuf;
+
+	// make sure the machine exists
+	MACHINE *m = registry_machine_find(machine_guid);
+	if(!m) {
+		info("Registry Request Verification: machine not found, person: '%s', machine '%s', url '%s'", person_guid, machine_guid, url);
+		return NULL;
+	}
+	if(mm) *mm = m;
+
+	// make sure the person exist
+	PERSON *p = registry_person_find(person_guid);
+	if(!p) {
+		info("Registry Request Verification: person not found, person: '%s', machine '%s', url '%s'", person_guid, machine_guid, url);
+		return NULL;
+	}
+	if(pp) *pp = p;
+
+	PERSON_URL *pu = dictionary_get(p->urls, url);
+	if(!pu) {
+		info("Registry Request Verification: URL not found for person, person: '%s', machine '%s', url '%s'", person_guid, machine_guid, url);
+		return NULL;
+	}
+	return pu;
+}
+
+PERSON *registry_request_delete(char *person_guid, char *machine_guid, char *url, char *delete_url, time_t when) {
+	(void)when;
+
+	PERSON *p = NULL;
+	MACHINE *m = NULL;
+	PERSON_URL *pu = registry_verify_request(person_guid, machine_guid, url, &p, &m);
+	if(!pu || !p || !m) return NULL;
+
+	// normalize the url
+	delete_url = registry_fix_url(delete_url, NULL);
+
+	// make sure the user is not deleting the url it uses
+	if(!strcmp(delete_url, pu->url->url)) {
+		info("Registry Delete Request: delete URL is the one currently accessed, person: '%s', machine '%s', url '%s', delete url '%s'", p->guid, m->guid, pu->url->url, delete_url);
+		return NULL;
+	}
+
+	registry_person_urls_lock(p);
+
+	PERSON_URL *dpu = dictionary_get(p->urls, delete_url);
+	if(!dpu) {
+		info("Registry Delete Request: URL not found for person: '%s', machine '%s', url '%s', delete url '%s'", p->guid, m->guid, pu->url->url, delete_url);
+		registry_person_urls_unlock(p);
+		return NULL;
+	}
+
+	registry_log('D', p, m, pu->url, dpu->url->url);
+
+	dictionary_del(p->urls, dpu->url->url);
+	registry_url_unlink_nolock(dpu->url);
+	free(dpu);
+
+	registry_person_urls_unlock(p);
+	return p;
+}
+
+
+// a structure to pass to the dictionary_get_all() callback handler
+struct machine_request_callback_data {
+	MACHINE *find_this_machine;
+	PERSON_URL *result;
+};
+
+// the callback function
+// this will be run for every PERSON_URL of this PERSON
+int machine_request_callback(void *entry, void *data) {
+	PERSON_URL *mypu = (PERSON_URL *)entry;
+	struct machine_request_callback_data *myrdata = (struct machine_request_callback_data *)data;
+
+	if(mypu->machine == myrdata->find_this_machine) {
+		myrdata->result = mypu;
+		return -1; // this will also stop the walk through
+	}
+
+	return 0; // continue
+}
+
+MACHINE *registry_request_machine(char *person_guid, char *machine_guid, char *url, char *request_machine, time_t when) {
+	(void)when;
+
+	char mbuf[36 + 1];
+
+	PERSON *p = NULL;
+	MACHINE *m = NULL;
+	PERSON_URL *pu = registry_verify_request(person_guid, machine_guid, url, &p, &m);
+	if(!pu || !p || !m) return NULL;
+
+	// make sure the machine GUID is valid
+	if(registry_regenerate_guid(request_machine, mbuf) == -1) {
+		info("Registry Machine URLs request: invalid machine GUID, person: '%s', machine '%s', url '%s', request machine '%s'", p->guid, m->guid, pu->url->url, request_machine);
+		return NULL;
+	}
+	request_machine = mbuf;
+
+	// make sure the machine exists
+	m = registry_machine_find(request_machine);
+	if(!m) {
+		info("Registry Machine URLs request: machine not found, person: '%s', machine '%s', url '%s', request machine '%s'", p->guid, m->guid, pu->url->url, request_machine);
+		return NULL;
+	}
+
+	// Verify the user has in the past accessed this machine
+	// We will walk through the PERSON_URLs to find the machine
+	// linking to our machine
+
+	// a structure to pass to the dictionary_get_all() callback handler
+	struct machine_request_callback_data rdata = { m, NULL };
+
+	// request a walk through on the dictionary
+	// no need for locking here, the underlying dictionary has its own
+	dictionary_get_all(p->urls, machine_request_callback, &rdata);
+
+	if(rdata.result)
+		return m;
+
+	return NULL;
+}
+
+
+// ----------------------------------------------------------------------------
+// REGISTRY JSON generation
+
+#define REGISTRY_STATUS_OK "ok"
+#define REGISTRY_STATUS_FAILED "failed"
+#define REGISTRY_STATUS_DISABLED "disabled"
+
+static inline void registry_set_person_cookie(struct web_client *w, PERSON *p) {
+	char edate[100];
+	time_t et = time(NULL) + registry.persons_expiration;
+	struct tm etmbuf, *etm = gmtime_r(&et, &etmbuf);
+	strftime(edate, sizeof(edate), "%a, %d %b %Y %H:%M:%S %Z", etm);
+
+	snprintfz(w->cookie1, COOKIE_MAX, NETDATA_REGISTRY_COOKIE_NAME "=%s; Expires=%s", p->guid, edate);
+
+	if(registry.registry_domain && registry.registry_domain[0])
+		snprintfz(w->cookie2, COOKIE_MAX, NETDATA_REGISTRY_COOKIE_NAME "=%s; Domain=%s; Expires=%s", p->guid, registry.registry_domain, edate);
+}
+
+static inline void registry_json_header(struct web_client *w, const char *action, const char *status) {
+	w->response.data->contenttype = CT_APPLICATION_JSON;
+	buffer_flush(w->response.data);
+	buffer_sprintf(w->response.data, "{\n\t\"action\": \"%s\",\n\t\"status\": \"%s\",\n\t\"hostname\": \"%s\",\n\t\"machine_guid\": \"%s\"",
+				   action, status, registry.hostname, registry.machine_guid);
+}
+
+static inline void registry_json_footer(struct web_client *w) {
+	buffer_strcat(w->response.data, "\n}\n");
+}
+
+int registry_request_hello_json(struct web_client *w) {
+	registry_json_header(w, "hello", REGISTRY_STATUS_OK);
+
+	buffer_sprintf(w->response.data, ",\n\t\"registry\": \"%s\"",
+				   registry.registry_to_announce);
+
+	registry_json_footer(w);
+	return 200;
+}
+
+static inline int registry_json_disabled(struct web_client *w, const char *action) {
+	registry_json_header(w, action, REGISTRY_STATUS_DISABLED);
+
+	buffer_sprintf(w->response.data, ",\n\t\"registry\": \"%s\"",
+				   registry.registry_to_announce);
+
+	registry_json_footer(w);
+	return 200;
+}
+
+// structure used be the callbacks below
+struct registry_json_walk_person_urls_callback {
+	PERSON *p;
+	MACHINE *m;
+	struct web_client *w;
+	int count;
+};
+
+// callback for rendering PERSON_URLs
+static inline int registry_json_person_url_callback(void *entry, void *data) {
+	PERSON_URL *pu = (PERSON_URL *)entry;
+	struct registry_json_walk_person_urls_callback *c = (struct registry_json_walk_person_urls_callback *)data;
+	struct web_client *w = c->w;
+
+	if(unlikely(c->count++))
+		buffer_strcat(w->response.data, ",");
+
+	buffer_sprintf(w->response.data, "\n\t\t[ \"%s\", \"%s\", %u000, %u, \"%s\" ]",
+				   pu->machine->guid, pu->url->url, pu->last_t, pu->usages, pu->name);
+
+	return 1;
+}
+
+// callback for rendering MACHINE_URLs
+static inline int registry_json_machine_url_callback(void *entry, void *data) {
+	MACHINE_URL *mu = (MACHINE_URL *)entry;
+	struct registry_json_walk_person_urls_callback *c = (struct registry_json_walk_person_urls_callback *)data;
+	struct web_client *w = c->w;
+	MACHINE *m = c->m;
+
+	if(unlikely(c->count++))
+		buffer_strcat(w->response.data, ",");
+
+	buffer_sprintf(w->response.data, "\n\t\t[ \"%s\", \"%s\", %u000, %u ]",
+				   m->guid, mu->url->url, mu->last_t, mu->usages);
+
+	return 1;
+}
+
+
+// the main method for registering an access
+int registry_request_access_json(struct web_client *w, char *person_guid, char *machine_guid, char *url, char *name, time_t when) {
+	if(!registry.enabled)
+		return registry_json_disabled(w, "access");
+
+	PERSON *p = registry_request_access(person_guid, machine_guid, url, name, when);
+	if(!p) {
+		registry_json_header(w, "access", REGISTRY_STATUS_FAILED);
+		registry_json_footer(w);
+		return 412;
+	}
+
+	// set the cookie
+	registry_set_person_cookie(w, p);
+
+	// generate the response
+	registry_json_header(w, "access", REGISTRY_STATUS_OK);
+
+	buffer_sprintf(w->response.data, ",\n\t\"person_guid\": \"%s\",\n\t\"urls\": [", p->guid);
+	struct registry_json_walk_person_urls_callback c = { p, NULL, w, 0 };
+	dictionary_get_all(p->urls, registry_json_person_url_callback, &c);
+	buffer_strcat(w->response.data, "\n\t]\n");
+
+	registry_json_footer(w);
+	return 200;
+}
+
+// the main method for deleting a URL from a person
+int registry_request_delete_json(struct web_client *w, char *person_guid, char *machine_guid, char *url, char *delete_url, time_t when) {
+	if(!registry.enabled)
+		return registry_json_disabled(w, "delete");
+
+	PERSON *p = registry_request_delete(person_guid, machine_guid, url, delete_url, when);
+	if(!p) {
+		registry_json_header(w, "delete", REGISTRY_STATUS_FAILED);
+		registry_json_footer(w);
+		return 412;
+	}
+
+	// generate the response
+	registry_json_header(w, "delete", REGISTRY_STATUS_OK);
+	registry_json_footer(w);
+	return 200;
+}
+
+// the main method for searching the URLs of a netdata
+int registry_request_search_json(struct web_client *w, char *person_guid, char *machine_guid, char *url, char *request_machine, time_t when) {
+	if(!registry.enabled)
+		return registry_json_disabled(w, "search");
+
+	MACHINE *m = registry_request_machine(person_guid, machine_guid, url, request_machine, when);
+	if(!m) {
+		registry_json_header(w, "search", REGISTRY_STATUS_FAILED);
+		registry_json_footer(w);
+		return 404;
+	}
+
+	registry_json_header(w, "search", REGISTRY_STATUS_OK);
+
+	buffer_strcat(w->response.data, ",\n\t\"urls\": [");
+	struct registry_json_walk_person_urls_callback c = { NULL, m, w, 0 };
+	dictionary_get_all(m->urls, registry_json_machine_url_callback, &c);
+	buffer_strcat(w->response.data, "\n\t]\n");
+
+	registry_json_footer(w);
+	return 200;
+}
+
+// structure used be the callbacks below
+struct registry_person_url_callback_verify_machine_exists_data {
+	MACHINE *m;
+	int count;
+};
+
+int registry_person_url_callback_verify_machine_exists(void *entry, void *data) {
+	struct registry_person_url_callback_verify_machine_exists_data *d = (struct registry_person_url_callback_verify_machine_exists_data *)data;
+	PERSON_URL *pu = (PERSON_URL *)entry;
+	MACHINE *m = d->m;
+
+	if(pu->machine == m)
+		d->count++;
+
+	return 0;
+}
+
+// the main method for switching user identity
+int registry_request_switch_json(struct web_client *w, char *person_guid, char *machine_guid, char *url, char *new_person_guid, time_t when) {
+	(void)url;
+	(void)when;
+
+	if(!registry.enabled)
+		return registry_json_disabled(w, "switch");
+
+	PERSON *op = registry_person_find(person_guid);
+	if(!op) {
+		registry_json_header(w, "switch", REGISTRY_STATUS_FAILED);
+		registry_json_footer(w);
+		return 430;
+	}
+
+	PERSON *np = registry_person_find(new_person_guid);
+	if(!np) {
+		registry_json_header(w, "switch", REGISTRY_STATUS_FAILED);
+		registry_json_footer(w);
+		return 431;
+	}
+
+	MACHINE *m = registry_machine_find(machine_guid);
+	if(!m) {
+		registry_json_header(w, "switch", REGISTRY_STATUS_FAILED);
+		registry_json_footer(w);
+		return 432;
+	}
+
+	struct registry_person_url_callback_verify_machine_exists_data data = { m, 0 };
+
+	// verify the old person has access to this machine
+	dictionary_get_all(op->urls, registry_person_url_callback_verify_machine_exists, &data);
+	if(!data.count) {
+		registry_json_header(w, "switch", REGISTRY_STATUS_FAILED);
+		registry_json_footer(w);
+		return 433;
+	}
+
+	// verify the new person has access to this machine
+	data.count = 0;
+	dictionary_get_all(np->urls, registry_person_url_callback_verify_machine_exists, &data);
+	if(!data.count) {
+		registry_json_header(w, "switch", REGISTRY_STATUS_FAILED);
+		registry_json_footer(w);
+		return 434;
+	}
+
+	// set the cookie of the new person
+	// the user just switched identity
+	registry_set_person_cookie(w, np);
+
+	// generate the response
+	registry_json_header(w, "switch", REGISTRY_STATUS_OK);
+	buffer_sprintf(w->response.data, ",\n\t\"person_guid\": \"%s\"", np->guid);
+	registry_json_footer(w);
+	return 200;
+}
+
+
+// ----------------------------------------------------------------------------
+// REGISTRY THIS MACHINE UNIQUE ID
+
+char *registry_get_this_machine_guid(void) {
+	if(likely(registry.machine_guid[0]))
+		return registry.machine_guid;
+
+	// read it from disk
+	int fd = open(registry.machine_guid_filename, O_RDONLY);
+	if(fd != -1) {
+		char buf[36 + 1];
+		if(read(fd, buf, 36) != 36)
+			error("Failed to read machine GUID from '%s'", registry.machine_guid_filename);
+		else {
+			buf[36] = '\0';
+			if(registry_regenerate_guid(buf, registry.machine_guid) == -1) {
+				error("Failed to validate machine GUID '%s' from '%s'. Ignoring it - this might mean this netdata will appear as duplicate in the registry.",
+					  buf, registry.machine_guid_filename);
+
+				registry.machine_guid[0] = '\0';
+			}
+		}
+		close(fd);
+	}
+
+	// generate a new one?
+	if(!registry.machine_guid[0]) {
+		uuid_t uuid;
+
+		uuid_generate_time(uuid);
+		uuid_unparse_lower(uuid, registry.machine_guid);
+		registry.machine_guid[36] = '\0';
+
+		// save it
+		fd = open(registry.machine_guid_filename, O_WRONLY|O_CREAT|O_TRUNC, 444);
+		if(fd == -1)
+			fatal("Cannot create unique machine id file '%s'. Please fix this.", registry.machine_guid_filename);
+
+		if(write(fd, registry.machine_guid, 36) != 36)
+			fatal("Cannot write the unique machine id file '%s'. Please fix this.", registry.machine_guid_filename);
+
+		close(fd);
+	}
+
+	return registry.machine_guid;
+}
+
+
+// ----------------------------------------------------------------------------
+// REGISTRY LOAD/SAVE
+
+int registry_machine_save_url(void *entry, void *file) {
+	MACHINE_URL *mu = entry;
+	FILE *fp = file;
+
+	debug(D_REGISTRY, "Registry: registry_machine_save_url('%s')", mu->url->url);
+
+	int ret = fprintf(fp, "V\t%08x\t%08x\t%08x\t%02x\t%s\n",
+			mu->first_t,
+			mu->last_t,
+			mu->usages,
+			mu->flags,
+			mu->url->url
+	);
+
+	// error handling is done at registry_save()
+
+	return ret;
+}
+
+int registry_machine_save(void *entry, void *file) {
+	MACHINE *m = entry;
+	FILE *fp = file;
+
+	debug(D_REGISTRY, "Registry: registry_machine_save('%s')", m->guid);
+
+	int ret = fprintf(fp, "M\t%08x\t%08x\t%08x\t%s\n",
+			m->first_t,
+			m->last_t,
+			m->usages,
+			m->guid
+	);
+
+	if(ret >= 0) {
+		int ret2 = dictionary_get_all(m->urls, registry_machine_save_url, fp);
+		if(ret2 < 0) return ret2;
+		ret += ret2;
+	}
+
+	// error handling is done at registry_save()
+
+	return ret;
+}
+
+static inline int registry_person_save_url(void *entry, void *file) {
+	PERSON_URL *pu = entry;
+	FILE *fp = file;
+
+	debug(D_REGISTRY, "Registry: registry_person_save_url('%s')", pu->url->url);
+
+	int ret = fprintf(fp, "U\t%08x\t%08x\t%08x\t%02x\t%s\t%s\t%s\n",
+			pu->first_t,
+			pu->last_t,
+			pu->usages,
+			pu->flags,
+			pu->machine->guid,
+			pu->name,
+			pu->url->url
+	);
+
+	// error handling is done at registry_save()
+
+	return ret;
+}
+
+static inline int registry_person_save(void *entry, void *file) {
+	PERSON *p = entry;
+	FILE *fp = file;
+
+	debug(D_REGISTRY, "Registry: registry_person_save('%s')", p->guid);
+
+	int ret = fprintf(fp, "P\t%08x\t%08x\t%08x\t%s\n",
+			p->first_t,
+			p->last_t,
+			p->usages,
+			p->guid
+	);
+
+	if(ret >= 0) {
+		int ret2 = dictionary_get_all(p->urls, registry_person_save_url, fp);
+		if (ret2 < 0) return ret2;
+		ret += ret2;
+	}
+
+	// error handling is done at registry_save()
+
+	return ret;
+}
+
+int registry_save(void) {
+	if(!registry.enabled) return -1;
+
+	// make sure the log is not updated
+	registry_log_lock();
+
+	if(unlikely(!registry_should_save_db())) {
+		registry_log_unlock();
+		return -2;
+	}
+
+	char tmp_filename[FILENAME_MAX + 1];
+	char old_filename[FILENAME_MAX + 1];
+
+	snprintfz(old_filename, FILENAME_MAX, "%s.old", registry.db_filename);
+	snprintfz(tmp_filename, FILENAME_MAX, "%s.tmp", registry.db_filename);
+
+	debug(D_REGISTRY, "Registry: Creating file '%s'", tmp_filename);
+	FILE *fp = fopen(tmp_filename, "w");
+	if(!fp) {
+		error("Registry: Cannot create file: %s", tmp_filename);
+		registry_log_unlock();
+		return -1;
+	}
+
+	// dictionary_get_all() has its own locking, so this is safe to do
+
+	debug(D_REGISTRY, "Saving all machines");
+	int bytes1 = dictionary_get_all(registry.machines, registry_machine_save, fp);
+	if(bytes1 < 0) {
+		error("Registry: Cannot save registry machines - return value %d", bytes1);
+		fclose(fp);
+		registry_log_unlock();
+		return bytes1;
+	}
+	debug(D_REGISTRY, "Registry: saving machines took %d bytes", bytes1);
+
+	debug(D_REGISTRY, "Saving all persons");
+	int bytes2 = dictionary_get_all(registry.persons, registry_person_save, fp);
+	if(bytes2 < 0) {
+		error("Registry: Cannot save registry persons - return value %d", bytes2);
+		fclose(fp);
+		registry_log_unlock();
+		return bytes2;
+	}
+	debug(D_REGISTRY, "Registry: saving persons took %d bytes", bytes2);
+
+	// save the totals
+	fprintf(fp, "T\t%016llx\t%016llx\t%016llx\t%016llx\t%016llx\t%016llx\n",
+			registry.persons_count,
+			registry.machines_count,
+			registry.usages_count + 1, // this is required - it is lost on db rotation
+			registry.urls_count,
+			registry.persons_urls_count,
+			registry.machines_urls_count
+	);
+
+	fclose(fp);
+
+	errno = 0;
+
+	// remove the .old db
+	debug(D_REGISTRY, "Registry: Removing old db '%s'", old_filename);
+	if(unlink(old_filename) == -1 && errno != ENOENT)
+		error("Registry: cannot remove old registry file '%s'", old_filename);
+
+	// rename the db to .old
+	debug(D_REGISTRY, "Registry: Link current db '%s' to .old: '%s'", registry.db_filename, old_filename);
+	if(link(registry.db_filename, old_filename) == -1 && errno != ENOENT)
+		error("Registry: cannot move file '%s' to '%s'. Saving registry DB failed!", tmp_filename, registry.db_filename);
+
+	else {
+		// remove the database (it is saved in .old)
+		debug(D_REGISTRY, "Registry: removing db '%s'", registry.db_filename);
+		if (unlink(registry.db_filename) == -1 && errno != ENOENT)
+			error("Registry: cannot remove old registry file '%s'", registry.db_filename);
+
+		// move the .tmp to make it active
+		debug(D_REGISTRY, "Registry: linking tmp db '%s' to active db '%s'", tmp_filename, registry.db_filename);
+		if (link(tmp_filename, registry.db_filename) == -1) {
+			error("Registry: cannot move file '%s' to '%s'. Saving registry DB failed!", tmp_filename,
+				  registry.db_filename);
+
+			// move the .old back
+			debug(D_REGISTRY, "Registry: linking old db '%s' to active db '%s'", old_filename, registry.db_filename);
+			if(link(old_filename, registry.db_filename) == -1)
+				error("Registry: cannot move file '%s' to '%s'. Recovering the old registry DB failed!", old_filename, registry.db_filename);
+		}
+		else {
+			debug(D_REGISTRY, "Registry: removing tmp db '%s'", tmp_filename);
+			if(unlink(tmp_filename) == -1)
+				error("Registry: cannot remove tmp registry file '%s'", tmp_filename);
+
+			// it has been moved successfully
+			// discard the current registry log
+			registry_log_recreate_nolock();
+
+			registry.log_count = 0;
+		}
+	}
+
+	// continue operations
+	registry_log_unlock();
+
+	return -1;
+}
+
+static inline size_t registry_load(void) {
+	char *s, buf[4096 + 1];
+	PERSON *p = NULL;
+	MACHINE *m = NULL;
+	URL *u = NULL;
+	size_t line = 0;
+
+	debug(D_REGISTRY, "Registry: loading active db from: '%s'", registry.db_filename);
+	FILE *fp = fopen(registry.db_filename, "r");
+	if(!fp) {
+		error("Registry: cannot open registry file: '%s'", registry.db_filename);
+		return 0;
+	}
+
+	size_t len = 0;
+	buf[4096] = '\0';
+	while((s = fgets_trim_len(buf, 4096, fp, &len))) {
+		line++;
+
+		debug(D_REGISTRY, "Registry: read line %zu to length %zu: %s", line, len, s);
+		switch(*s) {
+			case 'T': // totals
+				if(unlikely(len != 103 || s[1] != '\t' || s[18] != '\t' || s[35] != '\t' || s[52] != '\t' || s[69] != '\t' || s[86] != '\t' || s[103] != '\0')) {
+					error("Registry totals line %u is wrong (len = %zu).", line, len);
+					continue;
+				}
+				registry.persons_count = strtoull(&s[2], NULL, 16);
+				registry.machines_count = strtoull(&s[19], NULL, 16);
+				registry.usages_count = strtoull(&s[36], NULL, 16);
+				registry.urls_count = strtoull(&s[53], NULL, 16);
+				registry.persons_urls_count = strtoull(&s[70], NULL, 16);
+				registry.machines_urls_count = strtoull(&s[87], NULL, 16);
+				break;
+
+			case 'P': // person
+				m = NULL;
+				// verify it is valid
+				if(unlikely(len != 65 || s[1] != '\t' || s[10] != '\t' || s[19] != '\t' || s[28] != '\t' || s[65] != '\0')) {
+					error("Registry person line %u is wrong (len = %zu).", line, len);
+					continue;
+				}
+
+				s[1] = s[10] = s[19] = s[28] = '\0';
+				p = registry_person_allocate(&s[29], strtoul(&s[2], NULL, 16));
+				p->last_t = strtoul(&s[11], NULL, 16);
+				p->usages = strtoul(&s[20], NULL, 16);
+				debug(D_REGISTRY, "Registry loaded person '%s', first: %u, last: %u, usages: %u", p->guid, p->first_t, p->last_t, p->usages);
+				break;
+
+			case 'M': // machine
+				p = NULL;
+				// verify it is valid
+				if(unlikely(len != 65 || s[1] != '\t' || s[10] != '\t' || s[19] != '\t' || s[28] != '\t' || s[65] != '\0')) {
+					error("Registry person line %u is wrong (len = %zu).", line, len);
+					continue;
+				}
+
+				s[1] = s[10] = s[19] = s[28] = '\0';
+				m = registry_machine_allocate(&s[29], strtoul(&s[2], NULL, 16));
+				m->last_t = strtoul(&s[11], NULL, 16);
+				m->usages = strtoul(&s[20], NULL, 16);
+				debug(D_REGISTRY, "Registry loaded machine '%s', first: %u, last: %u, usages: %u", m->guid, m->first_t, m->last_t, m->usages);
+				break;
+
+			case 'U': // person URL
+				if(unlikely(!p)) {
+					error("Registry: ignoring line %zu, no person loaded: %s", line, s);
+					continue;
+				}
+
+				// verify it is valid
+				if(len < 69 || s[1] != '\t' || s[10] != '\t' || s[19] != '\t' || s[28] != '\t' || s[31] != '\t' || s[68] != '\t') {
+					error("Registry person URL line %u is wrong (len = %zu).", line, len);
+					continue;
+				}
+
+				s[1] = s[10] = s[19] = s[28] = s[31] = s[68] = '\0';
+
+				// skip the name to find the url
+				char *url = &s[69];
+				while(*url && *url != '\t') url++;
+				if(!*url) {
+					error("Registry person URL line %u does not have a url.", line);
+					continue;
+				}
+				*url++ = '\0';
+
+				u = registry_url_allocate_nolock(url, strlen(url));
+
+				time_t first_t = strtoul(&s[2], NULL, 16);
+
+				m = registry_machine_find(&s[32]);
+				if(!m) m = registry_machine_allocate(&s[32], first_t);
+
+				PERSON_URL *pu = registry_person_url_allocate(p, m, u, &s[69], strlen(&s[69]), first_t);
+				pu->last_t = strtoul(&s[11], NULL, 16);
+				pu->usages = strtoul(&s[20], NULL, 16);
+				pu->flags = strtoul(&s[29], NULL, 16);
+				debug(D_REGISTRY, "Registry loaded person URL '%s' with name '%s' of machine '%s', first: %u, last: %u, usages: %u, flags: %02x", u->url, pu->name, m->guid, pu->first_t, pu->last_t, pu->usages, pu->flags);
+				break;
+
+			case 'V': // machine URL
+				if(unlikely(!m)) {
+					error("Registry: ignoring line %zu, no machine loaded: %s", line, s);
+					continue;
+				}
+
+				// verify it is valid
+				if(len < 32 || s[1] != '\t' || s[10] != '\t' || s[19] != '\t' || s[28] != '\t' || s[31] != '\t') {
+					error("Registry person URL line %u is wrong (len = %zu).", line, len);
+					continue;
+				}
+
+				s[1] = s[10] = s[19] = s[28] = s[31] = '\0';
+				u = registry_url_allocate_nolock(&s[32], strlen(&s[32]));
+
+				MACHINE_URL *mu = registry_machine_url_allocate(m, u, strtoul(&s[2], NULL, 16));
+				mu->last_t = strtoul(&s[11], NULL, 16);
+				mu->usages = strtoul(&s[20], NULL, 16);
+				mu->flags = strtoul(&s[29], NULL, 16);
+				debug(D_REGISTRY, "Registry loaded machine URL '%s', machine '%s', first: %u, last: %u, usages: %u, flags: %02x", u->url, m->guid, mu->first_t, mu->last_t, mu->usages, mu->flags);
+				break;
+
+			default:
+				error("Registry: ignoring line %zu of filename '%s': %s.", line, registry.db_filename, s);
+				break;
+		}
+	}
+	fclose(fp);
+
+	return line;
+}
+
+// ----------------------------------------------------------------------------
+// REGISTRY
+
+int registry_init(void) {
+	char filename[FILENAME_MAX + 1];
+
+	// registry enabled?
+	registry.enabled = config_get_boolean("registry", "enabled", 0);
+
+	// pathnames
+	registry.pathname = config_get("registry", "registry db directory", VARLIB_DIR "/registry");
+	if(mkdir(registry.pathname, 0755) == -1 && errno != EEXIST) {
+		error("Cannot create directory '%s'. Registry disabled.", registry.pathname);
+		registry.enabled = 0;
+		return -1;
+	}
+
+	// filenames
+	snprintfz(filename, FILENAME_MAX, "%s/netdata.public.unique.id", registry.pathname);
+	registry.machine_guid_filename = config_get("registry", "netdata unique id file", filename);
+	registry_get_this_machine_guid();
+
+	snprintfz(filename, FILENAME_MAX, "%s/registry.db", registry.pathname);
+	registry.db_filename = config_get("registry", "registry db file", filename);
+
+	snprintfz(filename, FILENAME_MAX, "%s/registry-log.db", registry.pathname);
+	registry.log_filename = config_get("registry", "registry log file", filename);
+
+	// configuration options
+	registry.save_registry_every_entries = config_get_number("registry", "registry save db every new entries", 1000000);
+	registry.persons_expiration = config_get_number("registry", "registry expire idle persons days", 365) * 86400;
+	registry.registry_domain = config_get("registry", "registry domain", "");
+	registry.registry_to_announce = config_get("registry", "registry to announce", "https://registry.my-netdata.io");
+	registry.hostname = config_get("registry", "registry hostname", config_get("global", "hostname", hostname));
+
+	registry.max_url_length = config_get_number("registry", "max URL length", 1024);
+	registry.max_name_length = config_get_number("registry", "max URL name length", 50);
+
+
+	// initialize entries counters
+	registry.persons_count = 0;
+	registry.machines_count = 0;
+	registry.usages_count = 0;
+	registry.urls_count = 0;
+	registry.persons_urls_count = 0;
+	registry.machines_urls_count = 0;
+
+	// initialize memory counters
+	registry.persons_memory = 0;
+	registry.machines_memory = 0;
+	registry.urls_memory = 0;
+	registry.persons_urls_memory = 0;
+	registry.machines_urls_memory = 0;
+
+	// initialize locks
+	pthread_mutex_init(&registry.persons_lock, NULL);
+	pthread_mutex_init(&registry.machines_lock, NULL);
+	pthread_mutex_init(&registry.urls_lock, NULL);
+	pthread_mutex_init(&registry.person_urls_lock, NULL);
+	pthread_mutex_init(&registry.machine_urls_lock, NULL);
+
+	// create dictionaries
+	registry.persons = dictionary_create(DICTIONARY_FLAGS);
+	registry.machines = dictionary_create(DICTIONARY_FLAGS);
+	registry.urls = dictionary_create(DICTIONARY_FLAGS);
+
+	// load the registry database
+	if(registry.enabled) {
+		registry_log_open_nolock();
+		registry_load();
+		registry_log_load();
+	}
+
+	return 0;
+}
+
+void registry_free(void) {
+	if(!registry.enabled) return;
+
+	// we need to destroy the dictionaries ourselves
+	// since the dictionaries use memory we allocated
+
+	while(registry.persons->values_index.root) {
+		PERSON *p = ((NAME_VALUE *)registry.persons->values_index.root)->value;
+
+		// fprintf(stderr, "\nPERSON: '%s', first: %u, last: %u, usages: %u\n", p->guid, p->first_t, p->last_t, p->usages);
+
+		while(p->urls->values_index.root) {
+			PERSON_URL *pu = ((NAME_VALUE *)p->urls->values_index.root)->value;
+
+			// fprintf(stderr, "\tURL: '%s', first: %u, last: %u, usages: %u, flags: 0x%02x\n", pu->url->url, pu->first_t, pu->last_t, pu->usages, pu->flags);
+
+			debug(D_REGISTRY, "Registry: deleting url '%s' from person '%s'", pu->url->url, p->guid);
+			dictionary_del(p->urls, pu->url->url);
+
+			debug(D_REGISTRY, "Registry: unlinking url '%s' from person", pu->url->url);
+			registry_url_unlink_nolock(pu->url);
+
+			debug(D_REGISTRY, "Registry: freeing person url");
+			free(pu);
+		}
+
+		debug(D_REGISTRY, "Registry: deleting person '%s' from persons registry", p->guid);
+		dictionary_del(registry.persons, p->guid);
+
+		debug(D_REGISTRY, "Registry: destroying URL dictionary of person '%s'", p->guid);
+		dictionary_destroy(p->urls);
+
+		debug(D_REGISTRY, "Registry: freeing person '%s'", p->guid);
+		free(p);
+	}
+
+	while(registry.machines->values_index.root) {
+		MACHINE *m = ((NAME_VALUE *)registry.machines->values_index.root)->value;
+
+		// fprintf(stderr, "\nMACHINE: '%s', first: %u, last: %u, usages: %u\n", m->guid, m->first_t, m->last_t, m->usages);
+
+		while(m->urls->values_index.root) {
+			MACHINE_URL *mu = ((NAME_VALUE *)m->urls->values_index.root)->value;
+
+			// fprintf(stderr, "\tURL: '%s', first: %u, last: %u, usages: %u, flags: 0x%02x\n", mu->url->url, mu->first_t, mu->last_t, mu->usages, mu->flags);
+
+			//debug(D_REGISTRY, "Registry: destroying persons dictionary from url '%s'", mu->url->url);
+			//dictionary_destroy(mu->persons);
+
+			debug(D_REGISTRY, "Registry: deleting url '%s' from person '%s'", mu->url->url, m->guid);
+			dictionary_del(m->urls, mu->url->url);
+
+			debug(D_REGISTRY, "Registry: unlinking url '%s' from machine", mu->url->url);
+			registry_url_unlink_nolock(mu->url);
+
+			debug(D_REGISTRY, "Registry: freeing machine url");
+			free(mu);
+		}
+
+		debug(D_REGISTRY, "Registry: deleting machine '%s' from machines registry", m->guid);
+		dictionary_del(registry.machines, m->guid);
+
+		debug(D_REGISTRY, "Registry: destroying URL dictionary of machine '%s'", m->guid);
+		dictionary_destroy(m->urls);
+
+		debug(D_REGISTRY, "Registry: freeing machine '%s'", m->guid);
+		free(m);
+	}
+
+	// and free the memory of remaining dictionary structures
+
+	debug(D_REGISTRY, "Registry: destroying persons dictionary");
+	dictionary_destroy(registry.persons);
+
+	debug(D_REGISTRY, "Registry: destroying machines dictionary");
+	dictionary_destroy(registry.machines);
+
+	debug(D_REGISTRY, "Registry: destroying urls dictionary");
+	dictionary_destroy(registry.urls);
+}
+
+// ----------------------------------------------------------------------------
+// STATISTICS
+
+void registry_statistics(void) {
+	if(!registry.enabled) return;
+
+	static RRDSET *sts = NULL, *stc = NULL, *stm = NULL;
+
+	if(!sts) sts = rrdset_find("netdata.registry_sessions");
+	if(!sts) {
+		sts = rrdset_create("netdata", "registry_sessions", NULL, "registry", NULL, "NetData Registry Sessions", "session", 131000, rrd_update_every, RRDSET_TYPE_LINE);
+
+		rrddim_add(sts, "sessions",  NULL,  1, 1, RRDDIM_ABSOLUTE);
+	}
+	else rrdset_next(sts);
+
+	rrddim_set(sts, "sessions", registry.usages_count);
+	rrdset_done(sts);
+
+	// ------------------------------------------------------------------------
+
+	if(!stc) stc = rrdset_find("netdata.registry_entries");
+	if(!stc) {
+		stc = rrdset_create("netdata", "registry_entries", NULL, "registry", NULL, "NetData Registry Entries", "entries", 131100, rrd_update_every, RRDSET_TYPE_LINE);
+
+		rrddim_add(stc, "persons",        NULL,  1, 1, RRDDIM_ABSOLUTE);
+		rrddim_add(stc, "machines",       NULL,  1, 1, RRDDIM_ABSOLUTE);
+		rrddim_add(stc, "urls",           NULL,  1, 1, RRDDIM_ABSOLUTE);
+		rrddim_add(stc, "persons_urls",   NULL,  1, 1, RRDDIM_ABSOLUTE);
+		rrddim_add(stc, "machines_urls",  NULL,  1, 1, RRDDIM_ABSOLUTE);
+	}
+	else rrdset_next(stc);
+
+	rrddim_set(stc, "persons",       registry.persons_count);
+	rrddim_set(stc, "machines",      registry.machines_count);
+	rrddim_set(stc, "urls",          registry.urls_count);
+	rrddim_set(stc, "persons_urls",  registry.persons_urls_count);
+	rrddim_set(stc, "machines_urls", registry.machines_urls_count);
+	rrdset_done(stc);
+
+	// ------------------------------------------------------------------------
+
+	if(!stm) stm = rrdset_find("netdata.registry_mem");
+	if(!stm) {
+		stm = rrdset_create("netdata", "registry_mem", NULL, "registry", NULL, "NetData Registry Memory", "KB", 131300, rrd_update_every, RRDSET_TYPE_STACKED);
+
+		rrddim_add(stm, "persons",        NULL,  1, 1024, RRDDIM_ABSOLUTE);
+		rrddim_add(stm, "machines",       NULL,  1, 1024, RRDDIM_ABSOLUTE);
+		rrddim_add(stm, "urls",           NULL,  1, 1024, RRDDIM_ABSOLUTE);
+		rrddim_add(stm, "persons_urls",   NULL,  1, 1024, RRDDIM_ABSOLUTE);
+		rrddim_add(stm, "machines_urls",  NULL,  1, 1024, RRDDIM_ABSOLUTE);
+	}
+	else rrdset_next(stm);
+
+	rrddim_set(stm, "persons",       registry.persons_memory + registry.persons_count * sizeof(NAME_VALUE) + sizeof(DICTIONARY));
+	rrddim_set(stm, "machines",      registry.machines_memory + registry.machines_count * sizeof(NAME_VALUE) + sizeof(DICTIONARY));
+	rrddim_set(stm, "urls",          registry.urls_memory + registry.urls_count * sizeof(NAME_VALUE) + sizeof(DICTIONARY));
+	rrddim_set(stm, "persons_urls",  registry.persons_urls_memory + registry.persons_count * sizeof(DICTIONARY) + registry.persons_urls_count * sizeof(NAME_VALUE));
+	rrddim_set(stm, "machines_urls", registry.machines_urls_memory + registry.machines_count * sizeof(DICTIONARY) + registry.machines_urls_count * sizeof(NAME_VALUE));
+	rrdset_done(stm);
+}