path: root/src/registry.c

#ifdef HAVE_CONFIG_H
#include <config.h>
#endif

#include <uuid/uuid.h>
#include <inttypes.h>
#include <stdlib.h>
#include <string.h>
#include <ctype.h>
#include <unistd.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <errno.h>
#include <fcntl.h>

#include "log.h"
#include "common.h"
#include "dictionary.h"
#include "appconfig.h"

#include "web_client.h"
#include "rrd.h"
#include "rrd2json.h"
#include "registry.h"


// ----------------------------------------------------------------------------
// TODO
//
// 1. the default tracking cookie expires in 1 year, but the persons are not
//    removed from the db - this means the database only grows - ideally the
//    database should be cleaned in registry_save() for both on-disk and
//    on-memory entries.
//
//    Cleanup:
//    i. Find all the PERSONs that have expired cookie
//    ii. For each of their PERSON_URLs:
//     - decrement the linked MACHINE links
//     - if the linked MACHINE has no other links, remove the linked MACHINE too
//     - remove the PERSON_URL
//
// 2. add protection to prevent abusing the registry by flooding it with
//    requests to fill the memory and crash it.
//
//    Possible protections:
//    - limit the number of URLs per person
//    - limit the number of URLs per machine
//    - limit the number of persons
//    - limit the number of machines
//    - [DONE] limit the size of URLs
//    - [DONE] limit the size of PERSON_URL names
//    - limit the number of requests that add data to the registry,
//      per client IP per hour



#define REGISTRY_URL_FLAGS_DEFAULT 0x00
#define REGISTRY_URL_FLAGS_EXPIRED 0x01

#define DICTIONARY_FLAGS DICTIONARY_FLAG_VALUE_LINK_DONT_CLONE | DICTIONARY_FLAG_NAME_LINK_DONT_CLONE

// ----------------------------------------------------------------------------
// COMMON structures

struct registry {
	int enabled;

	char machine_guid[36 + 1];

	// entries counters / statistics
	unsigned long long persons_count;
	unsigned long long machines_count;
	unsigned long long usages_count;
	unsigned long long urls_count;
	unsigned long long persons_urls_count;
	unsigned long long machines_urls_count;
	unsigned long long log_count;

	// memory counters / statistics
	unsigned long long persons_memory;
	unsigned long long machines_memory;
	unsigned long long urls_memory;
	unsigned long long persons_urls_memory;
	unsigned long long machines_urls_memory;

	// configuration
	unsigned long long save_registry_every_entries;
	char *registry_domain;
	char *hostname;
	char *registry_to_announce;
	time_t persons_expiration; // seconds to expire idle persons

	size_t max_url_length;
	size_t max_name_length;

	// file/path names
	char *pathname;
	char *db_filename;
	char *log_filename;
	char *machine_guid_filename;

	// open files
	FILE *log_fp;

	// the database
	DICTIONARY *persons; 	// dictionary of PERSON *, with key the PERSON.guid
	DICTIONARY *machines; 	// dictionary of MACHINE *, with key the MACHINE.guid
	DICTIONARY *urls; 		// dictionary of URL *, with key the URL.url

	// concurrency locking
	// we keep different locks for different things
	// so that many tasks can be completed in parallel
	pthread_mutex_t persons_lock;
	pthread_mutex_t machines_lock;
	pthread_mutex_t urls_lock;
	pthread_mutex_t person_urls_lock;
	pthread_mutex_t machine_urls_lock;
	pthread_mutex_t log_lock;
} registry;


// ----------------------------------------------------------------------------
// URL structures
// Save memory by de-duplicating URLs
// so instead of storing URLs all over the place
// we store them here and we keep pointers elsewhere

struct url {
	uint32_t links;	// the number of links to this URL - when none is left, we free it
	uint16_t len;	// the length of the URL in bytes
	char url[1];	// the URL - dynamically allocated to more size
};
typedef struct url URL;


// ----------------------------------------------------------------------------
// MACHINE structures

// For each MACHINE-URL pair we keep this
struct machine_url {
	URL *url;					// de-duplicated URL
//	DICTIONARY *persons;		// dictionary of PERSON *

	uint8_t flags;
	uint32_t first_t;			// the first time we saw this
	uint32_t last_t;			// the last time we saw this
	uint32_t usages;			// how many times this has been accessed
};
typedef struct machine_url MACHINE_URL;

// A machine
struct machine {
	char guid[36 + 1];			// the GUID

	uint32_t links;				// the number of PERSON_URLs linked to this machine

	DICTIONARY *urls; 			// MACHINE_URL *

	uint32_t first_t;			// the first time we saw this
	uint32_t last_t;			// the last time we saw this
	uint32_t usages;			// how many times this has been accessed
};
typedef struct machine MACHINE;


// ----------------------------------------------------------------------------
// PERSON structures

// for each PERSON-URL pair we keep this
struct person_url {
	URL *url;					// de-duplicated URL
	MACHINE *machine;			// link the MACHINE of this URL

	uint8_t flags;
	uint32_t first_t;			// the first time we saw this
	uint32_t last_t;			// the last time we saw this
	uint32_t usages;			// how many times this has been accessed

	char name[1];				// the name of the URL, as known by the user
								// dynamically allocated to fit properly
};
typedef struct person_url PERSON_URL;

// A person
struct person {
	char guid[36 + 1];			// the person GUID

	DICTIONARY *urls;			// dictionary of PERSON_URL *

	uint32_t first_t;			// the first time we saw this
	uint32_t last_t;			// the last time we saw this
	uint32_t usages;			// how many times this has been accessed
};
typedef struct person PERSON;


// ----------------------------------------------------------------------------
// REGISTRY concurrency locking

static inline void registry_persons_lock(void) {
	pthread_mutex_lock(&registry.persons_lock);
}

static inline void registry_persons_unlock(void) {
	pthread_mutex_unlock(&registry.persons_lock);
}

static inline void registry_machines_lock(void) {
	pthread_mutex_lock(&registry.machines_lock);
}

static inline void registry_machines_unlock(void) {
	pthread_mutex_unlock(&registry.machines_lock);
}

static inline void registry_urls_lock(void) {
	pthread_mutex_lock(&registry.urls_lock);
}

static inline void registry_urls_unlock(void) {
	pthread_mutex_unlock(&registry.urls_lock);
}

// ideally, we should not lock the whole registry for
// updating a person's urls.
// however, to save the memory required for keeping a
// mutex (40 bytes) per person, we do...
static inline void registry_person_urls_lock(PERSON *p) {
	(void)p;
	pthread_mutex_lock(&registry.person_urls_lock);
}

static inline void registry_person_urls_unlock(PERSON *p) {
	(void)p;
	pthread_mutex_unlock(&registry.person_urls_lock);
}

// ideally, we should not lock the whole registry for
// updating a machine's urls.
// however, to save the memory required for keeping a
// mutex (40 bytes) per machine, we do...
static inline void registry_machine_urls_lock(MACHINE *m) {
	(void)m;
	pthread_mutex_lock(&registry.machine_urls_lock);
}

static inline void registry_machine_urls_unlock(MACHINE *m) {
	(void)m;
	pthread_mutex_unlock(&registry.machine_urls_lock);
}

static inline void registry_log_lock(void) {
	pthread_mutex_lock(&registry.log_lock);
}

static inline void registry_log_unlock(void) {
	pthread_mutex_unlock(&registry.log_lock);
}


// ----------------------------------------------------------------------------
// common functions

// parse a GUID and re-generated to be always lower case
// this is used as a protection against the variations of GUIDs
static inline int registry_regenerate_guid(const char *guid, char *result) {
	uuid_t uuid;
	if(unlikely(uuid_parse(guid, uuid) == -1)) {
		info("Registry: GUID '%s' is not a valid GUID.", guid);
		return -1;
	}
	else {
		uuid_unparse_lower(uuid, result);

#ifdef NETDATA_INTERNAL_CHECKS
		if(strcmp(guid, result))
			info("Registry: source GUID '%s' and re-generated GUID '%s' differ!", guid, result);
#endif /* NETDATA_INTERNAL_CHECKS */
	}

	return 0;
}

// make sure the names of the machines / URLs do not contain any tabs
// (which are used as our separator in the database files)
// and are properly trimmed (before and after)
static inline char *registry_fix_machine_name(char *name, size_t *len) {
	char *s = name?name:"";

	// skip leading spaces
	while(*s && isspace(*s)) s++;

	// make sure all spaces are a SPACE
	char *t = s;
	while(*t) {
		if(unlikely(isspace(*t)))
			*t = ' ';

		t++;
	}

	// remove trailing spaces
	while(--t >= s) {
		if(*t == ' ')
			*t = '\0';
		else
			break;
	}
	t++;

	if(likely(len))
		*len = (t - s);

	return s;
}

static inline char *registry_fix_url(char *url, size_t *len) {
	return registry_fix_machine_name(url, len);
}


// ----------------------------------------------------------------------------
// forward definition of functions

extern PERSON *registry_request_access(char *person_guid, char *machine_guid, char *url, char *name, time_t when);
extern PERSON *registry_request_delete(char *person_guid, char *machine_guid, char *url, char *delete_url, time_t when);


// ----------------------------------------------------------------------------
// URL

static inline URL *registry_url_allocate_nolock(const char *url, size_t urllen) {
	// protection from too big URLs
	if(urllen > registry.max_url_length)
		urllen = registry.max_url_length;

	debug(D_REGISTRY, "Registry: registry_url_allocate_nolock('%s'): allocating %zu bytes", url, sizeof(URL) + urllen);
	URL *u = malloc(sizeof(URL) + urllen);
	if(!u) fatal("Cannot allocate %zu bytes for URL '%s'", sizeof(URL) + urllen);

	// a simple strcpy() should do the job
	// but I prefer to be safe, since the caller specified urllen
	strncpyz(u->url, url, urllen);

	u->len = urllen;
	u->links = 0;

	registry.urls_memory += sizeof(URL) + urllen;

	debug(D_REGISTRY, "Registry: registry_url_allocate_nolock('%s'): indexing it", url);
	dictionary_set(registry.urls, u->url, u, sizeof(URL));

	return u;
}

static inline URL *registry_url_get(const char *url, size_t urllen) {
	debug(D_REGISTRY, "Registry: registry_url_get('%s')", url);

	registry_urls_lock();

	URL *u = dictionary_get(registry.urls, url);
	if(!u) {
		u = registry_url_allocate_nolock(url, urllen);
		registry.urls_count++;
	}

	registry_urls_unlock();

	return u;
}

static inline void registry_url_link_nolock(URL *u) {
	u->links++;
	debug(D_REGISTRY, "Registry: registry_url_link_nolock('%s'): URL has now %u links", u->url, u->links);
}

static inline void registry_url_unlink_nolock(URL *u) {
	u->links--;
	if(!u->links) {
		debug(D_REGISTRY, "Registry: registry_url_unlink_nolock('%s'): No more links for this URL", u->url);
		dictionary_del(registry.urls, u->url);
		free(u);
	}
	else
		debug(D_REGISTRY, "Registry: registry_url_unlink_nolock('%s'): URL has %u links left", u->url, u->links);
}


// ----------------------------------------------------------------------------
// MACHINE

static inline MACHINE *registry_machine_find(const char *machine_guid) {
	debug(D_REGISTRY, "Registry: registry_machine_find('%s')", machine_guid);
	return dictionary_get(registry.machines, machine_guid);
}

static inline MACHINE_URL *registry_machine_url_allocate(MACHINE *m, URL *u, time_t when) {
	debug(D_REGISTRY, "registry_machine_link_to_url('%s', '%s'): allocating %zu bytes", m->guid, u->url, sizeof(MACHINE_URL));

	MACHINE_URL *mu = malloc(sizeof(MACHINE_URL));
	if(!mu) fatal("registry_machine_link_to_url('%s', '%s'): cannot allocate %zu bytes.", m->guid, u->url, sizeof(MACHINE_URL));

	// mu->persons = dictionary_create(DICTIONARY_FLAGS);
	// dictionary_set(mu->persons, p->guid, p, sizeof(PERSON));

	mu->first_t = mu->last_t = when;
	mu->usages = 1;
	mu->url = u;
	mu->flags = REGISTRY_URL_FLAGS_DEFAULT;

	registry.machines_urls_memory += sizeof(MACHINE_URL);

	debug(D_REGISTRY, "registry_machine_link_to_url('%s', '%s'): indexing URL in machine", m->guid, u->url);
	dictionary_set(m->urls, u->url, mu, sizeof(MACHINE_URL));
	registry_url_link_nolock(u);

	return mu;
}

static inline MACHINE *registry_machine_allocate(const char *machine_guid, time_t when) {
	debug(D_REGISTRY, "Registry: registry_machine_allocate('%s'): creating new machine, sizeof(MACHINE)=%zu", machine_guid, sizeof(MACHINE));

	MACHINE *m = malloc(sizeof(MACHINE));
	if(!m) fatal("Registry: cannot allocate memory for new machine '%s'", machine_guid);

	strncpyz(m->guid, machine_guid, 36);

	debug(D_REGISTRY, "Registry: registry_machine_allocate('%s'): creating dictionary of urls", machine_guid);
	m->urls = dictionary_create(DICTIONARY_FLAGS);

	m->first_t = m->last_t = when;
	m->usages = 0;

	registry.machines_memory += sizeof(MACHINE);

	registry.machines_count++;
	dictionary_set(registry.machines, m->guid, m, sizeof(MACHINE));

	return m;
}

// 1. validate machine GUID
// 2. if it is valid, find it or create it and return it
// 3. if it is not valid, return NULL
static inline MACHINE *registry_machine_get(const char *machine_guid, time_t when) {
	MACHINE *m = NULL;

	registry_machines_lock();

	if(likely(machine_guid && *machine_guid)) {
		// validate it is a GUID
		char buf[36 + 1];
		if(unlikely(registry_regenerate_guid(machine_guid, buf) == -1))
			info("Registry: machine guid '%s' is not a valid guid. Ignoring it.", machine_guid);
		else {
			machine_guid = buf;
			m = registry_machine_find(machine_guid);
			if(!m) m = registry_machine_allocate(machine_guid, when);
		}
	}

	registry_machines_unlock();

	return m;
}


// ----------------------------------------------------------------------------
// PERSON

static inline PERSON *registry_person_find(const char *person_guid) {
	debug(D_REGISTRY, "Registry: registry_person_find('%s')", person_guid);
	return dictionary_get(registry.persons, person_guid);
}

static inline PERSON_URL *registry_person_url_allocate(PERSON *p, MACHINE *m, URL *u, char *name, size_t namelen, time_t when) {
	// protection from too big names
	if(namelen > registry.max_name_length)
		namelen = registry.max_name_length;

	debug(D_REGISTRY, "registry_person_url_allocate('%s', '%s', '%s'): allocating %zu bytes", p->guid, m->guid, u->url,
		  sizeof(PERSON_URL) + namelen);

	PERSON_URL *pu = malloc(sizeof(PERSON_URL) + namelen);
	if(!pu) fatal("registry_person_url_allocate('%s', '%s', '%s'): cannot allocate %zu bytes.", p->guid, m->guid, u->url, sizeof(PERSON_URL) + namelen);

	// a simple strcpy() should do the job
	// but I prefer to be safe, since the caller specified urllen
	strncpyz(pu->name, name, namelen);

	pu->machine = m;
	pu->first_t = pu->last_t = when;
	pu->usages = 1;
	pu->url = u;
	pu->flags = REGISTRY_URL_FLAGS_DEFAULT;
	m->links++;

	registry.persons_urls_memory += sizeof(PERSON_URL) + namelen;

	debug(D_REGISTRY, "registry_person_url_allocate('%s', '%s', '%s'): indexing URL in person", p->guid, m->guid, u->url);
	dictionary_set(p->urls, u->url, pu, sizeof(PERSON_URL));
	registry_url_link_nolock(u);

	return pu;
}

static inline PERSON_URL *registry_person_url_reallocate(PERSON *p, MACHINE *m, URL *u, char *name, size_t namelen, time_t when, PERSON_URL *pu) {
	// this function is needed to change the name of a PERSON_URL

	debug(D_REGISTRY, "registry_person_url_reallocate('%s', '%s', '%s'): allocating %zu bytes", p->guid, m->guid, u->url,
		  sizeof(PERSON_URL) + namelen);

	PERSON_URL *tpu = registry_person_url_allocate(p, m, u, name, namelen, when);
	tpu->first_t = pu->first_t;
	tpu->last_t = pu->last_t;
	tpu->usages = pu->usages;

	// ok, these are a hack - since the registry_person_url_allocate() is
	// adding these, we have to subtract them
	tpu->machine->links--;
	registry.persons_urls_memory -= sizeof(PERSON_URL) + strlen(pu->name);
	registry_url_unlink_nolock(u);

	free(pu);

	return tpu;
}

static inline PERSON *registry_person_allocate(const char *person_guid, time_t when) {
	PERSON *p = NULL;

	debug(D_REGISTRY, "Registry: registry_person_allocate('%s'): allocating new person, sizeof(PERSON)=%zu", (person_guid)?person_guid:"", sizeof(PERSON));

	p = malloc(sizeof(PERSON));
	if(!p) fatal("Registry: cannot allocate memory for new person.");

	if(!person_guid) {
		for (; ;) {
			uuid_t uuid;
			uuid_generate(uuid);
			uuid_unparse_lower(uuid, p->guid);

			debug(D_REGISTRY, "Registry: Checking if the generated person guid '%s' is unique", p->guid);
			if (!dictionary_get(registry.persons, p->guid)) {
				debug(D_REGISTRY, "Registry: generated person guid '%s' is unique", p->guid);
				break;
			}
			else
				info("Registry: generated person guid '%s' found in the registry. Retrying...", p->guid);
		}
	}
	else
		strncpyz(p->guid, person_guid, 36);

	debug(D_REGISTRY, "Registry: registry_person_allocate('%s'): creating dictionary of urls", p->guid);
	p->urls = dictionary_create(DICTIONARY_FLAGS);

	p->first_t = p->last_t = when;
	p->usages = 0;

	registry.persons_memory += sizeof(PERSON);

	registry.persons_count++;
	dictionary_set(registry.persons, p->guid, p, sizeof(PERSON));

	return p;
}


// 1. validate person GUID
// 2. if it is valid, find it
// 3. if it is not valid, create a new one
// 4. return it
static inline PERSON *registry_person_get(const char *person_guid, time_t when) {
	PERSON *p = NULL;

	registry_persons_lock();

	if(person_guid && *person_guid) {
		char buf[36 + 1];
		// validate it is a GUID
		if(unlikely(registry_regenerate_guid(person_guid, buf) == -1))
			info("Registry: person guid '%s' is not a valid guid. Ignoring it.", person_guid);
		else {
			person_guid = buf;
			p = registry_person_find(person_guid);
			if(!p) person_guid = NULL;
		}
	}

	if(!p) p = registry_person_allocate(NULL, when);

	registry_persons_unlock();

	return p;
}

// ----------------------------------------------------------------------------
// LINKING OF OBJECTS

static inline PERSON_URL *registry_person_link_to_url(PERSON *p, MACHINE *m, URL *u, char *name, size_t namelen, time_t when) {
	debug(D_REGISTRY, "registry_person_link_to_url('%s', '%s', '%s'): searching for URL in person", p->guid, m->guid, u->url);

	registry_person_urls_lock(p);

	PERSON_URL *pu = dictionary_get(p->urls, u->url);
	if(!pu) {
		debug(D_REGISTRY, "registry_person_link_to_url('%s', '%s', '%s'): not found", p->guid, m->guid, u->url);
		pu = registry_person_url_allocate(p, m, u, name, namelen, when);
		registry.persons_urls_count++;
	}
	else {
		debug(D_REGISTRY, "registry_person_link_to_url('%s', '%s', '%s'): found", p->guid, m->guid, u->url);
		pu->usages++;
		if(likely(pu->last_t < when)) pu->last_t = when;

		if(pu->machine != m) {
			MACHINE_URL *mu = dictionary_get(pu->machine->urls, u->url);
			if(mu) {
				info("registry_person_link_to_url('%s', '%s', '%s'): URL switched machines (old was '%s') - expiring it from previous machine.",
					 p->guid, m->guid, u->url, pu->machine->guid);
				mu->flags |= REGISTRY_URL_FLAGS_EXPIRED;
			}
			else {
				info("registry_person_link_to_url('%s', '%s', '%s'): URL switched machines (old was '%s') - but the URL is not linked to the old machine.",
					 p->guid, m->guid, u->url, pu->machine->guid);
			}

			pu->machine->links--;
			pu->machine = m;
		}

		if(strcmp(pu->name, name)) {
			// the name of the PERSON_URL has changed !
			pu = registry_person_url_reallocate(p, m, u, name, namelen, when, pu);
		}
	}

	p->usages++;
	if(likely(p->last_t < when)) p->last_t = when;

	if(pu->flags & REGISTRY_URL_FLAGS_EXPIRED) {
		info("registry_person_link_to_url('%s', '%s', '%s'): accessing an expired URL. Re-enabling URL.", p->guid, m->guid, u->url);
		pu->flags &= ~REGISTRY_URL_FLAGS_EXPIRED;
	}

	registry_person_urls_unlock(p);

	return pu;
}

static inline MACHINE_URL *registry_machine_link_to_url(PERSON *p, MACHINE *m, URL *u, time_t when) {
	debug(D_REGISTRY, "registry_machine_link_to_url('%s', '%s', '%s'): searching for URL in machine", p->guid, m->guid, u->url);

	registry_machine_urls_lock(m);

	MACHINE_URL *mu = dictionary_get(m->urls, u->url);
	if(!mu) {
		debug(D_REGISTRY, "registry_machine_link_to_url('%s', '%s', '%s'): not found", p->guid, m->guid, u->url);
		mu = registry_machine_url_allocate(m, u, when);
		registry.machines_urls_count++;
	}
	else {
		debug(D_REGISTRY, "registry_machine_link_to_url('%s', '%s', '%s'): found", p->guid, m->guid, u->url);
		mu->usages++;
		if(likely(mu->last_t < when)) mu->last_t = when;
	}

	//debug(D_REGISTRY, "registry_machine_link_to_url('%s', '%s', '%s'): indexing person in machine", p->guid, m->guid, u->url);
	//dictionary_set(mu->persons, p->guid, p, sizeof(PERSON));

	m->usages++;
	if(likely(m->last_t < when)) m->last_t = when;

	if(mu->flags & REGISTRY_URL_FLAGS_EXPIRED) {
		info("registry_machine_link_to_url('%s', '%s', '%s'): accessing an expired URL.", p->guid, m->guid, u->url);
		mu->flags &= ~REGISTRY_URL_FLAGS_EXPIRED;
	}

	registry_machine_urls_unlock(m);

	return mu;
}

// ----------------------------------------------------------------------------
// REGISTRY LOG LOAD/SAVE

static inline int registry_should_save_db(void) {
	debug(D_REGISTRY, "log entries %llu, max %llu", registry.log_count, registry.save_registry_every_entries);
	return registry.log_count > registry.save_registry_every_entries;
}

static inline void registry_log(const char action, PERSON *p, MACHINE *m, URL *u, char *name) {
	if(likely(registry.log_fp)) {
		// we lock only if the file is open
		// to allow replaying the log at registry_log_load()
		registry_log_lock();

		if(unlikely(fprintf(registry.log_fp, "%c\t%08x\t%s\t%s\t%s\t%s\n",
				action,
				p->last_t,
				p->guid,
				m->guid,
				name,
				u->url) < 0))
			error("Registry: failed to save log. Registry data may be lost in case of abnormal restart.");

		// we increase the counter even on failures
		// so that the registry will be saved periodically
		registry.log_count++;

		registry_log_unlock();

		// this must be outside the log_lock(), or a deadlock will happen.
		// registry_save() checks the same inside the log_lock, so only
		// one thread will save the db
		if(unlikely(registry_should_save_db()))
			registry_save();
	}
}

static inline int registry_log_open_nolock(void) {
	if(registry.log_fp)
		fclose(registry.log_fp);

	registry.log_fp = fopen(registry.log_filename, "a");

	if(registry.log_fp) {
		if (setvbuf(registry.log_fp, NULL, _IOLBF, 0) != 0)
			error("Cannot set line buffering on registry log file.");
		return 0;
	}

	error("Cannot open registry log file '%s'. Registry data will be lost in case of netdata or server crash.", registry.log_filename);
	return -1;
}

static inline void registry_log_close_nolock(void) {
	if(registry.log_fp) {
		fclose(registry.log_fp);
		registry.log_fp = NULL;
	}
}

static inline void registry_log_recreate_nolock(void) {
	if(registry.log_fp != NULL) {
		registry_log_close_nolock();

		// open it with truncate
		registry.log_fp = fopen(registry.log_filename, "w");
		if(registry.log_fp) fclose(registry.log_fp);
		else error("Cannot truncate registry log '%s'", registry.log_filename);

		registry.log_fp = NULL;

		registry_log_open_nolock();
	}
}

int registry_log_load(void) {
	char *s, buf[4096 + 1];
	size_t line = -1;

	// closing the log is required here
	// otherwise we will append to it the values we read
	registry_log_close_nolock();

	debug(D_REGISTRY, "Registry: loading active db from: %s", registry.log_filename);
	FILE *fp = fopen(registry.log_filename, "r");
	if(!fp)
		error("Registry: cannot open registry file: %s", registry.log_filename);
	else {
		line = 0;
		size_t len = 0;
		while ((s = fgets_trim_len(buf, 4096, fp, &len))) {
			line++;

			switch (s[0]) {
				case 'A': // accesses
				case 'D': // deletes

					// verify it is valid
					if (unlikely(len < 85 || s[1] != '\t' || s[10] != '\t' || s[47] != '\t' || s[84] != '\t')) {
						error("Registry: log line %u is wrong (len = %zu).", line, len);
						continue;
					}
					s[1] = s[10] = s[47] = s[84] = '\0';

					// get the variables
					time_t when = strtoul(&s[2], NULL, 16);
					char *person_guid = &s[11];
					char *machine_guid = &s[48];
					char *name = &s[85];

					// skip the name to find the url
					char *url = name;
					while(*url && *url != '\t') url++;
					if(!*url) {
						error("Registry: log line %u does not have a url.", line);
						continue;
					}
					*url++ = '\0';

					// make sure the person exists
					// without this, a new person guid will be created
					PERSON *p = registry_person_find(person_guid);
					if(!p) p = registry_person_allocate(person_guid, when);

					if(s[0] == 'A')
						registry_request_access(p->guid, machine_guid, url, name, when);
					else
						registry_request_delete(p->guid, machine_guid, url, name, when);

					break;

				default:
					error("Registry: ignoring line %zu of filename '%s': %s.", line, registry.log_filename, s);
					break;
			}
		}
	}

	// open the log again
	registry_log_open_nolock();

	return line;
}


// ----------------------------------------------------------------------------
// REGISTRY REQUESTS

PERSON *registry_request_access(char *person_guid, char *machine_guid, char *url, char *name, time_t when) {
	debug(D_REGISTRY, "registry_request_access('%s', '%s', '%s'): NEW REQUEST", (person_guid)?person_guid:"", machine_guid, url);

	MACHINE *m = registry_machine_get(machine_guid, when);
	if(!m) return NULL;

	// make sure the name is valid
	size_t namelen;
	name = registry_fix_machine_name(name, &namelen);

	size_t urllen;
	url = registry_fix_url(url, &urllen);

	URL *u = registry_url_get(url, urllen);
	PERSON *p = registry_person_get(person_guid, when);

	registry_person_link_to_url(p, m, u, name, namelen, when);
	registry_machine_link_to_url(p, m, u, when);

	registry_log('A', p, m, u, name);

	registry.usages_count++;
	return p;
}

// verify the person, the machine and the URL exist in our DB
PERSON_URL *registry_verify_request(char *person_guid, char *machine_guid, char *url, PERSON **pp, MACHINE **mm) {
	char pbuf[36 + 1], mbuf[36 + 1];

	if(!person_guid || !*person_guid || !machine_guid || !*machine_guid || !url || !*url) {
		info("Registry Request Verification: invalid request! person: '%s', machine '%s', url '%s'", person_guid?person_guid:"UNSET", machine_guid?machine_guid:"UNSET", url?url:"UNSET");
		return NULL;
	}

	// normalize the url
	url = registry_fix_url(url, NULL);

	// make sure the person GUID is valid
	if(registry_regenerate_guid(person_guid, pbuf) == -1) {
		info("Registry Request Verification: invalid person GUID, person: '%s', machine '%s', url '%s'", person_guid, machine_guid, url);
		return NULL;
	}
	person_guid = pbuf;

	// make sure the machine GUID is valid
	if(registry_regenerate_guid(machine_guid, mbuf) == -1) {
		info("Registry Request Verification: invalid machine GUID, person: '%s', machine '%s', url '%s'", person_guid, machine_guid, url);
		return NULL;
	}
	machine_guid = mbuf;

	// make sure the machine exists
	MACHINE *m = registry_machine_find(machine_guid);
	if(!m) {
		info("Registry Request Verification: machine not found, person: '%s', machine '%s', url '%s'", person_guid, machine_guid, url);
		return NULL;
	}
	if(mm) *mm = m;

	// make sure the person exist
	PERSON *p = registry_person_find(person_guid);
	if(!p) {
		info("Registry Request Verification: person not found, person: '%s', machine '%s', url '%s'", person_guid, machine_guid, url);
		return NULL;
	}
	if(pp) *pp = p;

	PERSON_URL *pu = dictionary_get(p->urls, url);
	if(!pu) {
		info("Registry Request Verification: URL not found for person, person: '%s', machine '%s', url '%s'", person_guid, machine_guid, url);
		return NULL;
	}
	return pu;
}

PERSON *registry_request_delete(char *person_guid, char *machine_guid, char *url, char *delete_url, time_t when) {
	(void)when;

	PERSON *p = NULL;
	MACHINE *m = NULL;
	PERSON_URL *pu = registry_verify_request(person_guid, machine_guid, url, &p, &m);
	if(!pu || !p || !m) return NULL;

	// normalize the url
	delete_url = registry_fix_url(delete_url, NULL);

	// make sure the user is not deleting the url it uses
	if(!strcmp(delete_url, pu->url->url)) {
		info("Registry Delete Request: delete URL is the one currently accessed, person: '%s', machine '%s', url '%s', delete url '%s'", p->guid, m->guid, pu->url->url, delete_url);
		return NULL;
	}

	registry_person_urls_lock(p);

	PERSON_URL *dpu = dictionary_get(p->urls, delete_url);
	if(!dpu) {
		info("Registry Delete Request: URL not found for person: '%s', machine '%s', url '%s', delete url '%s'", p->guid, m->guid, pu->url->url, delete_url);
		registry_person_urls_unlock(p);
		return NULL;
	}

	registry_log('D', p, m, pu->url, dpu->url->url);

	dictionary_del(p->urls, dpu->url->url);
	registry_url_unlink_nolock(dpu->url);
	free(dpu);

	registry_person_urls_unlock(p);
	return p;
}


// a structure to pass to the dictionary_get_all() callback handler
struct machine_request_callback_data {
	MACHINE *find_this_machine;
	PERSON_URL *result;
};

// the callback function
// this will be run for every PERSON_URL of this PERSON
int machine_request_callback(void *entry, void *data) {
	PERSON_URL *mypu = (PERSON_URL *)entry;
	struct machine_request_callback_data *myrdata = (struct machine_request_callback_data *)data;

	if(mypu->machine == myrdata->find_this_machine) {
		myrdata->result = mypu;
		return -1; // this will also stop the walk through
	}

	return 0; // continue
}

MACHINE *registry_request_machine(char *person_guid, char *machine_guid, char *url, char *request_machine, time_t when) {
	(void)when;

	char mbuf[36 + 1];

	PERSON *p = NULL;
	MACHINE *m = NULL;
	PERSON_URL *pu = registry_verify_request(person_guid, machine_guid, url, &p, &m);
	if(!pu || !p || !m) return NULL;

	// make sure the machine GUID is valid
	if(registry_regenerate_guid(request_machine, mbuf) == -1) {
		info("Registry Machine URLs request: invalid machine GUID, person: '%s', machine '%s', url '%s', request machine '%s'", p->guid, m->guid, pu->url->url, request_machine);
		return NULL;
	}
	request_machine = mbuf;

	// make sure the machine exists
	m = registry_machine_find(request_machine);
	if(!m) {
		info("Registry Machine URLs request: machine not found, person: '%s', machine '%s', url '%s', request machine '%s'", p->guid, m->guid, pu->url->url, request_machine);
		return NULL;
	}

	// Verify the user has in the past accessed this machine
	// We will walk through the PERSON_URLs to find the machine
	// linking to our machine

	// a structure to pass to the dictionary_get_all() callback handler
	struct machine_request_callback_data rdata = { m, NULL };

	// request a walk through on the dictionary
	// no need for locking here, the underlying dictionary has its own
	dictionary_get_all(p->urls, machine_request_callback, &rdata);

	if(rdata.result)
		return m;

	return NULL;
}


// ----------------------------------------------------------------------------
// REGISTRY JSON generation

#define REGISTRY_STATUS_OK "ok"
#define REGISTRY_STATUS_FAILED "failed"
#define REGISTRY_STATUS_DISABLED "disabled"

static inline void registry_set_person_cookie(struct web_client *w, PERSON *p) {
	char edate[100];
	time_t et = time(NULL) + registry.persons_expiration;
	struct tm etmbuf, *etm = gmtime_r(&et, &etmbuf);
	strftime(edate, sizeof(edate), "%a, %d %b %Y %H:%M:%S %Z", etm);

	snprintfz(w->cookie1, COOKIE_MAX, NETDATA_REGISTRY_COOKIE_NAME "=%s; Expires=%s", p->guid, edate);

	if(registry.registry_domain && registry.registry_domain[0])
		snprintfz(w->cookie2, COOKIE_MAX, NETDATA_REGISTRY_COOKIE_NAME "=%s; Domain=%s; Expires=%s", p->guid, registry.registry_domain, edate);
}

static inline void registry_json_header(struct web_client *w, const char *action, const char *status) {
	w->response.data->contenttype = CT_APPLICATION_JSON;
	buffer_flush(w->response.data);
	buffer_sprintf(w->response.data, "{\n\t\"action\": \"%s\",\n\t\"status\": \"%s\",\n\t\"hostname\": \"%s\",\n\t\"machine_guid\": \"%s\"",
				   action, status, registry.hostname, registry.machine_guid);
}

static inline void registry_json_footer(struct web_client *w) {
	buffer_strcat(w->response.data, "\n}\n");
}

int registry_request_hello_json(struct web_client *w) {
	registry_json_header(w, "hello", REGISTRY_STATUS_OK);

	buffer_sprintf(w->response.data, ",\n\t\"registry\": \"%s\"",
				   registry.registry_to_announce);

	registry_json_footer(w);
	return 200;
}

static inline int registry_json_disabled(struct web_client *w, const char *action) {
	registry_json_header(w, action, REGISTRY_STATUS_DISABLED);

	buffer_sprintf(w->response.data, ",\n\t\"registry\": \"%s\"",
				   registry.registry_to_announce);

	registry_json_footer(w);
	return 200;
}

// structure used be the callbacks below
struct registry_json_walk_person_urls_callback {
	PERSON *p;
	MACHINE *m;
	struct web_client *w;
	int count;
};

// callback for rendering PERSON_URLs
static inline int registry_json_person_url_callback(void *entry, void *data) {
	PERSON_URL *pu = (PERSON_URL *)entry;
	struct registry_json_walk_person_urls_callback *c = (struct registry_json_walk_person_urls_callback *)data;
	struct web_client *w = c->w;

	if(unlikely(c->count++))
		buffer_strcat(w->response.data, ",");

	buffer_sprintf(w->response.data, "\n\t\t[ \"%s\", \"%s\", %u000, %u, \"%s\" ]",
				   pu->machine->guid, pu->url->url, pu->last_t, pu->usages, pu->name);

	return 1;
}

// callback for rendering MACHINE_URLs
static inline int registry_json_machine_url_callback(void *entry, void *data) {
	MACHINE_URL *mu = (MACHINE_URL *)entry;
	struct registry_json_walk_person_urls_callback *c = (struct registry_json_walk_person_urls_callback *)data;
	struct web_client *w = c->w;
	MACHINE *m = c->m;

	if(unlikely(c->count++))
		buffer_strcat(w->response.data, ",");

	buffer_sprintf(w->response.data, "\n\t\t[ \"%s\", \"%s\", %u000, %u ]",
				   m->guid, mu->url->url, mu->last_t, mu->usages);

	return 1;
}


// the main method for registering an access
int registry_request_access_json(struct web_client *w, char *person_guid, char *machine_guid, char *url, char *name, time_t when) {
	if(!registry.enabled)
		return registry_json_disabled(w, "access");

	PERSON *p = registry_request_access(person_guid, machine_guid, url, name, when);
	if(!p) {
		registry_json_header(w, "access", REGISTRY_STATUS_FAILED);
		registry_json_footer(w);
		return 412;
	}

	// set the cookie
	registry_set_person_cookie(w, p);

	// generate the response
	registry_json_header(w, "access", REGISTRY_STATUS_OK);

	buffer_sprintf(w->response.data, ",\n\t\"person_guid\": \"%s\",\n\t\"urls\": [", p->guid);
	struct registry_json_walk_person_urls_callback c = { p, NULL, w, 0 };
	dictionary_get_all(p->urls, registry_json_person_url_callback, &c);
	buffer_strcat(w->response.data, "\n\t]\n");

	registry_json_footer(w);
	return 200;
}

// the main method for deleting a URL from a person
int registry_request_delete_json(struct web_client *w, char *person_guid, char *machine_guid, char *url, char *delete_url, time_t when) {
	if(!registry.enabled)
		return registry_json_disabled(w, "delete");

	PERSON *p = registry_request_delete(person_guid, machine_guid, url, delete_url, when);
	if(!p) {
		registry_json_header(w, "delete", REGISTRY_STATUS_FAILED);
		registry_json_footer(w);
		return 412;
	}

	// generate the response
	registry_json_header(w, "delete", REGISTRY_STATUS_OK);
	registry_json_footer(w);
	return 200;
}

// the main method for searching the URLs of a netdata
int registry_request_search_json(struct web_client *w, char *person_guid, char *machine_guid, char *url, char *request_machine, time_t when) {
	if(!registry.enabled)
		return registry_json_disabled(w, "search");

	MACHINE *m = registry_request_machine(person_guid, machine_guid, url, request_machine, when);
	if(!m) {
		registry_json_header(w, "search", REGISTRY_STATUS_FAILED);
		registry_json_footer(w);
		return 404;
	}

	registry_json_header(w, "search", REGISTRY_STATUS_OK);

	buffer_strcat(w->response.data, ",\n\t\"urls\": [");
	struct registry_json_walk_person_urls_callback c = { NULL, m, w, 0 };
	dictionary_get_all(m->urls, registry_json_machine_url_callback, &c);
	buffer_strcat(w->response.data, "\n\t]\n");

	registry_json_footer(w);
	return 200;
}

// structure used be the callbacks below
struct registry_person_url_callback_verify_machine_exists_data {
	MACHINE *m;
	int count;
};

int registry_person_url_callback_verify_machine_exists(void *entry, void *data) {
	struct registry_person_url_callback_verify_machine_exists_data *d = (struct registry_person_url_callback_verify_machine_exists_data *)data;
	PERSON_URL *pu = (PERSON_URL *)entry;
	MACHINE *m = d->m;

	if(pu->machine == m)
		d->count++;

	return 0;
}

// the main method for switching user identity
int registry_request_switch_json(struct web_client *w, char *person_guid, char *machine_guid, char *url, char *new_person_guid, time_t when) {
	(void)url;
	(void)when;

	if(!registry.enabled)
		return registry_json_disabled(w, "switch");

	PERSON *op = registry_person_find(person_guid);
	if(!op) {
		registry_json_header(w, "switch", REGISTRY_STATUS_FAILED);
		registry_json_footer(w);
		return 430;
	}

	PERSON *np = registry_person_find(new_person_guid);
	if(!np) {
		registry_json_header(w, "switch", REGISTRY_STATUS_FAILED);
		registry_json_footer(w);
		return 431;
	}

	MACHINE *m = registry_machine_find(machine_guid);
	if(!m) {
		registry_json_header(w, "switch", REGISTRY_STATUS_FAILED);
		registry_json_footer(w);
		return 432;
	}

	struct registry_person_url_callback_verify_machine_exists_data data = { m, 0 };

	// verify the old person has access to this machine
	dictionary_get_all(op->urls, registry_person_url_callback_verify_machine_exists, &data);
	if(!data.count) {
		registry_json_header(w, "switch", REGISTRY_STATUS_FAILED);
		registry_json_footer(w);
		return 433;
	}

	// verify the new person has access to this machine
	data.count = 0;
	dictionary_get_all(np->urls, registry_person_url_callback_verify_machine_exists, &data);
	if(!data.count) {
		registry_json_header(w, "switch", REGISTRY_STATUS_FAILED);
		registry_json_footer(w);
		return 434;
	}

	// set the cookie of the new person
	// the user just switched identity
	registry_set_person_cookie(w, np);

	// generate the response
	registry_json_header(w, "switch", REGISTRY_STATUS_OK);
	buffer_sprintf(w->response.data, ",\n\t\"person_guid\": \"%s\"", np->guid);
	registry_json_footer(w);
	return 200;
}


// ----------------------------------------------------------------------------
// REGISTRY THIS MACHINE UNIQUE ID

char *registry_get_this_machine_guid(void) {
	if(likely(registry.machine_guid[0]))
		return registry.machine_guid;

	// read it from disk
	int fd = open(registry.machine_guid_filename, O_RDONLY);
	if(fd != -1) {
		char buf[36 + 1];
		if(read(fd, buf, 36) != 36)
			error("Failed to read machine GUID from '%s'", registry.machine_guid_filename);
		else {
			buf[36] = '\0';
			if(registry_regenerate_guid(buf, registry.machine_guid) == -1) {
				error("Failed to validate machine GUID '%s' from '%s'. Ignoring it - this might mean this netdata will appear as duplicate in the registry.",
					  buf, registry.machine_guid_filename);

				registry.machine_guid[0] = '\0';
			}
		}
		close(fd);
	}

	// generate a new one?
	if(!registry.machine_guid[0]) {
		uuid_t uuid;

		uuid_generate_time(uuid);
		uuid_unparse_lower(uuid, registry.machine_guid);
		registry.machine_guid[36] = '\0';

		// save it
		fd = open(registry.machine_guid_filename, O_WRONLY|O_CREAT|O_TRUNC, 444);
		if(fd == -1)
			fatal("Cannot create unique machine id file '%s'. Please fix this.", registry.machine_guid_filename);

		if(write(fd, registry.machine_guid, 36) != 36)
			fatal("Cannot write the unique machine id file '%s'. Please fix this.", registry.machine_guid_filename);

		close(fd);
	}

	return registry.machine_guid;
}


// ----------------------------------------------------------------------------
// REGISTRY LOAD/SAVE

int registry_machine_save_url(void *entry, void *file) {
	MACHINE_URL *mu = entry;
	FILE *fp = file;

	debug(D_REGISTRY, "Registry: registry_machine_save_url('%s')", mu->url->url);

	int ret = fprintf(fp, "V\t%08x\t%08x\t%08x\t%02x\t%s\n",
			mu->first_t,
			mu->last_t,
			mu->usages,
			mu->flags,
			mu->url->url
	);

	// error handling is done at registry_save()

	return ret;
}

int registry_machine_save(void *entry, void *file) {
	MACHINE *m = entry;
	FILE *fp = file;

	debug(D_REGISTRY, "Registry: registry_machine_save('%s')", m->guid);

	int ret = fprintf(fp, "M\t%08x\t%08x\t%08x\t%s\n",
			m->first_t,
			m->last_t,
			m->usages,
			m->guid
	);

	if(ret >= 0) {
		int ret2 = dictionary_get_all(m->urls, registry_machine_save_url, fp);
		if(ret2 < 0) return ret2;
		ret += ret2;
	}

	// error handling is done at registry_save()

	return ret;
}

static inline int registry_person_save_url(void *entry, void *file) {
	PERSON_URL *pu = entry;
	FILE *fp = file;

	debug(D_REGISTRY, "Registry: registry_person_save_url('%s')", pu->url->url);

	int ret = fprintf(fp, "U\t%08x\t%08x\t%08x\t%02x\t%s\t%s\t%s\n",
			pu->first_t,
			pu->last_t,
			pu->usages,
			pu->flags,
			pu->machine->guid,
			pu->name,
			pu->url->url
	);

	// error handling is done at registry_save()

	return ret;
}

static inline int registry_person_save(void *entry, void *file) {
	PERSON *p = entry;
	FILE *fp = file;

	debug(D_REGISTRY, "Registry: registry_person_save('%s')", p->guid);

	int ret = fprintf(fp, "P\t%08x\t%08x\t%08x\t%s\n",
			p->first_t,
			p->last_t,
			p->usages,
			p->guid
	);

	if(ret >= 0) {
		int ret2 = dictionary_get_all(p->urls, registry_person_save_url, fp);
		if (ret2 < 0) return ret2;
		ret += ret2;
	}

	// error handling is done at registry_save()

	return ret;
}

int registry_save(void) {
	if(!registry.enabled) return -1;

	// make sure the log is not updated
	registry_log_lock();

	if(unlikely(!registry_should_save_db())) {
		registry_log_unlock();
		return -2;
	}

	char tmp_filename[FILENAME_MAX + 1];
	char old_filename[FILENAME_MAX + 1];

	snprintfz(old_filename, FILENAME_MAX, "%s.old", registry.db_filename);
	snprintfz(tmp_filename, FILENAME_MAX, "%s.tmp", registry.db_filename);

	debug(D_REGISTRY, "Registry: Creating file '%s'", tmp_filename);
	FILE *fp = fopen(tmp_filename, "w");
	if(!fp) {
		error("Registry: Cannot create file: %s", tmp_filename);
		registry_log_unlock();
		return -1;
	}

	// dictionary_get_all() has its own locking, so this is safe to do

	debug(D_REGISTRY, "Saving all machines");
	int bytes1 = dictionary_get_all(registry.machines, registry_machine_save, fp);
	if(bytes1 < 0) {
		error("Registry: Cannot save registry machines - return value %d", bytes1);
		fclose(fp);
		registry_log_unlock();
		return bytes1;
	}
	debug(D_REGISTRY, "Registry: saving machines took %d bytes", bytes1);

	debug(D_REGISTRY, "Saving all persons");
	int bytes2 = dictionary_get_all(registry.persons, registry_person_save, fp);
	if(bytes2 < 0) {
		error("Registry: Cannot save registry persons - return value %d", bytes2);
		fclose(fp);
		registry_log_unlock();
		return bytes2;
	}
	debug(D_REGISTRY, "Registry: saving persons took %d bytes", bytes2);

	// save the totals
	fprintf(fp, "T\t%016llx\t%016llx\t%016llx\t%016llx\t%016llx\t%016llx\n",
			registry.persons_count,
			registry.machines_count,
			registry.usages_count + 1, // this is required - it is lost on db rotation
			registry.urls_count,
			registry.persons_urls_count,
			registry.machines_urls_count
	);

	fclose(fp);

	errno = 0;

	// remove the .old db
	debug(D_REGISTRY, "Registry: Removing old db '%s'", old_filename);
	if(unlink(old_filename) == -1 && errno != ENOENT)
		error("Registry: cannot remove old registry file '%s'", old_filename);

	// rename the db to .old
	debug(D_REGISTRY, "Registry: Link current db '%s' to .old: '%s'", registry.db_filename, old_filename);
	if(link(registry.db_filename, old_filename) == -1 && errno != ENOENT)
		error("Registry: cannot move file '%s' to '%s'. Saving registry DB failed!", tmp_filename, registry.db_filename);

	else {
		// remove the database (it is saved in .old)
		debug(D_REGISTRY, "Registry: removing db '%s'", registry.db_filename);
		if (unlink(registry.db_filename) == -1 && errno != ENOENT)
			error("Registry: cannot remove old registry file '%s'", registry.db_filename);

		// move the .tmp to make it active
		debug(D_REGISTRY, "Registry: linking tmp db '%s' to active db '%s'", tmp_filename, registry.db_filename);
		if (link(tmp_filename, registry.db_filename) == -1) {
			error("Registry: cannot move file '%s' to '%s'. Saving registry DB failed!", tmp_filename,
				  registry.db_filename);

			// move the .old back
			debug(D_REGISTRY, "Registry: linking old db '%s' to active db '%s'", old_filename, registry.db_filename);
			if(link(old_filename, registry.db_filename) == -1)
				error("Registry: cannot move file '%s' to '%s'. Recovering the old registry DB failed!", old_filename, registry.db_filename);
		}
		else {
			debug(D_REGISTRY, "Registry: removing tmp db '%s'", tmp_filename);
			if(unlink(tmp_filename) == -1)
				error("Registry: cannot remove tmp registry file '%s'", tmp_filename);

			// it has been moved successfully
			// discard the current registry log
			registry_log_recreate_nolock();

			registry.log_count = 0;
		}
	}

	// continue operations
	registry_log_unlock();

	return -1;
}

static inline size_t registry_load(void) {
	char *s, buf[4096 + 1];
	PERSON *p = NULL;
	MACHINE *m = NULL;
	URL *u = NULL;
	size_t line = 0;

	debug(D_REGISTRY, "Registry: loading active db from: '%s'", registry.db_filename);
	FILE *fp = fopen(registry.db_filename, "r");
	if(!fp) {
		error("Registry: cannot open registry file: '%s'", registry.db_filename);
		return 0;
	}

	size_t len = 0;
	buf[4096] = '\0';
	while((s = fgets_trim_len(buf, 4096, fp, &len))) {
		line++;

		debug(D_REGISTRY, "Registry: read line %zu to length %zu: %s", line, len, s);
		switch(*s) {
			case 'T': // totals
				if(unlikely(len != 103 || s[1] != '\t' || s[18] != '\t' || s[35] != '\t' || s[52] != '\t' || s[69] != '\t' || s[86] != '\t' || s[103] != '\0')) {
					error("Registry totals line %u is wrong (len = %zu).", line, len);
					continue;
				}
				registry.persons_count = strtoull(&s[2], NULL, 16);
				registry.machines_count = strtoull(&s[19], NULL, 16);
				registry.usages_count = strtoull(&s[36], NULL, 16);
				registry.urls_count = strtoull(&s[53], NULL, 16);
				registry.persons_urls_count = strtoull(&s[70], NULL, 16);
				registry.machines_urls_count = strtoull(&s[87], NULL, 16);
				break;

			case 'P': // person
				m = NULL;
				// verify it is valid
				if(unlikely(len != 65 || s[1] != '\t' || s[10] != '\t' || s[19] != '\t' || s[28] != '\t' || s[65] != '\0')) {
					error("Registry person line %u is wrong (len = %zu).", line, len);
					continue;
				}

				s[1] = s[10] = s[19] = s[28] = '\0';
				p = registry_person_allocate(&s[29], strtoul(&s[2], NULL, 16));
				p->last_t = strtoul(&s[11], NULL, 16);
				p->usages = strtoul(&s[20], NULL, 16);
				debug(D_REGISTRY, "Registry loaded person '%s', first: %u, last: %u, usages: %u", p->guid, p->first_t, p->last_t, p->usages);
				break;

			case 'M': // machine
				p = NULL;
				// verify it is valid
				if(unlikely(len != 65 || s[1] != '\t' || s[10] != '\t' || s[19] != '\t' || s[28] != '\t' || s[65] != '\0')) {
					error("Registry person line %u is wrong (len = %zu).", line, len);
					continue;
				}

				s[1] = s[10] = s[19] = s[28] = '\0';
				m = registry_machine_allocate(&s[29], strtoul(&s[2], NULL, 16));
				m->last_t = strtoul(&s[11], NULL, 16);
				m->usages = strtoul(&s[20], NULL, 16);
				debug(D_REGISTRY, "Registry loaded machine '%s', first: %u, last: %u, usages: %u", m->guid, m->first_t, m->last_t, m->usages);
				break;

			case 'U': // person URL
				if(unlikely(!p)) {
					error("Registry: ignoring line %zu, no person loaded: %s", line, s);
					continue;
				}

				// verify it is valid
				if(len < 69 || s[1] != '\t' || s[10] != '\t' || s[19] != '\t' || s[28] != '\t' || s[31] != '\t' || s[68] != '\t') {
					error("Registry person URL line %u is wrong (len = %zu).", line, len);
					continue;
				}

				s[1] = s[10] = s[19] = s[28] = s[31] = s[68] = '\0';

				// skip the name to find the url
				char *url = &s[69];
				while(*url && *url != '\t') url++;
				if(!*url) {
					error("Registry person URL line %u does not have a url.", line);
					continue;
				}
				*url++ = '\0';

				u = registry_url_allocate_nolock(url, strlen(url));

				time_t first_t = strtoul(&s[2], NULL, 16);

				m = registry_machine_find(&s[32]);
				if(!m) m = registry_machine_allocate(&s[32], first_t);

				PERSON_URL *pu = registry_person_url_allocate(p, m, u, &s[69], strlen(&s[69]), first_t);
				pu->last_t = strtoul(&s[11], NULL, 16);
				pu->usages = strtoul(&s[20], NULL, 16);
				pu->flags = strtoul(&s[29], NULL, 16);
				debug(D_REGISTRY, "Registry loaded person URL '%s' with name '%s' of machine '%s', first: %u, last: %u, usages: %u, flags: %02x", u->url, pu->name, m->guid, pu->first_t, pu->last_t, pu->usages, pu->flags);
				break;

			case 'V': // machine URL
				if(unlikely(!m)) {
					error("Registry: ignoring line %zu, no machine loaded: %s", line, s);
					continue;
				}

				// verify it is valid
				if(len < 32 || s[1] != '\t' || s[10] != '\t' || s[19] != '\t' || s[28] != '\t' || s[31] != '\t') {
					error("Registry person URL line %u is wrong (len = %zu).", line, len);
					continue;
				}

				s[1] = s[10] = s[19] = s[28] = s[31] = '\0';
				u = registry_url_allocate_nolock(&s[32], strlen(&s[32]));

				MACHINE_URL *mu = registry_machine_url_allocate(m, u, strtoul(&s[2], NULL, 16));
				mu->last_t = strtoul(&s[11], NULL, 16);
				mu->usages = strtoul(&s[20], NULL, 16);
				mu->flags = strtoul(&s[29], NULL, 16);
				debug(D_REGISTRY, "Registry loaded machine URL '%s', machine '%s', first: %u, last: %u, usages: %u, flags: %02x", u->url, m->guid, mu->first_t, mu->last_t, mu->usages, mu->flags);
				break;

			default:
				error("Registry: ignoring line %zu of filename '%s': %s.", line, registry.db_filename, s);
				break;
		}
	}
	fclose(fp);

	return line;
}

// ----------------------------------------------------------------------------
// REGISTRY

int registry_init(void) {
	char filename[FILENAME_MAX + 1];

	// registry enabled?
	registry.enabled = config_get_boolean("registry", "enabled", 0);

	// pathnames
	registry.pathname = config_get("registry", "registry db directory", VARLIB_DIR "/registry");
	if(mkdir(registry.pathname, 0755) == -1 && errno != EEXIST) {
		error("Cannot create directory '%s'. Registry disabled.", registry.pathname);
		registry.enabled = 0;
		return -1;
	}

	// filenames
	snprintfz(filename, FILENAME_MAX, "%s/netdata.public.unique.id", registry.pathname);
	registry.machine_guid_filename = config_get("registry", "netdata unique id file", filename);
	registry_get_this_machine_guid();

	snprintfz(filename, FILENAME_MAX, "%s/registry.db", registry.pathname);
	registry.db_filename = config_get("registry", "registry db file", filename);

	snprintfz(filename, FILENAME_MAX, "%s/registry-log.db", registry.pathname);
	registry.log_filename = config_get("registry", "registry log file", filename);

	// configuration options
	registry.save_registry_every_entries = config_get_number("registry", "registry save db every new entries", 1000000);
	registry.persons_expiration = config_get_number("registry", "registry expire idle persons days", 365) * 86400;
	registry.registry_domain = config_get("registry", "registry domain", "");
	registry.registry_to_announce = config_get("registry", "registry to announce", "https://registry.my-netdata.io");
	registry.hostname = config_get("registry", "registry hostname", config_get("global", "hostname", hostname));

	registry.max_url_length = config_get_number("registry", "max URL length", 1024);
	registry.max_name_length = config_get_number("registry", "max URL name length", 50);


	// initialize entries counters
	registry.persons_count = 0;
	registry.machines_count = 0;
	registry.usages_count = 0;
	registry.urls_count = 0;
	registry.persons_urls_count = 0;
	registry.machines_urls_count = 0;

	// initialize memory counters
	registry.persons_memory = 0;
	registry.machines_memory = 0;
	registry.urls_memory = 0;
	registry.persons_urls_memory = 0;
	registry.machines_urls_memory = 0;

	// initialize locks
	pthread_mutex_init(&registry.persons_lock, NULL);
	pthread_mutex_init(&registry.machines_lock, NULL);
	pthread_mutex_init(&registry.urls_lock, NULL);
	pthread_mutex_init(&registry.person_urls_lock, NULL);
	pthread_mutex_init(&registry.machine_urls_lock, NULL);

	// create dictionaries
	registry.persons = dictionary_create(DICTIONARY_FLAGS);
	registry.machines = dictionary_create(DICTIONARY_FLAGS);
	registry.urls = dictionary_create(DICTIONARY_FLAGS);

	// load the registry database
	if(registry.enabled) {
		registry_log_open_nolock();
		registry_load();
		registry_log_load();
	}

	return 0;
}

void registry_free(void) {
	if(!registry.enabled) return;

	// we need to destroy the dictionaries ourselves
	// since the dictionaries use memory we allocated

	while(registry.persons->values_index.root) {
		PERSON *p = ((NAME_VALUE *)registry.persons->values_index.root)->value;

		// fprintf(stderr, "\nPERSON: '%s', first: %u, last: %u, usages: %u\n", p->guid, p->first_t, p->last_t, p->usages);

		while(p->urls->values_index.root) {
			PERSON_URL *pu = ((NAME_VALUE *)p->urls->values_index.root)->value;

			// fprintf(stderr, "\tURL: '%s', first: %u, last: %u, usages: %u, flags: 0x%02x\n", pu->url->url, pu->first_t, pu->last_t, pu->usages, pu->flags);

			debug(D_REGISTRY, "Registry: deleting url '%s' from person '%s'", pu->url->url, p->guid);
			dictionary_del(p->urls, pu->url->url);

			debug(D_REGISTRY, "Registry: unlinking url '%s' from person", pu->url->url);
			registry_url_unlink_nolock(pu->url);

			debug(D_REGISTRY, "Registry: freeing person url");
			free(pu);
		}

		debug(D_REGISTRY, "Registry: deleting person '%s' from persons registry", p->guid);
		dictionary_del(registry.persons, p->guid);

		debug(D_REGISTRY, "Registry: destroying URL dictionary of person '%s'", p->guid);
		dictionary_destroy(p->urls);

		debug(D_REGISTRY, "Registry: freeing person '%s'", p->guid);
		free(p);
	}

	while(registry.machines->values_index.root) {
		MACHINE *m = ((NAME_VALUE *)registry.machines->values_index.root)->value;

		// fprintf(stderr, "\nMACHINE: '%s', first: %u, last: %u, usages: %u\n", m->guid, m->first_t, m->last_t, m->usages);

		while(m->urls->values_index.root) {
			MACHINE_URL *mu = ((NAME_VALUE *)m->urls->values_index.root)->value;

			// fprintf(stderr, "\tURL: '%s', first: %u, last: %u, usages: %u, flags: 0x%02x\n", mu->url->url, mu->first_t, mu->last_t, mu->usages, mu->flags);

			//debug(D_REGISTRY, "Registry: destroying persons dictionary from url '%s'", mu->url->url);
			//dictionary_destroy(mu->persons);

			debug(D_REGISTRY, "Registry: deleting url '%s' from person '%s'", mu->url->url, m->guid);
			dictionary_del(m->urls, mu->url->url);

			debug(D_REGISTRY, "Registry: unlinking url '%s' from machine", mu->url->url);
			registry_url_unlink_nolock(mu->url);

			debug(D_REGISTRY, "Registry: freeing machine url");
			free(mu);
		}

		debug(D_REGISTRY, "Registry: deleting machine '%s' from machines registry", m->guid);
		dictionary_del(registry.machines, m->guid);

		debug(D_REGISTRY, "Registry: destroying URL dictionary of machine '%s'", m->guid);
		dictionary_destroy(m->urls);

		debug(D_REGISTRY, "Registry: freeing machine '%s'", m->guid);
		free(m);
	}

	// and free the memory of remaining dictionary structures

	debug(D_REGISTRY, "Registry: destroying persons dictionary");
	dictionary_destroy(registry.persons);

	debug(D_REGISTRY, "Registry: destroying machines dictionary");
	dictionary_destroy(registry.machines);

	debug(D_REGISTRY, "Registry: destroying urls dictionary");
	dictionary_destroy(registry.urls);
}

// ----------------------------------------------------------------------------
// STATISTICS

void registry_statistics(void) {
	if(!registry.enabled) return;

	static RRDSET *sts = NULL, *stc = NULL, *stm = NULL;

	if(!sts) sts = rrdset_find("netdata.registry_sessions");
	if(!sts) {
		sts = rrdset_create("netdata", "registry_sessions", NULL, "registry", NULL, "NetData Registry Sessions", "session", 131000, rrd_update_every, RRDSET_TYPE_LINE);

		rrddim_add(sts, "sessions",  NULL,  1, 1, RRDDIM_ABSOLUTE);
	}
	else rrdset_next(sts);

	rrddim_set(sts, "sessions", registry.usages_count);
	rrdset_done(sts);

	// ------------------------------------------------------------------------

	if(!stc) stc = rrdset_find("netdata.registry_entries");
	if(!stc) {
		stc = rrdset_create("netdata", "registry_entries", NULL, "registry", NULL, "NetData Registry Entries", "entries", 131100, rrd_update_every, RRDSET_TYPE_LINE);

		rrddim_add(stc, "persons",        NULL,  1, 1, RRDDIM_ABSOLUTE);
		rrddim_add(stc, "machines",       NULL,  1, 1, RRDDIM_ABSOLUTE);
		rrddim_add(stc, "urls",           NULL,  1, 1, RRDDIM_ABSOLUTE);
		rrddim_add(stc, "persons_urls",   NULL,  1, 1, RRDDIM_ABSOLUTE);
		rrddim_add(stc, "machines_urls",  NULL,  1, 1, RRDDIM_ABSOLUTE);
	}
	else rrdset_next(stc);

	rrddim_set(stc, "persons",       registry.persons_count);
	rrddim_set(stc, "machines",      registry.machines_count);
	rrddim_set(stc, "urls",          registry.urls_count);
	rrddim_set(stc, "persons_urls",  registry.persons_urls_count);
	rrddim_set(stc, "machines_urls", registry.machines_urls_count);
	rrdset_done(stc);

	// ------------------------------------------------------------------------

	if(!stm) stm = rrdset_find("netdata.registry_mem");
	if(!stm) {
		stm = rrdset_create("netdata", "registry_mem", NULL, "registry", NULL, "NetData Registry Memory", "KB", 131300, rrd_update_every, RRDSET_TYPE_STACKED);

		rrddim_add(stm, "persons",        NULL,  1, 1024, RRDDIM_ABSOLUTE);
		rrddim_add(stm, "machines",       NULL,  1, 1024, RRDDIM_ABSOLUTE);
		rrddim_add(stm, "urls",           NULL,  1, 1024, RRDDIM_ABSOLUTE);
		rrddim_add(stm, "persons_urls",   NULL,  1, 1024, RRDDIM_ABSOLUTE);
		rrddim_add(stm, "machines_urls",  NULL,  1, 1024, RRDDIM_ABSOLUTE);
	}
	else rrdset_next(stm);

	rrddim_set(stm, "persons",       registry.persons_memory + registry.persons_count * sizeof(NAME_VALUE) + sizeof(DICTIONARY));
	rrddim_set(stm, "machines",      registry.machines_memory + registry.machines_count * sizeof(NAME_VALUE) + sizeof(DICTIONARY));
	rrddim_set(stm, "urls",          registry.urls_memory + registry.urls_count * sizeof(NAME_VALUE) + sizeof(DICTIONARY));
	rrddim_set(stm, "persons_urls",  registry.persons_urls_memory + registry.persons_count * sizeof(DICTIONARY) + registry.persons_urls_count * sizeof(NAME_VALUE));
	rrddim_set(stm, "machines_urls", registry.machines_urls_memory + registry.machines_count * sizeof(DICTIONARY) + registry.machines_urls_count * sizeof(NAME_VALUE));
	rrdset_done(stm);
}