diff options
Diffstat (limited to 'database.c')
| -rw-r--r-- | database.c | 2709 |
1 files changed, 2709 insertions, 0 deletions
diff --git a/database.c b/database.c new file mode 100644 index 0000000..b01c8ed --- /dev/null +++ b/database.c @@ -0,0 +1,2709 @@ +/* This file is part of "reprepro" + * Copyright (C) 2007,2008,2016 Bernhard R. Link + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02111-1301 USA + */ +#include <config.h> + +#include <sys/types.h> +#include <sys/stat.h> +#include <errno.h> +#include <assert.h> +#include <limits.h> +#include <string.h> +#include <stdio.h> +#include <stdint.h> +#include <stdlib.h> +#include <fcntl.h> +#include <unistd.h> +#include <db.h> + +#include "globals.h" +#include "error.h" +#include "ignore.h" +#include "strlist.h" +#include "names.h" +#include "database.h" +#include "dirs.h" +#include "filecntl.h" +#include "files.h" +#include "filelist.h" +#include "reference.h" +#include "tracking.h" +#include "dpkgversions.h" +#include "distribution.h" +#include "database_p.h" +#include "chunks.h" + +#define STRINGIFY(x) #x +#define TOSTRING(x) STRINGIFY(x) +#define LIBDB_VERSION_STRING "bdb" TOSTRING(DB_VERSION_MAJOR) "." TOSTRING(DB_VERSION_MINOR) "." TOSTRING(DB_VERSION_PATCH) +#define CLEARDBT(dbt) { memset(&dbt, 0, sizeof(dbt)); } +#define SETDBT(dbt, datastr) {const char *my = datastr; memset(&dbt, 0, sizeof(dbt)); dbt.data = (void *)my; dbt.size = strlen(my) + 1;} +#define SETDBTl(dbt, datastr, datasize) {const char *my = datastr; memset(&dbt, 0, sizeof(dbt)); dbt.data = (void *)my; dbt.size = datasize;} + +static bool rdb_initialized, rdb_used, rdb_locked, rdb_verbose; +static int rdb_dircreationdepth; +static bool rdb_nopackages, rdb_readonly; +static /*@null@*/ char *rdb_version, *rdb_lastsupportedversion, + *rdb_dbversion, *rdb_lastsupporteddbversion; +static DB_ENV *rdb_env = NULL; + +struct table *rdb_checksums, *rdb_contents; +struct table *rdb_references; + +struct opened_tables { + struct opened_tables *next; + const char *name; + const char *subname; +}; + +struct opened_tables *opened_tables = NULL; + +static void database_free(void) { + if (!rdb_initialized) + return; + free(rdb_version); + rdb_version = NULL; + free(rdb_lastsupportedversion); + rdb_lastsupportedversion = NULL; + free(rdb_dbversion); + rdb_dbversion = NULL; + free(rdb_lastsupporteddbversion); + rdb_lastsupporteddbversion = NULL; + rdb_initialized = false; +} + +static inline char *dbfilename(const char *filename) { + return calc_dirconcat(global.dbdir, filename); +} + +static retvalue database_openenv(void) { + int dbret; + + dbret = db_env_create(&rdb_env, 0); + if (dbret != 0) { + fprintf(stderr, "db_env_create: %s\n", db_strerror(dbret)); + return RET_ERROR; + } + + // DB_INIT_LOCK is needed to open multiple databases in one file (e.g. for move command) + dbret = rdb_env->open(rdb_env, global.dbdir, + DB_CREATE | DB_INIT_MPOOL | DB_PRIVATE | DB_INIT_LOCK, 0664); + if (dbret != 0) { + rdb_env->err(rdb_env, dbret, "environment open: %s", global.dbdir); + return RET_ERROR; + } + + return RET_OK; +} + +static void database_closeenv(void) { + int dbret; + + dbret = rdb_env->close(rdb_env, 0); + if (dbret != 0) { + fprintf(stderr, "Error: DB_ENV->close: %s\n", db_strerror(dbret)); + } + rdb_env = NULL; +} + +/**********************/ +/* lock file handling */ +/**********************/ + +static retvalue database_lock(size_t waitforlock) { + char *lockfile; + int fd; + retvalue r; + size_t tries = 0; + + assert (!rdb_locked); + rdb_dircreationdepth = 0; + r = dir_create_needed(global.dbdir, &rdb_dircreationdepth); + if (RET_WAS_ERROR(r)) + return r; + + lockfile = dbfilename("lockfile"); + if (FAILEDTOALLOC(lockfile)) + return RET_ERROR_OOM; + fd = open(lockfile, O_WRONLY|O_CREAT|O_EXCL|O_NOFOLLOW|O_NOCTTY, + S_IRUSR|S_IWUSR); + while (fd < 0) { + int e = errno; + if (e == EEXIST) { + if (tries < waitforlock && ! interrupted()) { + unsigned int timetosleep = 10; + if (verbose >= 0) + printf( +"Could not acquire lock: %s already exists!\nWaiting 10 seconds before trying again.\n", + lockfile); + while (timetosleep > 0) + timetosleep = sleep(timetosleep); + tries++; + fd = open(lockfile, O_WRONLY|O_CREAT|O_EXCL + |O_NOFOLLOW|O_NOCTTY, + S_IRUSR|S_IWUSR); + continue; + + } + fprintf(stderr, +"The lock file '%s' already exists. There might be another instance with the\n" +"same database dir running. To avoid locking overhead, only one process\n" +"can access the database at the same time. Do not delete the lock file unless\n" +"you are sure no other version is still running!\n", lockfile); + + } else + fprintf(stderr, +"Error %d creating lock file '%s': %s!\n", + e, lockfile, strerror(e)); + free(lockfile); + return RET_ERRNO(e); + } + // TODO: do some more locking of this file to avoid problems + // with the non-atomity of O_EXCL with nfs-filesystems... + if (close(fd) != 0) { + int e = errno; + fprintf(stderr, +"(Late) Error %d creating lock file '%s': %s!\n", + e, lockfile, strerror(e)); + (void)unlink(lockfile); + free(lockfile); + return RET_ERRNO(e); + } + free(lockfile); + rdb_locked = true; + + r = database_openenv(); + if (RET_WAS_ERROR(r)) { + (void)unlink(lockfile); + free(lockfile); + return r; + } + return RET_OK; +} + +static void releaselock(void) { + char *lockfile; + + assert (rdb_locked); + + database_closeenv(); + lockfile = dbfilename("lockfile"); + if (lockfile == NULL) + return; + if (unlink(lockfile) != 0) { + int e = errno; + fprintf(stderr, "Error %d deleting lock file '%s': %s!\n", + e, lockfile, strerror(e)); + (void)unlink(lockfile); + } + free(lockfile); + dir_remove_new(global.dbdir, rdb_dircreationdepth); + rdb_locked = false; +} + +static retvalue writeversionfile(void); + +retvalue database_close(void) { + retvalue result = RET_OK, r; + + if (rdb_references != NULL) { + r = table_close(rdb_references); + RET_UPDATE(result, r); + rdb_references = NULL; + } + if (rdb_checksums != NULL) { + r = table_close(rdb_checksums); + RET_UPDATE(result, r); + rdb_checksums = NULL; + } + if (rdb_contents != NULL) { + r = table_close(rdb_contents); + RET_UPDATE(result, r); + rdb_contents = NULL; + } + r = writeversionfile(); + RET_UPDATE(result, r); + if (rdb_locked) + releaselock(); + database_free(); + return result; +} + +static retvalue database_hasdatabasefile(const char *filename, /*@out@*/bool *exists_p) { + char *fullfilename; + + fullfilename = dbfilename(filename); + if (FAILEDTOALLOC(fullfilename)) + return RET_ERROR_OOM; + *exists_p = isregularfile(fullfilename); + free(fullfilename); + return RET_OK; +} + +enum database_type { + dbt_QUERY, + dbt_BTREE, dbt_BTREEDUP, dbt_BTREEPAIRS, dbt_BTREEVERSIONS, + dbt_HASH, + dbt_COUNT /* must be last */ +}; +static const uint32_t types[dbt_COUNT] = { + DB_UNKNOWN, + DB_BTREE, DB_BTREE, DB_BTREE, DB_BTREE, + DB_HASH +}; + +static int debianversioncompare(UNUSED(DB *db), const DBT *a, const DBT *b); +#if DB_VERSION_MAJOR >= 6 +static int paireddatacompare(UNUSED(DB *db), const DBT *a, const DBT *b, size_t *locp); +#else +static int paireddatacompare(UNUSED(DB *db), const DBT *a, const DBT *b); +#endif + +static retvalue database_opentable(const char *filename, /*@null@*/const char *subtable, enum database_type type, uint32_t flags, /*@out@*/DB **result) { + DB *table; + int dbret; + + dbret = db_create(&table, rdb_env, 0); + if (dbret != 0) { + fprintf(stderr, "db_create: %s\n", db_strerror(dbret)); + return RET_DBERR(dbret); + } + if (type == dbt_BTREEPAIRS || type == dbt_BTREEVERSIONS) { + dbret = table->set_flags(table, DB_DUPSORT); + if (dbret != 0) { + table->err(table, dbret, "db_set_flags(DB_DUPSORT):"); + (void)table->close(table, 0); + return RET_DBERR(dbret); + } + } else if (type == dbt_BTREEDUP) { + dbret = table->set_flags(table, DB_DUP); + if (dbret != 0) { + table->err(table, dbret, "db_set_flags(DB_DUP):"); + (void)table->close(table, 0); + return RET_DBERR(dbret); + } + } + if (type == dbt_BTREEPAIRS) { + dbret = table->set_dup_compare(table, paireddatacompare); + if (dbret != 0) { + table->err(table, dbret, "db_set_dup_compare:"); + (void)table->close(table, 0); + return RET_DBERR(dbret); + } + } + if (type == dbt_BTREEVERSIONS) { + dbret = table->set_dup_compare(table, debianversioncompare); + if (dbret != 0) { + table->err(table, dbret, "db_set_dup_compare:"); + (void)table->close(table, 0); + return RET_DBERR(dbret); + } + } + +#if DB_VERSION_MAJOR == 5 || DB_VERSION_MAJOR == 6 +#define DB_OPEN(database, filename, name, type, flags) \ + database->open(database, NULL, filename, name, type, flags, 0664) +#else +#if DB_VERSION_MAJOR == 4 +#define DB_OPEN(database, filename, name, type, flags) \ + database->open(database, NULL, filename, name, type, flags, 0664) +#else +#if DB_VERSION_MAJOR == 3 +#define DB_OPEN(database, filename, name, type, flags) \ + database->open(database, filename, name, type, flags, 0664) +#else +#error Unexpected DB_VERSION_MAJOR! +#endif +#endif +#endif + dbret = DB_OPEN(table, filename, subtable, types[type], flags); + if (dbret == ENOENT && !ISSET(flags, DB_CREATE)) { + (void)table->close(table, 0); + return RET_NOTHING; + } + if (dbret != 0) { + if (subtable != NULL) + table->err(table, dbret, "db_open(%s:%s)[%d]", + filename, subtable, dbret); + else + table->err(table, dbret, "db_open(%s)[%d]", + filename, dbret); + (void)table->close(table, 0); + return RET_DBERR(dbret); + } + *result = table; + return RET_OK; +} + +retvalue database_listsubtables(const char *filename, struct strlist *result) { + DB *table; + DBC *cursor; + DBT key, data; + int dbret; + retvalue ret, r; + struct strlist ids; + + r = database_opentable(filename, NULL, + dbt_QUERY, DB_RDONLY, &table); + if (!RET_IS_OK(r)) + return r; + + cursor = NULL; + if ((dbret = table->cursor(table, NULL, &cursor, 0)) != 0) { + table->err(table, dbret, "cursor(%s):", filename); + (void)table->close(table, 0); + return RET_ERROR; + } + CLEARDBT(key); + CLEARDBT(data); + + strlist_init(&ids); + + ret = RET_NOTHING; + while ((dbret=cursor->c_get(cursor, &key, &data, DB_NEXT)) == 0) { + char *identifier = strndup(key.data, key.size); + if (FAILEDTOALLOC(identifier)) { + (void)table->close(table, 0); + strlist_done(&ids); + return RET_ERROR_OOM; + } + r = strlist_add(&ids, identifier); + if (RET_WAS_ERROR(r)) { + (void)table->close(table, 0); + strlist_done(&ids); + return r; + } + ret = RET_OK; + CLEARDBT(key); + CLEARDBT(data); + } + + if (dbret != 0 && dbret != DB_NOTFOUND) { + table->err(table, dbret, "c_get(%s):", filename); + (void)table->close(table, 0); + strlist_done(&ids); + return RET_DBERR(dbret); + } + if ((dbret = cursor->c_close(cursor)) != 0) { + table->err(table, dbret, "c_close(%s):", filename); + (void)table->close(table, 0); + strlist_done(&ids); + return RET_DBERR(dbret); + } + + dbret = table->close(table, 0); + if (dbret != 0) { + table->err(table, dbret, "close(%s):", filename); + strlist_done(&ids); + return RET_DBERR(dbret); + } else { + strlist_move(result, &ids); + return ret; + } +} + +retvalue database_dropsubtable(const char *table, const char *subtable) { + char *filename; + DB *db; + int dbret; + + filename = dbfilename(table); + if (FAILEDTOALLOC(filename)) + return RET_ERROR_OOM; + + if ((dbret = db_create(&db, NULL, 0)) != 0) { + fprintf(stderr, "db_create: %s %s\n", + filename, db_strerror(dbret)); + free(filename); + return RET_DBERR(dbret); + } + dbret = db->remove(db, filename, subtable, 0); + if (dbret == ENOENT) { + free(filename); + return RET_NOTHING; + } + if (dbret != 0) { + fprintf(stderr, "Error removing '%s' from %s!\n", + subtable, filename); + free(filename); + return RET_DBERR(dbret); + } + + free(filename); + return RET_OK; +} + +static inline bool targetisdefined(const char *identifier, struct distribution *distributions) { + struct distribution *d; + struct target *t; + + for (d = distributions ; d != NULL ; d = d->next) { + for (t = d->targets; t != NULL ; t = t->next) { + if (strcmp(t->identifier, identifier) == 0) { + t->existed = true; + return true; + } + } + } + return false; +} + +static retvalue warnidentifiers(const struct strlist *identifiers, struct distribution *distributions, bool readonly) { + struct distribution *d; + struct target *t; + const char *identifier; + retvalue r; + int i; + + for (i = 0; i < identifiers->count ; i++) { + identifier = identifiers->values[i]; + + if (targetisdefined(identifier, distributions)) + continue; + + fprintf(stderr, +"Error: packages database contains unused '%s' database.\n", identifier); + if (ignored[IGN_undefinedtarget] == 0) { + (void)fputs( +"This usually means you removed some component, architecture or even\n" +"a whole distribution from conf/distributions.\n" +"In that case you most likely want to call reprepro clearvanished to get rid\n" +"of the databases belonging to those removed parts.\n" +"(Another reason to get this error is using conf/ and db/ directories\n" +" belonging to different reprepro repositories).\n", + stderr); + } + if (IGNORABLE(undefinedtarget)) { + (void)fputs( +"Ignoring as --ignore=undefinedtarget given.\n", + stderr); + ignored[IGN_undefinedtarget]++; + continue; + } + + (void)fputs( +"To ignore use --ignore=undefinedtarget.\n", + stderr); + return RET_ERROR; + } + if (readonly) + return RET_OK; + for (d = distributions ; d != NULL ; d = d->next) { + bool architecture_existed[d->architectures.count]; + bool have_old = false; + + /* check for new architectures */ + memset(architecture_existed, 0, sizeof(architecture_existed)); + + for (t = d->targets; t != NULL ; t = t->next) { + int o; + + if (!t->existed) + continue; + + o = atomlist_ofs(&d->architectures, + t->architecture); + assert (o >= 0); + if (o >= 0) { + architecture_existed[o] = true; + /* only warn about new ones if there + * is at least one old one, otherwise + * it's just a new distribution */ + have_old = true; + } + } + for (i = 0 ; have_old && i < d->architectures.count ; i++) { + architecture_t a; + + if (architecture_existed[i]) + continue; + + a = d->architectures.atoms[i]; + + fprintf(stderr, +"New architecture '%s' in '%s'. Perhaps you want to call\n" +"reprepro flood '%s' '%s'\n" +"to populate it with architecture 'all' packages from other architectures.\n", + atoms_architectures[a], d->codename, + d->codename, atoms_architectures[a]); + } + + /* create databases, so we know next time what is new */ + for (t = d->targets; t != NULL ; t = t->next) { + if (t->existed) + continue; + /* create database now, to test it can be created + * early, and to know when new architectures + * arrive in the future. */ + r = target_initpackagesdb(t, READWRITE); + if (RET_WAS_ERROR(r)) + return r; + r = target_closepackagesdb(t); + if (RET_WAS_ERROR(r)) + return r; + } + } + return RET_OK; +} + +static retvalue warnunusedtracking(const struct strlist *codenames, const struct distribution *distributions) { + const char *codename; + const struct distribution *d; + int i; + + for (i = 0; i < codenames->count ; i++) { + codename = codenames->values[i]; + + d = distributions; + while (d != NULL && strcmp(d->codename, codename) != 0) + d = d->next; + if (d != NULL && d->tracking != dt_NONE) + continue; + + fprintf(stderr, +"Error: tracking database contains unused '%s' database.\n", codename); + if (ignored[IGN_undefinedtracking] == 0) { + if (d == NULL) + (void)fputs( +"This either means you removed a distribution from the distributions config\n" +"file without calling clearvanished (or at least removealltracks), you\n" +"experienced a bug in retrack in versions < 3.0.0, you found a new bug or your\n" +"config does not belong to this database.\n", + stderr); + else + (void)fputs( +"This either means you removed the Tracking: options from this distribution without\n" +"calling removealltracks for it, or your config does not belong to this database.\n", + stderr); + } + if (IGNORABLE(undefinedtracking)) { + (void)fputs( +"Ignoring as --ignore=undefinedtracking given.\n", + stderr); + ignored[IGN_undefinedtracking]++; + continue; + } + + (void)fputs("To ignore use --ignore=undefinedtracking.\n", + stderr); + return RET_ERROR; + } + return RET_OK; +} + +static retvalue readline(/*@out@*/char **result, FILE *f, const char *versionfilename) { + char buffer[21]; + size_t l; + + if (fgets(buffer, 20, f) == NULL) { + int e = errno; + if (e == 0) { + fprintf(stderr, +"Error reading '%s': unexpected empty file\n", + versionfilename); + return RET_ERROR; + } else { + fprintf(stderr, "Error reading '%s': %s(errno is %d)\n", + versionfilename, strerror(e), e); + return RET_ERRNO(e); + } + } + l = strlen(buffer); + while (l > 0 && (buffer[l-1] == '\r' || buffer[l-1] == '\n')) { + buffer[--l] = '\0'; + } + if (l == 0) { + fprintf(stderr, "Error reading '%s': unexpected empty line.\n", + versionfilename); + return RET_ERROR; + } + *result = strdup(buffer); + if (FAILEDTOALLOC(*result)) + return RET_ERROR_OOM; + return RET_OK; +} + +static retvalue readversionfile(bool nopackagesyet) { + char *versionfilename; + FILE *f; + retvalue r; + int c; + + versionfilename = dbfilename("version"); + if (FAILEDTOALLOC(versionfilename)) + return RET_ERROR_OOM; + f = fopen(versionfilename, "r"); + if (f == NULL) { + int e = errno; + + if (e != ENOENT) { + fprintf(stderr, "Error opening '%s': %s(errno is %d)\n", + versionfilename, strerror(e), e); + free(versionfilename); + return RET_ERRNO(e); + } + free(versionfilename); + if (nopackagesyet) { + /* set to default for new packages.db files: */ + rdb_version = strdup(VERSION); + if (FAILEDTOALLOC(rdb_version)) + return RET_ERROR_OOM; + } else + rdb_version = NULL; + rdb_lastsupportedversion = NULL; + rdb_dbversion = NULL; + rdb_lastsupporteddbversion = NULL; + return RET_NOTHING; + } + /* first line is the version creating this database */ + r = readline(&rdb_version, f, versionfilename); + if (RET_WAS_ERROR(r)) { + (void)fclose(f); + free(versionfilename); + return r; + } + /* second line says which versions of reprepro will be able to cope + * with this database */ + r = readline(&rdb_lastsupportedversion, f, versionfilename); + if (RET_WAS_ERROR(r)) { + (void)fclose(f); + free(versionfilename); + return r; + } + /* next line is the version of the underlying database library */ + r = readline(&rdb_dbversion, f, versionfilename); + if (RET_WAS_ERROR(r)) { + (void)fclose(f); + free(versionfilename); + return r; + } + /* and then the minimum version of this library needed. */ + r = readline(&rdb_lastsupporteddbversion, f, versionfilename); + if (RET_WAS_ERROR(r)) { + (void)fclose(f); + free(versionfilename); + return r; + } + (void)fclose(f); + free(versionfilename); + + /* ensure we can understand it */ + + r = dpkgversions_cmp(VERSION, rdb_lastsupportedversion, &c); + if (RET_WAS_ERROR(r)) + return r; + if (c < 0) { + fprintf(stderr, +"According to %s/version this database was created with a future version\n" +"and uses features this version cannot understand. Aborting...\n", + global.dbdir); + return RET_ERROR; + } + + /* ensure it's a libdb database: */ + + if (strncmp(rdb_dbversion, "bdb", 3) != 0) { + fprintf(stderr, +"According to %s/version this database was created with a yet unsupported\n" +"database library. Aborting...\n", + global.dbdir); + return RET_ERROR; + } + if (strncmp(rdb_lastsupporteddbversion, "bdb", 3) != 0) { + fprintf(stderr, +"According to %s/version this database was created with a yet unsupported\n" +"database library. Aborting...\n", + global.dbdir); + return RET_ERROR; + } + r = dpkgversions_cmp(LIBDB_VERSION_STRING, + rdb_lastsupporteddbversion, &c); + if (RET_WAS_ERROR(r)) + return r; + if (c < 0) { + fprintf(stderr, +"According to %s/version this database was created with a future version\n" +"%s of libdb. The libdb version this binary is linked against cannot yet\n" +"handle this format. Aborting...\n", + global.dbdir, rdb_dbversion + 3); + return RET_ERROR; + } + return RET_OK; +} + +static retvalue writeversionfile(void) { + char *versionfilename, *finalversionfilename; + FILE *f; + int i, e; + + versionfilename = dbfilename("version.new"); + if (FAILEDTOALLOC(versionfilename)) + return RET_ERROR_OOM; + f = fopen(versionfilename, "w"); + if (f == NULL) { + e = errno; + fprintf(stderr, "Error creating '%s': %s(errno is %d)\n", + versionfilename, strerror(e), e); + free(versionfilename); + return RET_ERRNO(e); + } + if (rdb_version == NULL) + (void)fputs("0\n", f); + else { + (void)fputs(rdb_version, f); + (void)fputc('\n', f); + } + if (rdb_lastsupportedversion == NULL) { + (void)fputs("3.3.0\n", f); + } else { + int c; + retvalue r; + + r = dpkgversions_cmp(rdb_lastsupportedversion, + "3.3.0", &c); + if (!RET_IS_OK(r) || c < 0) + (void)fputs("3.3.0\n", f); + else { + (void)fputs(rdb_lastsupportedversion, f); + (void)fputc('\n', f); + } + } + if (rdb_dbversion == NULL) + fprintf(f, "bdb%d.%d.%d\n", DB_VERSION_MAJOR, DB_VERSION_MINOR, + DB_VERSION_PATCH); + else { + (void)fputs(rdb_dbversion, f); + (void)fputc('\n', f); + } + if (rdb_lastsupporteddbversion == NULL) + fprintf(f, "bdb%d.%d.0\n", DB_VERSION_MAJOR, DB_VERSION_MINOR); + else { + (void)fputs(rdb_lastsupporteddbversion, f); + (void)fputc('\n', f); + } + + e = ferror(f); + + if (e != 0) { + fprintf(stderr, "Error writing '%s': %s(errno is %d)\n", + versionfilename, strerror(e), e); + (void)fclose(f); + unlink(versionfilename); + free(versionfilename); + return RET_ERRNO(e); + } + if (fclose(f) != 0) { + e = errno; + fprintf(stderr, "Error writing '%s': %s(errno is %d)\n", + versionfilename, strerror(e), e); + unlink(versionfilename); + free(versionfilename); + return RET_ERRNO(e); + } + finalversionfilename = dbfilename("version"); + if (FAILEDTOALLOC(finalversionfilename)) { + unlink(versionfilename); + free(versionfilename); + return RET_ERROR_OOM; + } + + i = rename(versionfilename, finalversionfilename); + if (i != 0) { + e = errno; + fprintf(stderr, "Error %d moving '%s' to '%s': %s\n", + e, versionfilename, finalversionfilename, + strerror(e)); + (void)unlink(versionfilename); + free(versionfilename); + free(finalversionfilename); + return RET_ERRNO(e); + } + free(finalversionfilename); + free(versionfilename); + return RET_OK; +} + +static retvalue createnewdatabase(struct distribution *distributions) { + struct distribution *d; + struct target *t; + retvalue result = RET_NOTHING, r; + + for (d = distributions ; d != NULL ; d = d->next) { + for (t = d->targets ; t != NULL ; t = t->next) { + r = target_initpackagesdb(t, READWRITE); + RET_UPDATE(result, r); + if (RET_IS_OK(r)) { + r = target_closepackagesdb(t); + RET_UPDATE(result, r); + } + } + } + r = writeversionfile(); + RET_UPDATE(result, r); + return result; +} + +/* Initialize a database. + * - if not fast, make all kind of checks for consistency (TO BE IMPLEMENTED), + * - if readonly, do not create but return with RET_NOTHING + * - lock database, waiting a given amount of time if already locked + */ +retvalue database_create(struct distribution *alldistributions, bool fast, bool nopackages, bool allowunused, bool readonly, size_t waitforlock, bool verbosedb) { + retvalue r; + bool packagesfileexists, trackingfileexists, nopackagesyet; + + if (rdb_initialized || rdb_used) { + fputs("Internal Error: database initialized a 2nd time!\n", + stderr); + return RET_ERROR_INTERNAL; + } + + if (readonly && !isdir(global.dbdir)) { + if (verbose >= 0) + fprintf(stderr, +"Exiting without doing anything, as there is no database yet that could result in other actions.\n"); + return RET_NOTHING; + } + + rdb_initialized = true; + rdb_used = true; + + r = database_lock(waitforlock); + assert (r != RET_NOTHING); + if (!RET_IS_OK(r)) { + database_free(); + return r; + } + rdb_readonly = readonly; + rdb_verbose = verbosedb; + + r = database_hasdatabasefile("packages.db", &packagesfileexists); + if (RET_WAS_ERROR(r)) { + releaselock(); + database_free(); + return r; + } + r = database_hasdatabasefile("tracking.db", &trackingfileexists); + if (RET_WAS_ERROR(r)) { + releaselock(); + database_free(); + return r; + } + + nopackagesyet = !packagesfileexists && !trackingfileexists; + + r = readversionfile(nopackagesyet); + if (RET_WAS_ERROR(r)) { + releaselock(); + database_free(); + return r; + } + + if (nopackages) { + rdb_nopackages = true; + return RET_OK; + } + + if (nopackagesyet) { + // TODO: handle readonly, but only once packages files may no + // longer be generated when it is active... + + r = createnewdatabase(alldistributions); + if (RET_WAS_ERROR(r)) { + database_close(); + return r; + } + } + + /* after this point we should call database_close, + * as other stuff was handled, + * so writing the version file cannot harm (and not doing so could) */ + + if (!allowunused && !fast && packagesfileexists) { + struct strlist identifiers; + + r = database_listpackages(&identifiers); + if (RET_WAS_ERROR(r)) { + database_close(); + return r; + } + if (r == RET_NOTHING) + strlist_init(&identifiers); + r = warnidentifiers(&identifiers, + alldistributions, readonly); + if (RET_WAS_ERROR(r)) { + strlist_done(&identifiers); + database_close(); + return r; + } + strlist_done(&identifiers); + } + if (!allowunused && !fast && trackingfileexists) { + struct strlist codenames; + + r = tracking_listdistributions(&codenames); + if (RET_WAS_ERROR(r)) { + database_close(); + return r; + } + if (RET_IS_OK(r)) { + r = warnunusedtracking(&codenames, alldistributions); + if (RET_WAS_ERROR(r)) { + strlist_done(&codenames); + database_close(); + return r; + } + strlist_done(&codenames); + } + } + + return RET_OK; +} + +/**************************************************************************** + * Stuff string parts * + ****************************************************************************/ + +static const char databaseerror[] = "Internal error of the underlying BerkeleyDB database:\n"; + +/**************************************************************************** + * Stuff to handle data in tables * + **************************************************************************** + There is nothing that cannot be solved by another layer of indirection, except + too many levels of indirection. (Source forgotten) */ + +struct cursor { + DBC *cursor; + uint32_t flags; + retvalue r; +}; + +struct table { + char *name, *subname; + DB *berkeleydb; + DB *sec_berkeleydb; + bool readonly, verbose; + uint32_t flags; +}; + +static void table_printerror(struct table *table, int dbret, const char *action) { + char *error_msg; + + switch (dbret) { + case DB_MALFORMED_KEY: + error_msg = "DB_MALFORMED_KEY: Primary key does not contain the separator '|'."; + break; + case RET_ERROR_OOM: + error_msg = "RET_ERROR_OOM: Out of memory."; + break; + default: + error_msg = NULL; + break; + } + + if (error_msg == NULL) { + if (table->subname != NULL) + table->berkeleydb->err(table->berkeleydb, dbret, + "%sWithin %s subtable %s at %s", + databaseerror, table->name, table->subname, + action); + else + table->berkeleydb->err(table->berkeleydb, dbret, + "%sWithin %s at %s", + databaseerror, table->name, action); + } else { + if (table->subname != NULL) + table->berkeleydb->errx(table->berkeleydb, + "%sWithin %s subtable %s at %s: %s", + databaseerror, table->name, table->subname, + action, error_msg); + else + table->berkeleydb->errx(table->berkeleydb, + "%sWithin %s at %s: %s", + databaseerror, table->name, action, error_msg); + } +} + +static void print_opened_tables(FILE *stream) { + if (opened_tables == NULL) { + fprintf(stream, "No tables are opened.\n"); + } else { + fprintf(stream, "Opened tables:\n"); + for (struct opened_tables *iter = opened_tables; iter != NULL; iter = iter->next) { + fprintf(stream, " * %s - '%s'\n", iter->name, iter->subname); + } + } +} + +retvalue table_close(struct table *table) { + struct opened_tables *prev = NULL; + int dbret; + retvalue result = RET_OK; + + if (verbose >= 15) + fprintf(stderr, "trace: table_close(table.name=%s, table.subname=%s) called.\n", + table == NULL ? NULL : table->name, table == NULL ? NULL : table->subname); + if (table == NULL) + return RET_NOTHING; + if (table->sec_berkeleydb != NULL) { + dbret = table->sec_berkeleydb->close(table->sec_berkeleydb, 0); + if (dbret != 0) { + fprintf(stderr, "db_sec_close(%s, %s): %s\n", + table->name, table->subname, + db_strerror(dbret)); + result = RET_DBERR(dbret); + } + } + if (table->berkeleydb == NULL) { + assert (table->readonly); + dbret = 0; + } else + dbret = table->berkeleydb->close(table->berkeleydb, 0); + if (dbret != 0) { + fprintf(stderr, "db_close(%s, %s): %s\n", + table->name, table->subname, + db_strerror(dbret)); + result = RET_DBERR(dbret); + } + + for (struct opened_tables *iter = opened_tables; iter != NULL; iter = iter->next) { + if(strcmp2(iter->name, table->name) == 0 && strcmp2(iter->subname, table->subname) == 0) { + if (prev == NULL) { + opened_tables = iter->next; + } else { + prev->next = iter->next; + } + free(iter); + break; + } + prev = iter; + } + + if (verbose >= 25) + print_opened_tables(stderr); + + free(table->name); + free(table->subname); + free(table); + return result; +} + +retvalue table_getrecord(struct table *table, bool secondary, const char *key, char **data_p, size_t *datalen_p) { + int dbret; + DBT Key, Data; + DB *db; + + assert (table != NULL); + if (table->berkeleydb == NULL) { + assert (table->readonly); + return RET_NOTHING; + } + + SETDBT(Key, key); + CLEARDBT(Data); + Data.flags = DB_DBT_MALLOC; + + if (secondary) + db = table->sec_berkeleydb; + else + db = table->berkeleydb; + dbret = db->get(db, NULL, &Key, &Data, 0); + // TODO: find out what error code means out of memory... + if (dbret == DB_NOTFOUND) + return RET_NOTHING; + if (dbret != 0) { + table_printerror(table, dbret, "get"); + return RET_DBERR(dbret); + } + if (FAILEDTOALLOC(Data.data)) + return RET_ERROR_OOM; + if (Data.size <= 0 || + ((const char*)Data.data)[Data.size-1] != '\0') { + if (table->subname != NULL) + fprintf(stderr, +"Database %s(%s) returned corrupted (not null-terminated) data!\n", + table->name, table->subname); + else + fprintf(stderr, +"Database %s returned corrupted (not null-terminated) data!\n", + table->name); + free(Data.data); + return RET_ERROR; + } + *data_p = Data.data; + if (datalen_p != NULL) + *datalen_p = Data.size-1; + return RET_OK; +} + +retvalue table_getpair(struct table *table, const char *key, const char *value, /*@out@*/const char **data_p, /*@out@*/size_t *datalen_p) { + int dbret; + DBT Key, Data; + size_t valuelen = strlen(value); + + assert (table != NULL); + if (table->berkeleydb == NULL) { + assert (table->readonly); + return RET_NOTHING; + } + + SETDBT(Key, key); + SETDBTl(Data, value, valuelen + 1); + + dbret = table->berkeleydb->get(table->berkeleydb, NULL, + &Key, &Data, DB_GET_BOTH); + if (dbret == DB_NOTFOUND || dbret == DB_KEYEMPTY) + return RET_NOTHING; + if (dbret != 0) { + table_printerror(table, dbret, "get(BOTH)"); + return RET_DBERR(dbret); + } + if (FAILEDTOALLOC(Data.data)) + return RET_ERROR_OOM; + if (Data.size < valuelen + 2 || + ((const char*)Data.data)[Data.size-1] != '\0') { + if (table->subname != NULL) + fprintf(stderr, +"Database %s(%s) returned corrupted (not paired) data!", + table->name, table->subname); + else + fprintf(stderr, +"Database %s returned corrupted (not paired) data!", + table->name); + return RET_ERROR; + } + *data_p = ((const char*)Data.data) + valuelen + 1; + *datalen_p = Data.size - valuelen - 2; + return RET_OK; +} + +retvalue table_gettemprecord(struct table *table, const char *key, const char **data_p, size_t *datalen_p) { + int dbret; + DBT Key, Data; + + assert (table != NULL); + if (table->berkeleydb == NULL) { + assert (table->readonly); + return RET_NOTHING; + } + + SETDBT(Key, key); + CLEARDBT(Data); + + dbret = table->berkeleydb->get(table->berkeleydb, NULL, + &Key, &Data, 0); + // TODO: find out what error code means out of memory... + if (dbret == DB_NOTFOUND) + return RET_NOTHING; + if (dbret != 0) { + table_printerror(table, dbret, "get"); + return RET_DBERR(dbret); + } + if (FAILEDTOALLOC(Data.data)) + return RET_ERROR_OOM; + if (data_p == NULL) { + assert (datalen_p == NULL); + return RET_OK; + } + if (Data.size <= 0 || + ((const char*)Data.data)[Data.size-1] != '\0') { + if (table->subname != NULL) + fprintf(stderr, +"Database %s(%s) returned corrupted (not null-terminated) data!\n", + table->name, table->subname); + else + fprintf(stderr, +"Database %s returned corrupted (not null-terminated) data!\n", + table->name); + return RET_ERROR; + } + *data_p = Data.data; + if (datalen_p != NULL) + *datalen_p = Data.size - 1; + return RET_OK; +} + +retvalue table_checkrecord(struct table *table, const char *key, const char *data) { + int dbret; + DBT Key, Data; + DBC *cursor; + retvalue r; + + SETDBT(Key, key); + SETDBT(Data, data); + dbret = table->berkeleydb->cursor(table->berkeleydb, NULL, &cursor, 0); + if (dbret != 0) { + table_printerror(table, dbret, "cursor"); + return RET_DBERR(dbret); + } + dbret=cursor->c_get(cursor, &Key, &Data, DB_GET_BOTH); + if (dbret == 0) { + r = RET_OK; + } else if (dbret == DB_NOTFOUND) { + r = RET_NOTHING; + } else { + table_printerror(table, dbret, "c_get"); + (void)cursor->c_close(cursor); + return RET_DBERR(dbret); + } + dbret = cursor->c_close(cursor); + if (dbret != 0) { + table_printerror(table, dbret, "c_close"); + return RET_DBERR(dbret); + } + return r; +} + +retvalue table_removerecord(struct table *table, const char *key, const char *data) { + int dbret; + DBT Key, Data; + DBC *cursor; + retvalue r; + + SETDBT(Key, key); + SETDBT(Data, data); + dbret = table->berkeleydb->cursor(table->berkeleydb, NULL, &cursor, 0); + if (dbret != 0) { + table_printerror(table, dbret, "cursor"); + return RET_DBERR(dbret); + } + dbret=cursor->c_get(cursor, &Key, &Data, DB_GET_BOTH); + + if (dbret == 0) + dbret = cursor->c_del(cursor, 0); + + if (dbret == 0) { + r = RET_OK; + } else if (dbret == DB_NOTFOUND) { + r = RET_NOTHING; + } else { + table_printerror(table, dbret, "c_get"); + (void)cursor->c_close(cursor); + return RET_DBERR(dbret); + } + dbret = cursor->c_close(cursor); + if (dbret != 0) { + table_printerror(table, dbret, "c_close"); + return RET_DBERR(dbret); + } + return r; +} + +bool table_recordexists(struct table *table, const char *key) { + retvalue r; + + r = table_gettemprecord(table, key, NULL, NULL); + return RET_IS_OK(r); +} + +retvalue table_addrecord(struct table *table, const char *key, const char *data, size_t datalen, bool ignoredups) { + int dbret; + DBT Key, Data; + + assert (table != NULL); + assert (!table->readonly && table->berkeleydb != NULL); + + SETDBT(Key, key); + SETDBTl(Data, data, datalen + 1); + dbret = table->berkeleydb->put(table->berkeleydb, NULL, + &Key, &Data, ISSET(table->flags, DB_DUPSORT) ? DB_NODUPDATA : 0); + if (dbret != 0 && !(ignoredups && dbret == DB_KEYEXIST)) { + table_printerror(table, dbret, "put"); + return RET_DBERR(dbret); + } + if (table->verbose) { + if (table->subname != NULL) + printf("db: '%s' added to %s(%s).\n", + key, table->name, table->subname); + else + printf("db: '%s' added to %s.\n", + key, table->name); + } + return RET_OK; +} + +retvalue table_adduniqsizedrecord(struct table *table, const char *key, const char *data, size_t data_size, bool allowoverwrite, bool nooverwrite) { + int dbret; + DBT Key, Data; + + assert (table != NULL); + assert (!table->readonly && table->berkeleydb != NULL); + assert (data_size > 0 && data[data_size-1] == '\0'); + + SETDBT(Key, key); + SETDBTl(Data, data, data_size); + dbret = table->berkeleydb->put(table->berkeleydb, NULL, + &Key, &Data, allowoverwrite?0:DB_NOOVERWRITE); + if (nooverwrite && dbret == DB_KEYEXIST) { + /* if nooverwrite is set, do nothing and ignore: */ + return RET_NOTHING; + } + if (dbret != 0) { + table_printerror(table, dbret, "put(uniq)"); + return RET_DBERR(dbret); + } + if (table->verbose) { + if (table->subname != NULL) + printf("db: '%s' added to %s(%s).\n", + key, table->name, table->subname); + else + printf("db: '%s' added to %s.\n", + key, table->name); + } + return RET_OK; +} +retvalue table_adduniqrecord(struct table *table, const char *key, const char *data) { + if (verbose >= 15) + fprintf(stderr, "trace: table_adduniqrecord(table={name: %s, subname: %s}, key=%s) called.\n", + table->name, table->subname, key); + return table_adduniqsizedrecord(table, key, data, strlen(data)+1, + false, false); +} + +retvalue table_deleterecord(struct table *table, const char *key, bool ignoremissing) { + int dbret; + DBT Key; + + assert (table != NULL); + assert (!table->readonly && table->berkeleydb != NULL); + + SETDBT(Key, key); + dbret = table->berkeleydb->del(table->berkeleydb, NULL, &Key, 0); + if (dbret != 0) { + if (dbret == DB_NOTFOUND && ignoremissing) + return RET_NOTHING; + table_printerror(table, dbret, "del"); + if (dbret == DB_NOTFOUND) + return RET_ERROR_MISSING; + else + return RET_DBERR(dbret); + } + if (table->verbose) { + if (table->subname != NULL) + printf("db: '%s' removed from %s(%s).\n", + key, table->name, table->subname); + else + printf("db: '%s' removed from %s.\n", + key, table->name); + } + return RET_OK; +} + +retvalue table_replacerecord(struct table *table, const char *key, const char *data) { + retvalue r; + + if (verbose >= 15) + fprintf(stderr, "trace: table_replacerecord(table={name: %s, subname: %s}, key=%s) called.\n", + table->name, table->subname, key); + r = table_deleterecord(table, key, false); + if (r != RET_ERROR_MISSING && RET_WAS_ERROR(r)) + return r; + return table_adduniqrecord(table, key, data); +} + +static retvalue newcursor(struct table *table, uint32_t flags, struct cursor **cursor_p) { + DB *berkeleydb; + struct cursor *cursor; + int dbret; + + if (verbose >= 15) + fprintf(stderr, "trace: newcursor(table={name: %s, subname: %s}) called.\n", + table->name, table->subname); + + if (table->sec_berkeleydb == NULL) { + berkeleydb = table->berkeleydb; + } else { + berkeleydb = table->sec_berkeleydb; + } + + if (berkeleydb == NULL) { + assert (table->readonly); + *cursor_p = NULL; + return RET_NOTHING; + } + + cursor = zNEW(struct cursor); + if (FAILEDTOALLOC(cursor)) + return RET_ERROR_OOM; + + cursor->cursor = NULL; + cursor->flags = flags; + cursor->r = RET_OK; + dbret = berkeleydb->cursor(berkeleydb, NULL, + &cursor->cursor, 0); + if (dbret != 0) { + table_printerror(table, dbret, "cursor"); + free(cursor); + return RET_DBERR(dbret); + } + *cursor_p = cursor; + return RET_OK; +} + +retvalue table_newglobalcursor(struct table *table, bool duplicate, struct cursor **cursor_p) { + retvalue r; + + r = newcursor(table, duplicate ? DB_NEXT : DB_NEXT_NODUP, cursor_p); + if (r == RET_NOTHING) { + // table_newglobalcursor returned RET_OK when table->berkeleydb == NULL. Is that return value wanted? + r = RET_OK; + } + return r; +} + +static inline retvalue parse_data(struct table *table, DBT Key, DBT Data, /*@null@*//*@out@*/const char **key_p, /*@out@*/const char **data_p, /*@out@*/size_t *datalen_p) { + if (Key.size <= 0 || Data.size <= 0 || + ((const char*)Key.data)[Key.size-1] != '\0' || + ((const char*)Data.data)[Data.size-1] != '\0') { + if (table->subname != NULL) + fprintf(stderr, +"Database %s(%s) returned corrupted (not null-terminated) data!", + table->name, table->subname); + else + fprintf(stderr, +"Database %s returned corrupted (not null-terminated) data!", + table->name); + return RET_ERROR; + } + if (key_p != NULL) + *key_p = Key.data; + *data_p = Data.data; + if (datalen_p != NULL) + *datalen_p = Data.size - 1; + return RET_OK; +} + +static inline retvalue parse_pair(struct table *table, DBT Key, DBT Data, /*@null@*//*@out@*/const char **key_p, /*@out@*/const char **value_p, /*@out@*/const char **data_p, /*@out@*/size_t *datalen_p) { + /*@dependant@*/ const char *separator; + + if (Key.size == 0 || Data.size == 0 || + ((const char*)Key.data)[Key.size-1] != '\0' || + ((const char*)Data.data)[Data.size-1] != '\0') { + if (table->subname != NULL) + fprintf(stderr, +"Database %s(%s) returned corrupted (not null-terminated) data!", + table->name, table->subname); + else + fprintf(stderr, +"Database %s returned corrupted (not null-terminated) data!", + table->name); + return RET_ERROR; + } + separator = memchr(Data.data, '\0', Data.size-1); + if (separator == NULL) { + if (table->subname != NULL) + fprintf(stderr, +"Database %s(%s) returned corrupted data!\n", + table->name, table->subname); + else + fprintf(stderr, +"Database %s returned corrupted data!\n", + table->name); + return RET_ERROR; + } + if (key_p != NULL) + *key_p = Key.data; + *value_p = Data.data; + *data_p = separator + 1; + *datalen_p = Data.size - (separator - (const char*)Data.data) - 2; + return RET_OK; +} + +retvalue table_newduplicatecursor(struct table *table, const char *key, long long skip, struct cursor **cursor_p, const char **key_p, const char **data_p, size_t *datalen_p) { + struct cursor *cursor; + int dbret; + DBT Key, Data; + retvalue r; + + r = newcursor(table, DB_NEXT_DUP, &cursor); + if(!RET_IS_OK(r)) { + return r; + } + SETDBT(Key, key); + CLEARDBT(Data); + dbret = cursor->cursor->c_get(cursor->cursor, &Key, &Data, DB_SET); + if (dbret == DB_NOTFOUND || dbret == DB_KEYEMPTY) { + (void)cursor->cursor->c_close(cursor->cursor); + free(cursor); + return RET_NOTHING; + } + if (dbret != 0) { + table_printerror(table, dbret, "c_get(DB_SET)"); + (void)cursor->cursor->c_close(cursor->cursor); + free(cursor); + return RET_DBERR(dbret); + } + + while (skip > 0) { + CLEARDBT(Key); + CLEARDBT(Data); + + dbret = cursor->cursor->c_get(cursor->cursor, &Key, &Data, cursor->flags); + if (dbret == DB_NOTFOUND) { + (void)cursor->cursor->c_close(cursor->cursor); + free(cursor); + return RET_NOTHING; + } + if (dbret != 0) { + table_printerror(table, dbret, "c_get(DB_NEXT_DUP)"); + (void)cursor->cursor->c_close(cursor->cursor); + free(cursor); + return RET_DBERR(dbret); + } + + skip--; + } + + r = parse_data(table, Key, Data, key_p, data_p, datalen_p); + if (RET_WAS_ERROR(r)) { + (void)cursor->cursor->c_close(cursor->cursor); + free(cursor); + return r; + } + *cursor_p = cursor; + return RET_OK; +} + +retvalue table_newduplicatepairedcursor(struct table *table, const char *key, struct cursor **cursor_p, const char **value_p, const char **data_p, size_t *datalen_p) { + struct cursor *cursor; + int dbret; + DBT Key, Data; + retvalue r; + + r = newcursor(table, DB_NEXT_DUP, cursor_p); + if(!RET_IS_OK(r)) { + return r; + } + cursor = *cursor_p; + SETDBT(Key, key); + CLEARDBT(Data); + dbret = cursor->cursor->c_get(cursor->cursor, &Key, &Data, DB_SET); + if (dbret == DB_NOTFOUND || dbret == DB_KEYEMPTY) { + (void)cursor->cursor->c_close(cursor->cursor); + free(cursor); + return RET_NOTHING; + } + if (dbret != 0) { + table_printerror(table, dbret, "c_get(DB_SET)"); + (void)cursor->cursor->c_close(cursor->cursor); + free(cursor); + return RET_DBERR(dbret); + } + r = parse_pair(table, Key, Data, NULL, value_p, data_p, datalen_p); + assert (r != RET_NOTHING); + if (RET_WAS_ERROR(r)) { + (void)cursor->cursor->c_close(cursor->cursor); + free(cursor); + return r; + } + + *cursor_p = cursor; + return RET_OK; +} + +retvalue table_newpairedcursor(struct table *table, const char *key, const char *value, struct cursor **cursor_p, const char **data_p, size_t *datalen_p) { + struct cursor *cursor; + int dbret; + DBT Key, Data; + retvalue r; + size_t valuelen = strlen(value); + + /* cursor_next is not allowed with this type: */ + r = newcursor(table, DB_GET_BOTH, cursor_p); + if(!RET_IS_OK(r)) { + return r; + } + cursor = *cursor_p; + SETDBT(Key, key); + SETDBTl(Data, value, valuelen + 1); + dbret = cursor->cursor->c_get(cursor->cursor, &Key, &Data, DB_GET_BOTH); + if (dbret != 0) { + if (dbret == DB_NOTFOUND || dbret == DB_KEYEMPTY) { + table_printerror(table, dbret, "c_get(DB_GET_BOTH)"); + r = RET_DBERR(dbret); + } else + r = RET_NOTHING; + (void)cursor->cursor->c_close(cursor->cursor); + free(cursor); + return r; + } + if (Data.size < valuelen + 2 || + ((const char*)Data.data)[Data.size-1] != '\0') { + if (table->subname != NULL) + fprintf(stderr, +"Database %s(%s) returned corrupted (not paired) data!", + table->name, table->subname); + else + fprintf(stderr, +"Database %s returned corrupted (not paired) data!", + table->name); + (void)cursor->cursor->c_close(cursor->cursor); + free(cursor); + return RET_ERROR; + } + if (data_p != NULL) + *data_p = ((const char*)Data.data) + valuelen + 1; + if (datalen_p != NULL) + *datalen_p = Data.size - valuelen - 2; + *cursor_p = cursor; + return RET_OK; +} + +retvalue cursor_close(struct table *table, struct cursor *cursor) { + int dbret; + retvalue r; + + if (cursor == NULL) + return RET_OK; + + r = cursor->r; + dbret = cursor->cursor->c_close(cursor->cursor); + cursor->cursor = NULL; + free(cursor); + if (dbret != 0) { + table_printerror(table, dbret, "c_close"); + RET_UPDATE(r, RET_DBERR(dbret)); + } + return r; +} + +static bool cursor_next(struct table *table, struct cursor *cursor, DBT *Key, DBT *Data) { + int dbret; + + if (cursor == NULL) + return false; + + CLEARDBT(*Key); + CLEARDBT(*Data); + + dbret = cursor->cursor->c_get(cursor->cursor, Key, Data, + cursor->flags); + if (dbret == DB_NOTFOUND) + return false; + + if (dbret != 0) { + table_printerror(table, dbret, + (cursor->flags==DB_NEXT) + ? "c_get(DB_NEXT)" + : (cursor->flags==DB_NEXT_DUP) + ? "c_get(DB_NEXT_DUP)" + : "c_get(DB_???NEXT)"); + cursor->r = RET_DBERR(dbret); + return false; + } + return true; +} + +bool cursor_nexttempdata(struct table *table, struct cursor *cursor, const char **key, const char **data, size_t *len_p) { + DBT Key, Data; + bool success; + retvalue r; + + success = cursor_next(table, cursor, &Key, &Data); + if (!success) + return false; + r = parse_data(table, Key, Data, key, data, len_p); + if (RET_WAS_ERROR(r)) { + cursor->r = r; + return false; + } + return true; +} + +bool cursor_nextpair(struct table *table, struct cursor *cursor, /*@null@*/const char **key_p, const char **value_p, const char **data_p, size_t *datalen_p) { + DBT Key, Data; + bool success; + retvalue r; + + success = cursor_next(table, cursor, &Key, &Data); + if (!success) + return false; + r = parse_pair(table, Key, Data, key_p, value_p, data_p, datalen_p); + if (RET_WAS_ERROR(r)) { + cursor->r = r; + return false; + } + return true; +} + +retvalue cursor_replace(struct table *table, struct cursor *cursor, const char *data, size_t datalen) { + DBT Key, Data; + int dbret; + + assert (cursor != NULL); + assert (!table->readonly); + + CLEARDBT(Key); + SETDBTl(Data, data, datalen + 1); + + dbret = cursor->cursor->c_put(cursor->cursor, &Key, &Data, DB_CURRENT); + + if (dbret != 0) { + table_printerror(table, dbret, "c_put(DB_CURRENT)"); + return RET_DBERR(dbret); + } + return RET_OK; +} + +retvalue cursor_delete(struct table *table, struct cursor *cursor, const char *key, const char *value) { + int dbret; + + assert (cursor != NULL); + assert (!table->readonly); + + dbret = cursor->cursor->c_del(cursor->cursor, 0); + + if (dbret != 0) { + table_printerror(table, dbret, "c_del"); + return RET_DBERR(dbret); + } + if (table->verbose) { + if (value != NULL) + if (table->subname != NULL) + printf("db: '%s' '%s' removed from %s(%s).\n", + key, value, + table->name, table->subname); + else + printf("db: '%s' '%s' removed from %s.\n", + key, value, table->name); + else + if (table->subname != NULL) + printf("db: '%s' removed from %s(%s).\n", + key, table->name, table->subname); + else + printf("db: '%s' removed from %s.\n", + key, table->name); + } + return RET_OK; +} + +static bool table_isempty(struct table *table) { + DBC *cursor; + DBT Key, Data; + int dbret; + + dbret = table->berkeleydb->cursor(table->berkeleydb, NULL, + &cursor, 0); + if (dbret != 0) { + table_printerror(table, dbret, "cursor"); + return true; + } + CLEARDBT(Key); + CLEARDBT(Data); + + dbret = cursor->c_get(cursor, &Key, &Data, DB_NEXT); + if (dbret == DB_NOTFOUND) { + (void)cursor->c_close(cursor); + return true; + } + if (dbret != 0) { + table_printerror(table, dbret, "c_get(DB_NEXT)"); + (void)cursor->c_close(cursor); + return true; + } + dbret = cursor->c_close(cursor); + if (dbret != 0) + table_printerror(table, dbret, "c_close"); + return false; +} + +retvalue database_haspackages(const char *identifier) { + struct table *packages; + retvalue r; + bool empty; + + r = database_openpackages(identifier, true, &packages); + if (RET_WAS_ERROR(r)) + return r; + empty = table_isempty(packages); + (void)table_close(packages); + return empty?RET_NOTHING:RET_OK; +} + +/**************************************************************************** + * Open the different types of tables with their needed flags: * + ****************************************************************************/ +static retvalue database_table_secondary(const char *filename, const char *subtable, enum database_type type, uint32_t flags, + const char *secondary_filename, enum database_type secondary_type, /*@out@*/struct table **table_p) { + struct table *table; + struct opened_tables *opened_table; + retvalue r; + + if (verbose >= 15) + fprintf(stderr, "trace: database_table_secondary(filename=%s, subtable=%s, type=%i, flags=%u, secondary_filename=%s, secondary_type=%i) called.\n", + filename, subtable, type, flags, secondary_filename, secondary_type); + + for (struct opened_tables *iter = opened_tables; iter != NULL; iter = iter->next) { + if(strcmp2(iter->name, filename) == 0 && strcmp2(iter->subname, subtable) == 0) { + fprintf(stderr, + "Internal Error: Trying to open table '%s' from file '%s' multiple times.\n" + "This should normally not happen (to avoid triggering bugs in the underlying BerkeleyDB)\n", + subtable, filename); + return RET_ERROR; + } + } + + table = zNEW(struct table); + if (FAILEDTOALLOC(table)) + return RET_ERROR_OOM; + /* TODO: is filename always an static constant? then we could drop the dup */ + table->name = strdup(filename); + if (FAILEDTOALLOC(table->name)) { + free(table); + return RET_ERROR_OOM; + } + if (subtable != NULL) { + table->subname = strdup(subtable); + if (FAILEDTOALLOC(table->subname)) { + free(table->name); + free(table); + return RET_ERROR_OOM; + } + } else + table->subname = NULL; + table->readonly = ISSET(flags, DB_RDONLY); + table->verbose = rdb_verbose; + table->flags = flags; + r = database_opentable(filename, subtable, type, flags, + &table->berkeleydb); + if (RET_WAS_ERROR(r)) { + free(table->subname); + free(table->name); + free(table); + return r; + } + if (r == RET_NOTHING) { + if (ISSET(flags, DB_RDONLY)) { + /* sometimes we don't want a return here, when? */ + table->berkeleydb = NULL; + r = RET_OK; + } else { + free(table->subname); + free(table->name); + free(table); + return r; + } + + } + + if (secondary_filename != NULL) { + r = database_opentable(secondary_filename, subtable, secondary_type, flags, + &table->sec_berkeleydb); + if (RET_WAS_ERROR(r)) { + table->berkeleydb->close(table->berkeleydb, 0); + free(table->subname); + free(table->name); + free(table); + return r; + } + if (r == RET_NOTHING) { + if (ISSET(flags, DB_RDONLY)) { + /* sometimes we don't want a return here, when? */ + table->sec_berkeleydb = NULL; + r = RET_OK; + } else { + table->berkeleydb->close(table->berkeleydb, 0); + free(table->subname); + free(table->name); + free(table); + return r; + } + + } + } + + opened_table = zNEW(struct opened_tables); + if (FAILEDTOALLOC(opened_table)) { + free(table->subname); + free(table->name); + free(table); + return RET_ERROR_OOM; + } + opened_table->name = table->name; + opened_table->subname = table->subname; + opened_table->next = opened_tables; + opened_tables = opened_table; + + if (verbose >= 25) + print_opened_tables(stderr); + + *table_p = table; + return r; +} + +static retvalue database_table(const char *filename, const char *subtable, enum database_type type, uint32_t flags, /*@out@*/struct table **table_p) { + return database_table_secondary(filename, subtable, type, flags, NULL, 0, table_p); +} + +retvalue database_openreferences(void) { + retvalue r; + + assert (rdb_references == NULL); + r = database_table("references.db", "references", + dbt_BTREEDUP, DB_CREATE, &rdb_references); + assert (r != RET_NOTHING); + if (RET_WAS_ERROR(r)) { + rdb_references = NULL; + return r; + } else + rdb_references->verbose = false; + return RET_OK; +} + +static int debianversioncompare(UNUSED(DB *db), const DBT *a, const DBT *b) { + const char *a_version; + const char *b_version; + int versioncmp; + // There is no way to indicate an error to the caller + // Thus return -1 in case of an error + retvalue r = -1; + + if (a->size == 0 || ((char*)a->data)[a->size-1] != '\0') { + fprintf(stderr, "Database value '%.*s' empty or not NULL terminated.\n", a->size, (char*)a->data); + return r; + } + if (b->size == 0 || ((char*)b->data)[b->size-1] != '\0') { + fprintf(stderr, "Database value '%.*s' empty or not NULL terminated.\n", b->size, (char*)b->data); + return r; + } + + a_version = strchr(a->data, '|'); + if (a_version == NULL) { + fprintf(stderr, "Database value '%s' malformed. It should be 'package|version'.\n", (char*)a->data); + return r; + } + a_version++; + b_version = strchr(b->data, '|'); + if (b_version == NULL) { + fprintf(stderr, "Database value '%s' malformed. It should be 'package|version'.\n", (char*)b->data); + return r; + } + b_version++; + + r = dpkgversions_cmp(a_version, b_version, &versioncmp); + if (RET_WAS_ERROR(r)) { + fprintf(stderr, "Parse errors processing versions.\n"); + return r; + } + + return -versioncmp; +} + +/* only compare the first 0-terminated part of the data */ +static int paireddatacompare(UNUSED(DB *db), const DBT *a, const DBT *b +#if DB_VERSION_MAJOR >= 6 +#warning Berkeley DB >= 6.0 is not yet tested and highly experimental + , UNUSED(size_t *locp) + /* "The locp parameter is currently unused, and must be set to NULL or corruption can occur." + * What the ...! How am I supposed to handle that? */ +#endif +) { + if (a->size < b->size) + return strncmp(a->data, b->data, a->size); + else + return strncmp(a->data, b->data, b->size); +} + +retvalue database_opentracking(const char *codename, bool readonly, struct table **table_p) { + struct table *table; + retvalue r; + + if (rdb_nopackages) { + (void)fputs( +"Internal Error: Accessing packages database while that was not prepared!\n", + stderr); + return RET_ERROR; + } + + r = database_table("tracking.db", codename, + dbt_BTREEPAIRS, readonly?DB_RDONLY:DB_CREATE, &table); + assert (r != RET_NOTHING); + if (RET_WAS_ERROR(r)) + return r; + *table_p = table; + return RET_OK; +} + +static int get_package_name(DB *secondary, const DBT *pkey, const DBT *pdata, DBT *skey) { + const char *separator; + size_t length; + + separator = memchr(pkey->data, '|', pkey->size); + if (unlikely(separator == NULL)) { + return DB_MALFORMED_KEY; + } + + length = (size_t)separator - (size_t)pkey->data; + skey->flags = DB_DBT_APPMALLOC; + skey->data = strndup(pkey->data, length); + if (FAILEDTOALLOC(skey->data)) { + return RET_ERROR_OOM; + } + skey->size = length + 1; + return 0; +} + +static retvalue database_translate_legacy_packages(void) { + struct cursor *databases_cursor, *cursor; + struct table *legacy_databases, *legacy_table, *packages; + const char *chunk, *packagename; + char *identifier, *key, *legacy_filename, *packages_filename, *packageversion; + retvalue r, result; + int ret, e; + size_t chunk_len; + DBT Key, Data; + + if (verbose >= 15) + fprintf(stderr, "trace: database_translate_legacy_packages() called.\n"); + + if (!isdir(global.dbdir)) { + fprintf(stderr, "Cannot find directory '%s'!\n", global.dbdir); + return RET_ERROR; + } + + packages_filename = dbfilename("packages.db"); + legacy_filename = dbfilename("packages.legacy.db"); + ret = rename(packages_filename, legacy_filename); + if (ret != 0) { + e = errno; + fprintf(stderr, "error %d renaming %s to %s: %s\n", + e, packages_filename, legacy_filename, strerror(e)); + return (e != 0)?e:EINVAL; + } + if (verbose >= 15) + fprintf(stderr, "trace: Moved '%s' to '%s'.\n", packages_filename, legacy_filename); + + r = database_table("packages.legacy.db", NULL, dbt_BTREE, DB_RDONLY, &legacy_databases); + assert (r != RET_NOTHING); + if (RET_WAS_ERROR(r)) + return r; + + r = table_newglobalcursor(legacy_databases, true, &databases_cursor); + assert (r != RET_NOTHING); + if (RET_WAS_ERROR(r)) { + (void)table_close(legacy_databases); + return r; + } + result = RET_NOTHING; + // Iterate over all databases inside the packages.db file. + while (cursor_next(legacy_databases, databases_cursor, &Key, &Data)) { + identifier = strndup(Key.data, Key.size); + if (FAILEDTOALLOC(identifier)) { + RET_UPDATE(result, RET_ERROR_OOM); + break; + } + if (verbose >= 15) + fprintf(stderr, "Converting table '%s' to new layout...\n", identifier); + + r = database_table("packages.legacy.db", identifier, dbt_BTREE, DB_RDONLY, &legacy_table); + assert (r != RET_NOTHING); + if (RET_WAS_ERROR(r)) { + free(identifier); + RET_UPDATE(result, r); + break; + } + + r = table_newglobalcursor(legacy_table, true, &cursor); + assert (r != RET_NOTHING); + if (RET_WAS_ERROR(r)) { + (void)table_close(legacy_table); + free(identifier); + RET_UPDATE(result, r); + break; + } + + r = database_openpackages(identifier, false, &packages); + free(identifier); + identifier = NULL; + if (RET_WAS_ERROR(r)) { + (void)cursor_close(legacy_databases, databases_cursor); + (void)table_close(legacy_table); + RET_UPDATE(result, r); + break; + } + + while (cursor_nexttempdata(legacy_table, cursor, &packagename, &chunk, &chunk_len)) { + r = chunk_getvalue(chunk, "Version", &packageversion); + if (!RET_IS_OK(r)) { + RET_UPDATE(result, r); + break; + } + key = package_primarykey(packagename, packageversion); + r = table_addrecord(packages, key, chunk, chunk_len, false); + free(key); + if (RET_WAS_ERROR(r)) { + RET_UPDATE(result, r); + break; + } + } + + r = table_close(packages); + RET_UPDATE(result, r); + r = cursor_close(legacy_table, cursor); + RET_UPDATE(result, r); + r = table_close(legacy_table); + RET_UPDATE(result, r); + + if (RET_WAS_ERROR(result)) { + break; + } + result = RET_OK; + } + r = cursor_close(legacy_databases, databases_cursor); + RET_ENDUPDATE(result, r); + r = table_close(legacy_databases); + RET_ENDUPDATE(result, r); + + if (RET_IS_OK(result)) { + e = deletefile(legacy_filename); + if (e != 0) { + fprintf(stderr, "Could not delete '%s'!\n" +"It can now safely be deleted and it all that is left to be done!\n", + legacy_filename); + return RET_ERRNO(e); + } + } + + return result; +} + +retvalue database_openpackages(const char *identifier, bool readonly, struct table **table_p) { + struct table *table; + retvalue r; + + if (rdb_nopackages) { + (void)fputs( +"Internal Error: Accessing packages database while that was not prepared!\n", + stderr); + return RET_ERROR; + } + + r = database_table_secondary("packages.db", identifier, + dbt_BTREE, readonly?DB_RDONLY:DB_CREATE, + "packagenames.db", dbt_BTREEVERSIONS, &table); + assert (r != RET_NOTHING); + if (RET_WAS_ERROR(r)) + return r; + + if (table->berkeleydb != NULL && table->sec_berkeleydb == NULL) { + r = table_close(table); + if (RET_WAS_ERROR(r)) { + return r; + } + r = database_translate_legacy_packages(); + if (RET_WAS_ERROR(r)) { + return r; + } + return database_openpackages(identifier, readonly, table_p); + } + + if (table->berkeleydb != NULL && table->sec_berkeleydb != NULL) { + r = table->berkeleydb->associate(table->berkeleydb, NULL, + table->sec_berkeleydb, get_package_name, 0); + if (RET_WAS_ERROR(r)) { + return r; + } + } + + *table_p = table; + return RET_OK; +} + +/* Get a list of all identifiers having a package list */ +retvalue database_listpackages(struct strlist *identifiers) { + return database_listsubtables("packages.db", identifiers); +} + +/* drop a database */ +retvalue database_droppackages(const char *identifier) { + retvalue r; + + r = database_dropsubtable("packages.db", identifier); + if (RET_IS_OK(r)) + r = database_dropsubtable("packagenames.db", identifier); + return r; +} + +retvalue database_openfiles(void) { + retvalue r; + struct strlist identifiers; + bool oldfiles; + + assert (rdb_checksums == NULL); + assert (rdb_contents == NULL); + + r = database_listsubtables("contents.cache.db", &identifiers); + if (RET_IS_OK(r)) { + if (strlist_in(&identifiers, "filelists")) { + fprintf(stderr, +"Your %s/contents.cache.db file still contains a table of cached file lists\n" +"in the old (pre 3.0.0) format. You have to either delete that file (and lose\n" +"all caches of file lists) or run reprepro with argument translatefilelists\n" +"to translate the old caches into the new format.\n", + global.dbdir); + strlist_done(&identifiers); + return RET_ERROR; + } + strlist_done(&identifiers); + } + + r = database_table("checksums.db", "pool", + dbt_BTREE, DB_CREATE, + &rdb_checksums); + assert (r != RET_NOTHING); + if (RET_WAS_ERROR(r)) { + rdb_checksums = NULL; + return r; + } + r = database_hasdatabasefile("files.db", &oldfiles); + if (RET_WAS_ERROR(r)) { + (void)table_close(rdb_checksums); + rdb_checksums = NULL; + return r; + } + if (oldfiles) { + fprintf(stderr, +"Error: database uses deprecated format.\n" +"Please run translatelegacychecksums to update to the new format first.\n"); + return RET_ERROR; + } + + // TODO: only create this file once it is actually needed... + r = database_table("contents.cache.db", "compressedfilelists", + dbt_BTREE, DB_CREATE, &rdb_contents); + assert (r != RET_NOTHING); + if (RET_WAS_ERROR(r)) { + (void)table_close(rdb_checksums); + rdb_checksums = NULL; + rdb_contents = NULL; + } + return r; +} + +retvalue database_openreleasecache(const char *codename, struct table **cachedb_p) { + retvalue r; + char *oldcachefilename; + + /* Since 3.1.0 it's release.caches.db, before release.cache.db. + * The new file also contains the sha1 checksums and is extensible + * for more in the future. Thus if there is only the old variant, + * rename to the new. (So no old version by accident uses it and + * puts the additional sha1 data into the md5sum fields.) + * If both files are there, just delete both, as neither will + * be very current then. + * */ + + oldcachefilename = dbfilename("release.cache.db"); + if (FAILEDTOALLOC(oldcachefilename)) + return RET_ERROR_OOM; + if (isregularfile(oldcachefilename)) { + char *newcachefilename; + + newcachefilename = dbfilename("release.caches.db"); + if (FAILEDTOALLOC(newcachefilename)) { + free(oldcachefilename); + return RET_ERROR_OOM; + } + if (isregularfile(newcachefilename) + || rename(oldcachefilename, newcachefilename) != 0) { + fprintf(stderr, +"Deleting old-style export cache file %s!\n" +"This means that all index files (even unchanged) will be rewritten the\n" +"next time parts of their distribution are changed. This should only\n" +"happen once while migration from pre-3.1.0 to later versions.\n", + oldcachefilename); + + if (unlink(oldcachefilename) != 0) { + int e = errno; + fprintf(stderr, "Cannot delete '%s': %s!", + oldcachefilename, + strerror(e)); + free(oldcachefilename); + free(newcachefilename); + return RET_ERRNO(e); + } + (void)unlink(oldcachefilename); + } + free(newcachefilename); + } + free(oldcachefilename); + + r = database_table("release.caches.db", codename, + dbt_HASH, DB_CREATE, cachedb_p); + if (RET_IS_OK(r)) + (*cachedb_p)->verbose = false; + return r; +} + +static retvalue table_copy(struct table *oldtable, struct table *newtable) { + retvalue r; + struct cursor *cursor; + const char *filekey, *data; + size_t data_len; + + r = table_newglobalcursor(oldtable, true, &cursor); + if (!RET_IS_OK(r)) + return r; + while (cursor_nexttempdata(oldtable, cursor, &filekey, + &data, &data_len)) { + r = table_adduniqsizedrecord(newtable, filekey, + data, data_len+1, false, true); + if (RET_WAS_ERROR(r)) + return r; + } + return RET_OK; +} + +retvalue database_translate_filelists(void) { + char *dbname, *tmpdbname; + struct table *oldtable, *newtable; + struct strlist identifiers; + int ret; + retvalue r, r2; + + r = database_listsubtables("contents.cache.db", &identifiers); + if (RET_IS_OK(r)) { + if (!strlist_in(&identifiers, "filelists")) { + fprintf(stderr, +"Your %s/contents.cache.db file does not contain an old style database!\n", + global.dbdir); + strlist_done(&identifiers); + return RET_NOTHING; + } + strlist_done(&identifiers); + } + + dbname = dbfilename("contents.cache.db"); + if (FAILEDTOALLOC(dbname)) + return RET_ERROR_OOM; + tmpdbname = dbfilename("old.contents.cache.db"); + if (FAILEDTOALLOC(tmpdbname)) { + free(dbname); + return RET_ERROR_OOM; + } + ret = rename(dbname, tmpdbname); + if (ret != 0) { + int e = errno; + fprintf(stderr, "Could not rename '%s' into '%s': %s(%d)\n", + dbname, tmpdbname, strerror(e), e); + free(dbname); + free(tmpdbname); + return RET_ERRNO(e); + } + newtable = NULL; + r = database_table("contents.cache.db", "compressedfilelists", + dbt_BTREE, DB_CREATE, &newtable); + assert (r != RET_NOTHING); + oldtable = NULL; + if (RET_IS_OK(r)) { + r = database_table("old.contents.cache.db", "filelists", + dbt_BTREE, DB_RDONLY, &oldtable); + if (r == RET_NOTHING) { + fprintf(stderr, "Could not find old-style database!\n"); + r = RET_ERROR; + } + } + if (RET_IS_OK(r)) { + r = filelists_translate(oldtable, newtable); + if (r == RET_NOTHING) + r = RET_OK; + } + r2 = table_close(oldtable); + RET_ENDUPDATE(r, r2); + oldtable = NULL; + if (RET_IS_OK(r)) { + /* copy the new-style database, */ + r = database_table("old.contents.cache.db", "compressedfilelists", + dbt_BTREE, DB_RDONLY, &oldtable); + if (RET_IS_OK(r)) { + /* if there is one... */ + r = table_copy(oldtable, newtable); + r2 = table_close(oldtable); + RET_ENDUPDATE(r, r2); + } + if (r == RET_NOTHING) { + r = RET_OK; + } + } + r2 = table_close(newtable); + RET_ENDUPDATE(r, r2); + if (RET_IS_OK(r)) + (void)unlink(tmpdbname); + + if (RET_WAS_ERROR(r)) { + ret = rename(tmpdbname, dbname); + if (ret != 0) { + int e = errno; + fprintf(stderr, +"Could not rename '%s' back into '%s': %s(%d)\n", + dbname, tmpdbname, strerror(e), e); + free(tmpdbname); + free(dbname); + return RET_ERRNO(e); + } + free(tmpdbname); + free(dbname); + return r; + } + free(tmpdbname); + free(dbname); + return RET_OK; +} + +/* This is already implemented as standalone functions duplicating a bit + * of database_create and from files.c, + * because database_create is planed to error out if * there is still an old + * files.db and files.c is supposed to lose all support for it in the next + * major version */ + +static inline retvalue translate(struct table *oldmd5sums, struct table *newchecksums) { + long numold = 0, numnew = 0, numreplace = 0, numretro = 0; + struct cursor *cursor, *newcursor; + const char *filekey, *md5sum, *all; + size_t alllen; + retvalue r; + + /* first add all md5sums to checksums if not there yet */ + + r = table_newglobalcursor(oldmd5sums, true, &cursor); + if (RET_WAS_ERROR(r)) + return r; + while (cursor_nexttempdata(oldmd5sums, cursor, + &filekey, &md5sum, NULL)) { + struct checksums *n = NULL; + const char *combined; + size_t combinedlen; + + r = table_gettemprecord(newchecksums, filekey, + &all, &alllen); + if (RET_IS_OK(r)) + r = checksums_setall(&n, all, alllen); + if (RET_IS_OK(r)) { + if (checksums_matches(n, cs_md5sum, md5sum)) { + /* already there, nothing to do */ + checksums_free(n); + numnew++; + continue; + } + /* new item does not match */ + if (verbose > 0) + printf( +"Overwriting stale new-checksums entry '%s'!\n", + filekey); + numreplace++; + checksums_free(n); + n = NULL; + } + if (RET_WAS_ERROR(r)) { + (void)cursor_close(oldmd5sums, cursor); + return r; + } + /* parse and recreate, to only have sanitized strings + * in the database */ + r = checksums_parse(&n, md5sum); + assert (r != RET_NOTHING); + if (RET_WAS_ERROR(r)) { + (void)cursor_close(oldmd5sums, cursor); + return r; + } + + r = checksums_getcombined(n, &combined, &combinedlen); + assert (r != RET_NOTHING); + if (!RET_IS_OK(r)) { + (void)cursor_close(oldmd5sums, cursor); + return r; + } + numold++; + r = table_adduniqsizedrecord(newchecksums, filekey, + combined, combinedlen + 1, true, false); + assert (r != RET_NOTHING); + if (!RET_IS_OK(r)) { + (void)cursor_close(oldmd5sums, cursor); + return r; + } + } + r = cursor_close(oldmd5sums, cursor); + if (RET_WAS_ERROR(r)) + return r; + + /* then delete everything from checksums that is not in md5sums */ + + r = table_newglobalcursor(oldmd5sums, true, &cursor); + if (RET_WAS_ERROR(r)) + return r; + r = table_newglobalcursor(newchecksums, true, &newcursor); + if (RET_WAS_ERROR(r)) { + cursor_close(oldmd5sums, cursor); + return r; + } + while (cursor_nexttempdata(oldmd5sums, cursor, + &filekey, &md5sum, NULL)) { + bool more; + int cmp; + const char *newfilekey, *dummy; + + do { + more = cursor_nexttempdata(newchecksums, newcursor, + &newfilekey, &dummy, NULL); + /* should have been added in the last step */ + assert (more); + cmp = strcmp(filekey, newfilekey); + /* should have been added in the last step */ + assert (cmp >= 0); + more = cmp > 0; + if (more) { + numretro++; + if (verbose > 0) + printf( +"Deleting stale new-checksums entry '%s'!\n", + newfilekey); + r = cursor_delete(newchecksums, newcursor, + newfilekey, dummy); + if (RET_WAS_ERROR(r)) { + cursor_close(oldmd5sums, cursor); + cursor_close(newchecksums, newcursor); + return r; + } + } + } while (more); + } + r = cursor_close(oldmd5sums, cursor); + if (RET_WAS_ERROR(r)) + return r; + r = cursor_close(newchecksums, newcursor); + if (RET_WAS_ERROR(r)) + return r; + if (verbose >= 0) { + printf("%ld packages were already in the new checksums.db\n", + numnew); + printf("%ld packages were added to the new checksums.db\n", + numold - numreplace); + if (numretro != 0) + printf( +"%ld were only in checksums.db and not in files.db\n" +"This should only have happened if you added them with a newer version\n" +"and then deleted them with an older version of reprepro.\n", + numretro); + if (numreplace != 0) + printf( +"%ld were different checksums.db and not in files.db\n" +"This should only have happened if you added them with a newer version\n" +"and then deleted them with an older version of reprepro and\n" +"then readded them with a old version.\n", + numreplace); + if (numretro != 0 || numreplace != 0) + printf( +"If you never run a old version after a new version,\n" +"you might want to check with check and checkpool if something went wrong.\n"); + } + return RET_OK; +} + +retvalue database_translate_legacy_checksums(bool verbosedb) { + struct table *newchecksums, *oldmd5sums; + char *fullfilename; + retvalue r; + int e; + + if (rdb_initialized || rdb_used) { + fputs("Internal Error: database initialized a 2nd time!\n", + stderr); + return RET_ERROR_INTERNAL; + } + + if (!isdir(global.dbdir)) { + fprintf(stderr, "Cannot find directory '%s'!\n", + global.dbdir); + return RET_ERROR; + } + + rdb_initialized = true; + rdb_used = true; + + r = database_lock(0); + assert (r != RET_NOTHING); + if (!RET_IS_OK(r)) { + database_free(); + return r; + } + rdb_readonly = READWRITE; + rdb_verbose = verbosedb; + + r = readversionfile(false); + if (RET_WAS_ERROR(r)) { + releaselock(); + database_free(); + return r; + } + + r = database_table("files.db", "md5sums", dbt_BTREE, 0, &oldmd5sums); + if (r == RET_NOTHING) { + fprintf(stderr, +"There is no old files.db in %s. Nothing to translate!\n", + global.dbdir); + releaselock(); + database_free(); + return RET_NOTHING; + } else if (RET_WAS_ERROR(r)) { + releaselock(); + database_free(); + return r; + } + + r = database_table("checksums.db", "pool", dbt_BTREE, DB_CREATE, + &newchecksums); + assert (r != RET_NOTHING); + if (RET_WAS_ERROR(r)) { + (void)table_close(oldmd5sums); + releaselock(); + database_free(); + return r; + } + + r = translate(oldmd5sums, newchecksums); + if (RET_WAS_ERROR(r)) { + (void)table_close(oldmd5sums); + (void)table_close(newchecksums); + releaselock(); + database_free(); + return r; + } + + (void)table_close(oldmd5sums); + r = table_close(newchecksums); + if (RET_WAS_ERROR(r)) { + releaselock(); + database_free(); + return r; + } + fullfilename = dbfilename("files.db"); + if (FAILEDTOALLOC(fullfilename)) { + releaselock(); + database_free(); + return RET_ERROR_OOM; + } + e = deletefile(fullfilename); + if (e != 0) { + fprintf(stderr, "Could not delete '%s'!\n" +"It can now safely be deleted and it all that is left to be done!\n", + fullfilename); + database_free(); + return RET_ERRNO(e); + } + r = writeversionfile(); + releaselock(); + database_free(); + return r; +} |