Adding upstream version 2.4.59.upstream/2.4.59upstream

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

1 files changed, 801 insertions, 482 deletions

diff --git a/modules/md/md_reg.c b/modules/md/md_reg.c
index 233fea7..8bceb0e 100644
--- a/modules/md/md_reg.c
+++ b/modules/md/md_reg.c
@@ -26,21 +26,37 @@
 
 #include "md.h"
 #include "md_crypt.h"
+#include "md_event.h"
 #include "md_log.h"
 #include "md_json.h"
+#include "md_result.h"
 #include "md_reg.h"
 #include "md_store.h"
+#include "md_status.h"
+#include "md_tailscale.h"
 #include "md_util.h"
 
 #include "md_acme.h"
 #include "md_acme_acct.h"
 
 struct md_reg_t {
+    apr_pool_t *p;
     struct md_store_t *store;
     struct apr_hash_t *protos;
+    struct apr_hash_t *certs;
     int can_http;
     int can_https;
     const char *proxy_url;
+    const char *ca_file;
+    int domains_frozen;
+    md_timeslice_t *renew_window;
+    md_timeslice_t *warn_window;
+    md_job_notify_cb *notify;
+    void *notify_ctx;
+    apr_time_t min_delay;
+    int retry_failover;
+    int use_store_locks;
+    apr_time_t lock_wait_timeout;
 };
 
 /**************************************************************************************************/
@@ -67,20 +83,34 @@ static apr_status_t load_props(md_reg_t *reg, apr_pool_t *p)
     return rv;
 }
 
-apr_status_t md_reg_init(md_reg_t **preg, apr_pool_t *p, struct md_store_t *store,
-                         const char *proxy_url)
+apr_status_t md_reg_create(md_reg_t **preg, apr_pool_t *p, struct md_store_t *store,
+                           const char *proxy_url, const char *ca_file,
+                           apr_time_t min_delay, int retry_failover,
+                           int use_store_locks, apr_time_t lock_wait_timeout)
 {
     md_reg_t *reg;
     apr_status_t rv;
     
     reg = apr_pcalloc(p, sizeof(*reg));
+    reg->p = p;
     reg->store = store;
     reg->protos = apr_hash_make(p);
+    reg->certs = apr_hash_make(p);
     reg->can_http = 1;
     reg->can_https = 1;
     reg->proxy_url = proxy_url? apr_pstrdup(p, proxy_url) : NULL;
+    reg->ca_file = (ca_file && apr_strnatcasecmp("none", ca_file))?
+                    apr_pstrdup(p, ca_file) : NULL;
+    reg->min_delay = min_delay;
+    reg->retry_failover = retry_failover;
+    reg->use_store_locks = use_store_locks;
+    reg->lock_wait_timeout = lock_wait_timeout;
+
+    md_timeslice_create(&reg->renew_window, p, MD_TIME_LIFE_NORM, MD_TIME_RENEW_WINDOW_DEF); 
+    md_timeslice_create(&reg->warn_window, p, MD_TIME_LIFE_NORM, MD_TIME_WARN_WINDOW_DEF); 
     
-    if (APR_SUCCESS == (rv = md_acme_protos_add(reg->protos, p))) {
+    if (APR_SUCCESS == (rv = md_acme_protos_add(reg->protos, p))
+        && APR_SUCCESS == (rv = md_tailscale_protos_add(reg->protos, p))) {
         rv = load_props(reg, p);
     }
     
@@ -114,7 +144,7 @@ static apr_status_t check_values(md_reg_t *reg, apr_pool_t *p, const md_t *md, i
         
         for (i = 0; i < md->domains->nelts; ++i) {
             domain = APR_ARRAY_IDX(md->domains, i, const char *);
-            if (!md_util_is_dns_name(p, domain, 1)) {
+            if (!md_dns_is_name(p, domain, 1) && !md_dns_is_wildcard(p, domain)) {
                 md_log_perror(MD_LOG_MARK, MD_LOG_ERR, APR_EINVAL, p, 
                               "md %s with invalid domain name: %s", md->name, domain);
                 return APR_EINVAL;
@@ -145,12 +175,17 @@ static apr_status_t check_values(md_reg_t *reg, apr_pool_t *p, const md_t *md, i
         }
     }
     
-    if ((MD_UPD_CA_URL & fields) && md->ca_url) { /* setting to empty is ok */
-        rv = md_util_abs_uri_check(p, md->ca_url, &err);
-        if (err) {
-            md_log_perror(MD_LOG_MARK, MD_LOG_ERR, APR_EINVAL, p, 
-                          "CA url for %s invalid (%s): %s", md->name, err, md->ca_url);
-            return APR_EINVAL;
+    if ((MD_UPD_CA_URL & fields) && md->ca_urls) { /* setting to empty is ok */
+        int i;
+        const char *url;
+        for (i = 0; i < md->ca_urls->nelts; ++i) {
+            url = APR_ARRAY_IDX(md->ca_urls, i, const char*);
+            rv = md_util_abs_uri_check(p, url, &err);
+            if (err) {
+                md_log_perror(MD_LOG_MARK, MD_LOG_ERR, APR_EINVAL, p,
+                              "CA url for %s invalid (%s): %s", md->name, err, url);
+                return APR_EINVAL;
+            }
         }
     }
     
@@ -162,7 +197,8 @@ static apr_status_t check_values(md_reg_t *reg, apr_pool_t *p, const md_t *md, i
         /* hmm, in case we know the protocol, some checks could be done */
     }
 
-    if ((MD_UPD_AGREEMENT & fields) && md->ca_agreement) { /* setting to empty is ok */
+    if ((MD_UPD_AGREEMENT & fields) && md->ca_agreement
+        && strcmp("accepted", md->ca_agreement)) { /* setting to empty is ok */
         rv = md_util_abs_uri_check(p, md->ca_agreement, &err);
         if (err) {
             md_log_perror(MD_LOG_MARK, MD_LOG_ERR, APR_EINVAL, p, 
@@ -177,138 +213,72 @@ static apr_status_t check_values(md_reg_t *reg, apr_pool_t *p, const md_t *md, i
 /**************************************************************************************************/
 /* state assessment */
 
-static apr_status_t state_init(md_reg_t *reg, apr_pool_t *p, md_t *md, int save_changes)
+static apr_status_t state_init(md_reg_t *reg, apr_pool_t *p, md_t *md)
 {
-    md_state_t state = MD_S_UNKNOWN;
-    const md_creds_t *creds;
+    md_state_t state = MD_S_COMPLETE;
+    const char *state_descr = NULL;
+    const md_pubcert_t *pub;
     const md_cert_t *cert;
-    apr_time_t expires = 0, valid_from = 0;
-    apr_status_t rv;
+    const md_pkey_spec_t *spec;
+    apr_status_t rv = APR_SUCCESS;
     int i;
 
-    if (APR_SUCCESS == (rv = md_reg_creds_get(&creds, reg, MD_SG_DOMAINS, md, p))) {
-        state = MD_S_INCOMPLETE;
-        if (!creds->privkey) {
-            md_log_perror(MD_LOG_MARK, MD_LOG_DEBUG, rv, p, 
-                          "md{%s}: incomplete, without private key", md->name);
-        }
-        else if (!creds->cert) {
-            md_log_perror(MD_LOG_MARK, MD_LOG_DEBUG, rv, p, 
-                          "md{%s}: incomplete, has key but no certificate", md->name);
-        }
-        else {
-            valid_from = md_cert_get_not_before(creds->cert);
-            expires = md_cert_get_not_after(creds->cert);
-            if (md_cert_has_expired(creds->cert)) {
-                state = MD_S_EXPIRED;
-                md_log_perror(MD_LOG_MARK, MD_LOG_DEBUG, rv, p, 
-                              "md{%s}: expired, certificate has expired", md->name);
-                goto out;
-            }
-            if (!md_cert_is_valid_now(creds->cert)) {
-                state = MD_S_ERROR;
-                md_log_perror(MD_LOG_MARK, MD_LOG_ERR, rv, p, 
-                              "md{%s}: error, certificate valid in future (clock wrong?)", 
-                              md->name);
-                goto out;
-            }
-            if (!md_cert_covers_md(creds->cert, md)) {
+    if (md->renew_window == NULL) md->renew_window = reg->renew_window;
+    if (md->warn_window == NULL) md->warn_window = reg->warn_window;
+
+    if (md->domains && md->domains->pool != p) {
+        md_log_perror(MD_LOG_MARK, MD_LOG_ERR, 0, p,
+                      "md{%s}: state_init called with foreign pool", md->name);
+    }
+
+    for (i = 0; i < md_cert_count(md); ++i) {
+        spec = md_pkeys_spec_get(md->pks, i);
+        md_log_perror(MD_LOG_MARK, MD_LOG_TRACE2, rv, p,
+                      "md{%s}: check cert %s", md->name, md_pkey_spec_name(spec));
+        rv = md_reg_get_pubcert(&pub, reg, md, i, p);
+        if (APR_SUCCESS == rv) {
+            cert = APR_ARRAY_IDX(pub->certs, 0, const md_cert_t*);
+            if (!md_is_covered_by_alt_names(md, pub->alt_names)) {
                 state = MD_S_INCOMPLETE;
-                md_log_perror(MD_LOG_MARK, MD_LOG_INFO, rv, p, 
-                              "md{%s}: incomplete, cert no longer covers all domains, "
-                              "needs sign up for a new certificate", md->name);
-                goto out;
+                state_descr = apr_psprintf(p, "certificate(%s) does not cover all domains.",
+                                           md_pkey_spec_name(spec));
+                goto cleanup;
             }
-            if (!md->must_staple != !md_cert_must_staple(creds->cert)) {
+            if (!md->must_staple != !md_cert_must_staple(cert)) {
                 state = MD_S_INCOMPLETE;
-                md_log_perror(MD_LOG_MARK, MD_LOG_INFO, rv, p, 
-                              "md{%s}: OCSP Stapling is%s requested, but certificate "
-                              "has it%s enabled. Need to get a new certificate.", md->name,
-                              md->must_staple? "" : " not", 
+                state_descr = apr_psprintf(p, "'must-staple' is%s requested, but "
+                              "certificate(%s) has it%s enabled.",
+                              md->must_staple? "" : " not",
+                              md_pkey_spec_name(spec),
                               !md->must_staple? "" : " not");
-                goto out;
+                goto cleanup;
             }
-
-            for (i = 1; i < creds->pubcert->nelts; ++i) {
-                cert = APR_ARRAY_IDX(creds->pubcert, i, const md_cert_t *);
-                if (!md_cert_is_valid_now(cert)) {
-                    state = MD_S_ERROR;
-                    md_log_perror(MD_LOG_MARK, MD_LOG_ERR, rv, p, 
-                                  "md{%s}: error, the certificate itself is valid, however the %d. "
-                                  "certificate in the chain is not valid now (clock wrong?).", 
-                                  md->name, i);
-                    goto out;
-                }
-            } 
-
-            state = MD_S_COMPLETE;
-            md_log_perror(MD_LOG_MARK, MD_LOG_DEBUG, rv, p, "md{%s}: is complete", md->name);
+            md_log_perror(MD_LOG_MARK, MD_LOG_DEBUG, rv, p, "md{%s}: certificate(%d) is ok",
+                          md->name, i);
+        }
+        else if (APR_STATUS_IS_ENOENT(rv)) {
+            state = MD_S_INCOMPLETE;
+            state_descr = apr_psprintf(p, "certificate(%s) is missing",
+                                       md_pkey_spec_name(spec));
+            rv = APR_SUCCESS;
+            goto cleanup;
+        }
+        else {
+            state = MD_S_ERROR;
+            state_descr = "error initializing";
+            md_log_perror(MD_LOG_MARK, MD_LOG_WARNING, rv, p, "md{%s}: error", md->name);
+            goto cleanup;
         }
     }
 
-out:    
-    if (APR_SUCCESS != rv) {
-        state = MD_S_ERROR;
-        md_log_perror(MD_LOG_MARK, MD_LOG_WARNING, rv, p, "md{%s}: error", md->name);
-    }
-    
-    if (save_changes && md->state == state
-        && md->valid_from == valid_from && md->expires == expires) {
-        save_changes = 0;
-    }
+cleanup:
+    md_log_perror(MD_LOG_MARK, MD_LOG_TRACE2, rv, p, "md{%s}: state=%d, %s",
+                  md->name, state, state_descr);
     md->state = state;
-    md->valid_from = valid_from;
-    md->expires = expires;
-    if (save_changes && APR_SUCCESS == rv) {
-        return md_save(reg->store, p, MD_SG_DOMAINS, md, 0);
-    }
+    md->state_descr = state_descr;
     return rv;
 }
 
-apr_status_t md_reg_assess(md_reg_t *reg, md_t *md, int *perrored, int *prenew, apr_pool_t *p)
-{
-    int renew = 0;
-    int errored = 0;
-    
-    (void)reg;
-    switch (md->state) {
-        case MD_S_UNKNOWN:
-            md_log_perror( MD_LOG_MARK, MD_LOG_ERR, 0, p, "md(%s): in unknown state.", md->name);
-            break;
-        case MD_S_ERROR:
-            md_log_perror( MD_LOG_MARK, MD_LOG_ERR, 0, p,  
-                         "md(%s): in error state, unable to drive forward. If unable to "
-                         " detect the cause, you may remove the staging or even domain "
-                         " sub-directory for this MD and start all over.", md->name);
-            errored = 1;
-            break;
-        case MD_S_COMPLETE:
-            if (!md->expires) {
-                md_log_perror( MD_LOG_MARK, MD_LOG_WARNING, 0, p,  
-                             "md(%s): looks complete, but has unknown expiration date.", md->name);
-                errored = 1;
-            }
-            else if (md->expires <= apr_time_now()) {
-                /* Maybe we hibernated in the meantime? */
-                md->state = MD_S_EXPIRED;
-                renew = 1;
-            }
-            else {
-                renew = md_should_renew(md);
-            }
-            break;
-        case MD_S_INCOMPLETE:
-        case MD_S_EXPIRED:
-            renew = 1;
-            break;
-        case MD_S_MISSING:
-            break;
-    }
-    *prenew = renew;
-    *perrored = errored;
-    return APR_SUCCESS;
-}
-
 /**************************************************************************************************/
 /* iteration */
 
@@ -326,7 +296,7 @@ static int reg_md_iter(void *baton, md_store_t *store, md_t *md, apr_pool_t *pte
     
     (void)store;
     if (!ctx->exclude || strcmp(ctx->exclude, md->name)) {
-        state_init(ctx->reg, ptemp, (md_t*)md, 1);
+        state_init(ctx->reg, ptemp, (md_t*)md);
         return ctx->cb(ctx->baton, ctx->reg, md);
     }
     return 1;
@@ -357,7 +327,7 @@ md_t *md_reg_get(md_reg_t *reg, const char *name, apr_pool_t *p)
     md_t *md;
     
     if (APR_SUCCESS == md_load(reg->store, MD_SG_DOMAINS, name, &md, p)) {
-        state_init(reg, p, md, 1);
+        state_init(reg, p, md);
         return md;
     }
     return NULL;
@@ -389,7 +359,7 @@ md_t *md_reg_find(md_reg_t *reg, const char *domain, apr_pool_t *p)
     
     md_reg_do(find_domain, &ctx, reg, p);
     if (ctx.md) {
-        state_init(reg, p, ctx.md, 1);
+        state_init(reg, p, ctx.md);
     }
     return ctx.md;
 }
@@ -427,23 +397,11 @@ md_t *md_reg_find_overlap(md_reg_t *reg, const md_t *md, const char **pdomain, a
         *pdomain = ctx.s;
     }
     if (ctx.md) {
-        state_init(reg, p, ctx.md, 1);
+        state_init(reg, p, ctx.md);
     }
     return ctx.md;
 }
 
-apr_status_t md_reg_get_cred_files(md_reg_t *reg, const md_t *md, apr_pool_t *p,
-                                   const char **pkeyfile, const char **pcertfile)
-{
-    apr_status_t rv;
-    
-    rv = md_store_get_fname(pkeyfile, reg->store, MD_SG_DOMAINS, md->name, MD_FN_PRIVKEY, p);
-    if (APR_SUCCESS == rv) {
-        rv = md_store_get_fname(pcertfile, reg->store, MD_SG_DOMAINS, md->name, MD_FN_PUBCERT, p);
-    }
-    return rv;
-}
-
 /**************************************************************************************************/
 /* manipulation */
 
@@ -452,19 +410,28 @@ static apr_status_t p_md_add(void *baton, apr_pool_t *p, apr_pool_t *ptemp, va_l
     md_reg_t *reg = baton;
     apr_status_t rv = APR_SUCCESS;
     md_t *md, *mine;
+    int do_check;
     
     md = va_arg(ap, md_t *);
+    do_check = va_arg(ap, int);
+
+    if (reg->domains_frozen) return APR_EACCES; 
     mine = md_clone(ptemp, md);
-    if (APR_SUCCESS == (rv = check_values(reg, ptemp, md, MD_UPD_ALL))
-        && APR_SUCCESS == (rv = state_init(reg, ptemp, mine, 0))
-        && APR_SUCCESS == (rv = md_save(reg->store, p, MD_SG_DOMAINS, mine, 1))) {
-    }
+    if (do_check && APR_SUCCESS != (rv = check_values(reg, ptemp, md, MD_UPD_ALL))) goto leave;
+    if (APR_SUCCESS != (rv = state_init(reg, ptemp, mine))) goto leave;
+    rv = md_save(reg->store, p, MD_SG_DOMAINS, mine, 1);
+leave:
     return rv;
 }
 
+static apr_status_t add_md(md_reg_t *reg, md_t *md, apr_pool_t *p, int do_checks)
+{
+    return md_util_pool_vdo(p_md_add, reg, p, md, do_checks, NULL);
+}
+
 apr_status_t md_reg_add(md_reg_t *reg, md_t *md, apr_pool_t *p)
 {
-    return md_util_pool_vdo(p_md_add, reg, p, md, NULL);
+    return add_md(reg, md, p, 1);
 }
 
 static apr_status_t p_md_update(void *baton, apr_pool_t *p, apr_pool_t *ptemp, va_list ap)
@@ -473,31 +440,34 @@ static apr_status_t p_md_update(void *baton, apr_pool_t *p, apr_pool_t *ptemp, v
     apr_status_t rv = APR_SUCCESS;
     const char *name;
     const md_t *md, *updates;
-    int fields;
+    int fields, do_checks;
     md_t *nmd;
     
     name = va_arg(ap, const char *);
     updates = va_arg(ap, const md_t *);
     fields = va_arg(ap, int);
+    do_checks = va_arg(ap, int);
     
     if (NULL == (md = md_reg_get(reg, name, ptemp))) {
         md_log_perror(MD_LOG_MARK, MD_LOG_DEBUG, APR_ENOENT, ptemp, "md %s", name);
         return APR_ENOENT;
     }
     
-    md_log_perror(MD_LOG_MARK, MD_LOG_DEBUG, 0, ptemp, "update md %s", name);
+    md_log_perror(MD_LOG_MARK, MD_LOG_DEBUG, 0, ptemp, "md[%s]: update store", name);
     
-    if (APR_SUCCESS != (rv = check_values(reg, ptemp, updates, fields))) {
+    if (do_checks && APR_SUCCESS != (rv = check_values(reg, ptemp, updates, fields))) {
         return rv;
     }
     
+    if (reg->domains_frozen) return APR_EACCES; 
     nmd = md_copy(ptemp, md);
     if (MD_UPD_DOMAINS & fields) {
         nmd->domains = updates->domains;
         md_log_perror(MD_LOG_MARK, MD_LOG_TRACE1, 0, ptemp, "update domains: %s", name);
     }
     if (MD_UPD_CA_URL & fields) {
-        nmd->ca_url = updates->ca_url;
+        nmd->ca_urls = (updates->ca_urls?
+                        apr_array_copy(p, updates->ca_urls) : NULL);
         md_log_perror(MD_LOG_MARK, MD_LOG_TRACE1, 0, ptemp, "update ca url: %s", name);
     }
     if (MD_UPD_CA_PROTO & fields) {
@@ -516,18 +486,17 @@ static apr_status_t p_md_update(void *baton, apr_pool_t *p, apr_pool_t *ptemp, v
         md_log_perror(MD_LOG_MARK, MD_LOG_TRACE1, 0, ptemp, "update agreement: %s", name);
         nmd->ca_agreement = updates->ca_agreement;
     }
-    if (MD_UPD_CERT_URL & fields) {
-        md_log_perror(MD_LOG_MARK, MD_LOG_TRACE1, 0, ptemp, "update cert url: %s", name);
-        nmd->cert_url = updates->cert_url;
-    }
     if (MD_UPD_DRIVE_MODE & fields) {
         md_log_perror(MD_LOG_MARK, MD_LOG_TRACE1, 0, ptemp, "update drive-mode: %s", name);
-        nmd->drive_mode = updates->drive_mode;
+        nmd->renew_mode = updates->renew_mode;
     }
     if (MD_UPD_RENEW_WINDOW & fields) {
         md_log_perror(MD_LOG_MARK, MD_LOG_TRACE1, 0, ptemp, "update renew-window: %s", name);
-        nmd->renew_norm = updates->renew_norm;
-        nmd->renew_window = updates->renew_window;
+        *nmd->renew_window = *updates->renew_window;
+    }
+    if (MD_UPD_WARN_WINDOW & fields) {
+        md_log_perror(MD_LOG_MARK, MD_LOG_TRACE1, 0, ptemp, "update warn-window: %s", name);
+        *nmd->warn_window = *updates->warn_window;
     }
     if (MD_UPD_CA_CHALLENGES & fields) {
         md_log_perror(MD_LOG_MARK, MD_LOG_TRACE1, 0, ptemp, "update ca challenges: %s", name);
@@ -536,10 +505,7 @@ static apr_status_t p_md_update(void *baton, apr_pool_t *p, apr_pool_t *ptemp, v
     }
     if (MD_UPD_PKEY_SPEC & fields) {
         md_log_perror(MD_LOG_MARK, MD_LOG_TRACE1, 0, ptemp, "update pkey spec: %s", name);
-        nmd->pkey_spec = NULL;
-        if (updates->pkey_spec) {
-            nmd->pkey_spec = apr_pmemdup(p, updates->pkey_spec, sizeof(md_pkey_spec_t));
-        }
+        nmd->pks = md_pkeys_spec_clone(p, updates->pks);
     }
     if (MD_UPD_REQUIRE_HTTPS & fields) {
         md_log_perror(MD_LOG_MARK, MD_LOG_TRACE1, 0, ptemp, "update require-https: %s", name);
@@ -553,118 +519,239 @@ static apr_status_t p_md_update(void *baton, apr_pool_t *p, apr_pool_t *ptemp, v
         md_log_perror(MD_LOG_MARK, MD_LOG_TRACE1, 0, ptemp, "update must-staple: %s", name);
         nmd->must_staple = updates->must_staple;
     }
+    if (MD_UPD_PROTO & fields) {
+        md_log_perror(MD_LOG_MARK, MD_LOG_TRACE1, 0, ptemp, "update proto: %s", name);
+        nmd->acme_tls_1_domains = updates->acme_tls_1_domains;
+    }
+    if (MD_UPD_STAPLING & fields) {
+        md_log_perror(MD_LOG_MARK, MD_LOG_TRACE1, 0, ptemp, "update stapling: %s", name);
+        nmd->stapling = updates->stapling;
+    }
     
     if (fields && APR_SUCCESS == (rv = md_save(reg->store, p, MD_SG_DOMAINS, nmd, 0))) {
-        rv = state_init(reg, ptemp, nmd, 0);
+        rv = state_init(reg, ptemp, nmd);
     }
     return rv;
 }
 
 apr_status_t md_reg_update(md_reg_t *reg, apr_pool_t *p, 
-                           const char *name, const md_t *md, int fields)
+                           const char *name, const md_t *md, int fields, 
+                           int do_checks)
 {
-    return md_util_pool_vdo(p_md_update, reg, p, name, md, fields, NULL);
+    return md_util_pool_vdo(p_md_update, reg, p, name, md, fields, do_checks, NULL);
 }
 
-/**************************************************************************************************/
-/* certificate related */
-
-static int ok_or_noent(apr_status_t rv) 
+apr_status_t md_reg_delete_acct(md_reg_t *reg, apr_pool_t *p, const char *acct_id) 
 {
-    return (APR_SUCCESS == rv || APR_ENOENT == rv);
+    apr_status_t rv = APR_SUCCESS;
+    
+    rv = md_store_remove(reg->store, MD_SG_ACCOUNTS, acct_id, MD_FN_ACCOUNT, p, 1);
+    if (APR_SUCCESS == rv) {
+        md_store_remove(reg->store, MD_SG_ACCOUNTS, acct_id, MD_FN_ACCT_KEY, p, 1);
+    }
+    return rv;
 }
 
-static apr_status_t creds_load(void *baton, apr_pool_t *p, apr_pool_t *ptemp, va_list ap)
+/**************************************************************************************************/
+/* certificate related */
+
+static apr_status_t pubcert_load(void *baton, apr_pool_t *p, apr_pool_t *ptemp, va_list ap)
 {
     md_reg_t *reg = baton;
-    md_pkey_t *privkey;
-    apr_array_header_t *pubcert;
-    md_creds_t *creds, **pcreds;
+    apr_array_header_t *certs;
+    md_pubcert_t *pubcert, **ppubcert;
     const md_t *md;
+    int index;
+    const md_cert_t *cert;
     md_cert_state_t cert_state;
     md_store_group_t group;
     apr_status_t rv;
     
-    pcreds = va_arg(ap, md_creds_t **);
+    ppubcert = va_arg(ap, md_pubcert_t **);
     group = (md_store_group_t)va_arg(ap, int);
     md = va_arg(ap, const md_t *);
+    index = va_arg(ap, int);
     
-    if (ok_or_noent(rv = md_pkey_load(reg->store, group, md->name, &privkey, p))
-        && ok_or_noent(rv = md_pubcert_load(reg->store, group, md->name, &pubcert, p))) {
-        rv = APR_SUCCESS;
-            
-        creds = apr_pcalloc(p, sizeof(*creds));
-        creds->privkey = privkey;
-        if (pubcert && pubcert->nelts > 0) {
-            creds->pubcert = pubcert;
-            creds->cert = APR_ARRAY_IDX(pubcert, 0, md_cert_t *);
-        }
-        if (creds->cert) {
-            switch ((cert_state = md_cert_state_get(creds->cert))) {
-                case MD_CERT_VALID:
-                    creds->expired = 0;
-                    break;
-                case MD_CERT_EXPIRED:
-                    creds->expired = 1;
-                    break;
-                default:
-                    md_log_perror(MD_LOG_MARK, MD_LOG_ERR, APR_EINVAL, ptemp, 
-                                  "md %s has unexpected cert state: %d", md->name, cert_state);
-                    rv = APR_ENOTIMPL;
-                    break;
-            }
-        }
+    if (md->cert_files && md->cert_files->nelts) {
+        rv = md_chain_fload(&certs, p, APR_ARRAY_IDX(md->cert_files, index, const char *));
+    }
+    else {
+        md_pkey_spec_t *spec = md_pkeys_spec_get(md->pks, index);;
+        rv = md_pubcert_load(reg->store, group, md->name, spec, &certs, p);
+    }
+    if (APR_SUCCESS != rv) goto leave;
+    if (certs->nelts == 0) {
+        rv = APR_ENOENT;
+        goto leave;
+    }
+
+    pubcert = apr_pcalloc(p, sizeof(*pubcert));
+    pubcert->certs = certs;
+    cert = APR_ARRAY_IDX(certs, 0, const md_cert_t *);
+    if (APR_SUCCESS != (rv = md_cert_get_alt_names(&pubcert->alt_names, cert, p))) goto leave;
+    switch ((cert_state = md_cert_state_get(cert))) {
+        case MD_CERT_VALID:
+        case MD_CERT_EXPIRED:
+            break;
+        default:
+            md_log_perror(MD_LOG_MARK, MD_LOG_ERR, APR_EINVAL, ptemp, 
+                          "md %s has unexpected cert state: %d", md->name, cert_state);
+            rv = APR_ENOTIMPL;
+            break;
     }
-    *pcreds = (APR_SUCCESS == rv)? creds : NULL;
+leave:
+    *ppubcert = (APR_SUCCESS == rv)? pubcert : NULL;
     return rv;
 }
 
-apr_status_t md_reg_creds_get(const md_creds_t **pcreds, md_reg_t *reg, 
-                              md_store_group_t group, const md_t *md, apr_pool_t *p)
+apr_status_t md_reg_get_pubcert(const md_pubcert_t **ppubcert, md_reg_t *reg, 
+                                const md_t *md, int i, apr_pool_t *p)
 {
     apr_status_t rv = APR_SUCCESS;
-    md_creds_t *creds;
-    
-    rv = md_util_pool_vdo(creds_load, reg, p, &creds, group, md, NULL);
-    *pcreds = (APR_SUCCESS == rv)? creds : NULL;
+    const md_pubcert_t *pubcert;
+    const char *name;
+
+    name = apr_psprintf(p, "%s[%d]", md->name, i);
+    pubcert = apr_hash_get(reg->certs, name, (apr_ssize_t)strlen(name));
+    if (!pubcert && !reg->domains_frozen) {
+        rv = md_util_pool_vdo(pubcert_load, reg, reg->p, &pubcert, MD_SG_DOMAINS, md, i, NULL);
+        if (APR_STATUS_IS_ENOENT(rv)) {
+            /* We cache it missing with an empty record */
+            pubcert = apr_pcalloc(reg->p, sizeof(*pubcert));
+        }
+        else if (APR_SUCCESS != rv) goto leave;
+        if (p != reg->p) name = apr_pstrdup(reg->p, name);
+        apr_hash_set(reg->certs, name, (apr_ssize_t)strlen(name), pubcert);
+    }
+leave:
+    if (APR_SUCCESS == rv && (!pubcert || !pubcert->certs)) {
+        rv = APR_ENOENT;
+    }
+    *ppubcert = (APR_SUCCESS == rv)? pubcert : NULL;
     return rv;
 }
 
-/**************************************************************************************************/
-/* synching */
+apr_status_t md_reg_get_cred_files(const char **pkeyfile, const char **pcertfile,
+                                   md_reg_t *reg, md_store_group_t group, 
+                                   const md_t *md, md_pkey_spec_t *spec, apr_pool_t *p)
+{
+    apr_status_t rv;
+    
+    rv = md_store_get_fname(pkeyfile, reg->store, group, md->name, md_pkey_filename(spec, p), p);
+    if (APR_SUCCESS != rv) return rv;
+    if (!md_file_exists(*pkeyfile, p)) return APR_ENOENT;
+    rv = md_store_get_fname(pcertfile, reg->store, group, md->name, md_chain_filename(spec, p), p);
+    if (APR_SUCCESS != rv) return rv;
+    if (!md_file_exists(*pcertfile, p)) return APR_ENOENT;
+    return APR_SUCCESS;
+}
 
-typedef struct {
-    apr_pool_t *p;
-    apr_array_header_t *store_mds;
-} sync_ctx;
+apr_time_t md_reg_valid_until(md_reg_t *reg, const md_t *md, apr_pool_t *p)
+{
+    const md_pubcert_t *pub;
+    const md_cert_t *cert;
+    int i;
+    apr_time_t t, valid_until = 0;
+    apr_status_t rv;
+    
+    for (i = 0; i < md_cert_count(md); ++i) {
+        rv = md_reg_get_pubcert(&pub, reg, md, i, p);
+        if (APR_SUCCESS == rv) {
+            cert = APR_ARRAY_IDX(pub->certs, 0, const md_cert_t*);
+            t = md_cert_get_not_after(cert);
+            if (valid_until == 0 || t < valid_until) {
+                valid_until = t;
+            }
+        }
+    }
+    return valid_until;
+}
 
-static int do_add_md(void *baton, md_store_t *store, md_t *md, apr_pool_t *ptemp)
+apr_time_t md_reg_renew_at(md_reg_t *reg, const md_t *md, apr_pool_t *p)
 {
-    sync_ctx *ctx = baton;
+    const md_pubcert_t *pub;
+    const md_cert_t *cert;
+    md_timeperiod_t certlife, renewal;
+    int i;
+    apr_time_t renew_at = 0;
+    apr_status_t rv;
+    
+    if (md->state == MD_S_INCOMPLETE) return apr_time_now();
+    for (i = 0; i < md_cert_count(md); ++i) {
+        rv = md_reg_get_pubcert(&pub, reg, md, i, p);
+        if (APR_STATUS_IS_ENOENT(rv)) return apr_time_now();
+        if (APR_SUCCESS == rv) {
+            cert = APR_ARRAY_IDX(pub->certs, 0, const md_cert_t*);
+            certlife.start = md_cert_get_not_before(cert);
+            certlife.end = md_cert_get_not_after(cert);
+
+            renewal = md_timeperiod_slice_before_end(&certlife, md->renew_window);
+            if (md_log_is_level(p, MD_LOG_TRACE1)) {
+                md_log_perror(MD_LOG_MARK, MD_LOG_TRACE2, 0, p, 
+                              "md[%s]: certificate(%d) valid[%s] renewal[%s]", 
+                              md->name, i,  
+                              md_timeperiod_print(p, &certlife),
+                              md_timeperiod_print(p, &renewal));
+            }
+            
+            if (renew_at == 0 || renewal.start < renew_at) {
+                renew_at = renewal.start; 
+            }
+        }
+    }
+    return renew_at;
+}
 
-    (void)store;
-    (void)ptemp;
-    APR_ARRAY_PUSH(ctx->store_mds, const md_t*) = md_clone(ctx->p, md);
-    return 1;
+int md_reg_should_renew(md_reg_t *reg, const md_t *md, apr_pool_t *p) 
+{
+    apr_time_t renew_at;
+    
+    renew_at = md_reg_renew_at(reg, md, p);
+    return renew_at && (renew_at <= apr_time_now());
 }
 
-static apr_status_t read_store_mds(md_reg_t *reg, sync_ctx *ctx)
+int md_reg_should_warn(md_reg_t *reg, const md_t *md, apr_pool_t *p)
 {
-    int rv;
+    const md_pubcert_t *pub;
+    const md_cert_t *cert;
+    md_timeperiod_t certlife, warn;
+    int i;
+    apr_status_t rv;
     
-    apr_array_clear(ctx->store_mds);
-    rv = md_store_md_iter(do_add_md, ctx, reg->store, ctx->p, MD_SG_DOMAINS, "*");
-    if (APR_STATUS_IS_ENOENT(rv)) {
-        rv = APR_SUCCESS;
+    if (md->state == MD_S_INCOMPLETE) return 0;
+    for (i = 0; i < md_cert_count(md); ++i) {
+        rv = md_reg_get_pubcert(&pub, reg, md, i, p);
+        if (APR_STATUS_IS_ENOENT(rv)) return 0;
+        if (APR_SUCCESS == rv) {
+            cert = APR_ARRAY_IDX(pub->certs, 0, const md_cert_t*);
+            certlife.start = md_cert_get_not_before(cert);
+            certlife.end = md_cert_get_not_after(cert);
+            
+            warn = md_timeperiod_slice_before_end(&certlife, md->warn_window);
+            if (md_log_is_level(p, MD_LOG_TRACE1)) {
+                md_log_perror(MD_LOG_MARK, MD_LOG_TRACE2, 0, p, 
+                              "md[%s]: certificate(%d) life[%s] warn[%s]", 
+                              md->name, i,  
+                              md_timeperiod_print(p, &certlife),
+                              md_timeperiod_print(p, &warn));
+            }
+            if (md_timeperiod_has_started(&warn, apr_time_now())) {
+                return 1;
+            }
+        }
     }
-    return rv;
+    return 0;
 }
 
+/**************************************************************************************************/
+/* syncing */
+
 apr_status_t md_reg_set_props(md_reg_t *reg, apr_pool_t *p, int can_http, int can_https)
 {
     if (reg->can_http != can_http || reg->can_https != can_https) {
         md_json_t *json;
         
+        if (reg->domains_frozen) return APR_EACCES; 
         reg->can_http = can_http;
         reg->can_https = can_https;
         
@@ -676,329 +763,561 @@ apr_status_t md_reg_set_props(md_reg_t *reg, apr_pool_t *p, int can_http, int ca
     }
     return APR_SUCCESS;
 }
- 
-/**
- * Procedure:
- * 1. Collect all defined "managed domains" (MD). It does not matter where a MD is defined. 
- *    All MDs need to be unique and have no overlaps in their domain names. 
- *    Fail the config otherwise. Also, if a vhost matches an MD, it
- *    needs to *only* have ServerAliases from that MD. There can be no more than one
- *    matching MD for a vhost. But an MD can apply to several vhosts.
- * 2. Synchronize with the persistent store. Iterate over all configured MDs and 
- *   a. create them in the store if they do not already exist, neither under the
- *      name or with a common domain.
- *   b. compare domain lists from store and config, if
- *      - store has dns name in other MD than from config, remove dns name from store def,
- *        issue WARNING.
- *      - store misses dns name from config, add dns name and update store
- *   c. compare MD acme url/protocol, update if changed
+
+static md_t *find_closest_match(apr_array_header_t *mds, const md_t *md)
+{
+    md_t *candidate, *m;
+    apr_size_t cand_n, n;
+    int i;
+    
+    candidate = md_get_by_name(mds, md->name);
+    if (!candidate) {
+        /* try to find an instance that contains all domain names from md */ 
+        for (i = 0; i < mds->nelts; ++i) {
+            m = APR_ARRAY_IDX(mds, i, md_t *);
+            if (md_contains_domains(m, md)) {
+                return m;
+            }
+        }
+        /* no matching name and no md in the list has all domains.
+         * We consider that managed domain as closest match that contains at least one
+         * domain name from md, ONLY if there is no other one that also has.
+         */
+        cand_n = 0;
+        for (i = 0; i < mds->nelts; ++i) {
+            m = APR_ARRAY_IDX(mds, i, md_t *);
+            n = md_common_name_count(md, m);
+            if (n > cand_n) {
+                candidate = m;
+                cand_n = n;
+            }
+        }
+    }
+    return candidate;
+}
+
+typedef struct {
+    apr_pool_t *p;
+    apr_array_header_t *master_mds;
+    apr_array_header_t *store_names;
+    apr_array_header_t *maybe_new_mds;
+    apr_array_header_t *new_mds;
+    apr_array_header_t *unassigned_mds;
+} sync_ctx_v2;
+
+static int iter_add_name(void *baton, const char *dir, const char *name, 
+                         md_store_vtype_t vtype, void *value, apr_pool_t *ptemp)
+{
+    sync_ctx_v2 *ctx = baton;
+    
+    (void)dir;
+    (void)value;
+    (void)ptemp;
+    (void)vtype;
+    APR_ARRAY_PUSH(ctx->store_names, const char*) = apr_pstrdup(ctx->p, name);
+    return APR_SUCCESS;
+}
+
+/* A better scaling version:
+ *  1. The consistency of the MDs in 'master_mds' has already been verified. E.g.
+ *     that no domain lists overlap etc.
+ *  2. All MD storage that exists will be overwritten by the settings we have.
+ *     And "exists" meaning that "store/MD_SG_DOMAINS/name" exists.
+ *  3. For MDs that have no directory in "store/MD_SG_DOMAINS", we load all MDs
+ *     outside the list of known names from MD_SG_DOMAINS. In this list, we
+ *     look for the MD with the most domain overlap. 
+ *      - if we find it, we assume this is a rename and move the old MD to the new name.
+ *      - if not, MD is completely new.
+ *  4. Any MD in store that does not match the "master_mds" will just be left as is. 
  */
-apr_status_t md_reg_sync(md_reg_t *reg, apr_pool_t *p, apr_pool_t *ptemp, 
-                         apr_array_header_t *master_mds) 
+apr_status_t md_reg_sync_start(md_reg_t *reg, apr_array_header_t *master_mds, apr_pool_t *p) 
 {
-    sync_ctx ctx;
+    sync_ctx_v2 ctx;
     apr_status_t rv;
-
-    ctx.p = ptemp;
-    ctx.store_mds = apr_array_make(ptemp,100, sizeof(md_t *));
-    rv = read_store_mds(reg, &ctx);
+    md_t *md, *oldmd;
+    const char *name;
+    int i, idx;
     
-    md_log_perror(MD_LOG_MARK, MD_LOG_DEBUG, rv, p, 
-                  "sync: found %d mds in store", ctx.store_mds->nelts);
-    if (APR_SUCCESS == rv) {
-        int i, fields;
-        md_t *md, *config_md, *smd, *omd;
-        const char *common;
-        
-        for (i = 0; i < master_mds->nelts; ++i) {
-            md = APR_ARRAY_IDX(master_mds, i, md_t *);
-            
-            /* find the store md that is closest match for the configured md */
-            smd = md_find_closest_match(ctx.store_mds, md);
-            if (smd) {
-                fields = 0;
-                
-                /* Once stored, we keep the name */
-                if (strcmp(md->name, smd->name)) {
-                    md->name = apr_pstrdup(p, smd->name);
-                }
-                
-                /* Make the stored domain list *exactly* the same, even if
-                 * someone only changed upper/lowercase, we'd like to persist that. */
-                if (!md_equal_domains(md, smd, 1)) {
-                    md_log_perror(MD_LOG_MARK, MD_LOG_DEBUG, rv, p, 
-                                 "%s: domains changed", smd->name);
-                    smd->domains = md_array_str_clone(ptemp, md->domains);
-                    fields |= MD_UPD_DOMAINS;
-                }
-                
-                /* Look for other store mds which have domains now being part of smd */
-                while (APR_SUCCESS == rv && (omd = md_get_by_dns_overlap(ctx.store_mds, md))) {
-                    /* find the name now duplicate */
-                    common = md_common_name(md, omd);
-                    assert(common);
-                    
-                    /* Is this md still configured or has it been abandoned in the config? */
-                    config_md = md_get_by_name(master_mds, omd->name);
-                    if (config_md && md_contains(config_md, common, 0)) {
-                        /* domain used in two configured mds, not allowed */
-                        rv = APR_EINVAL;
-                        md_log_perror(MD_LOG_MARK, MD_LOG_ERR, rv, p, 
-                                      "domain %s used in md %s and %s", 
-                                      common, md->name, omd->name);
-                    }
-                    else {
-                        /* remove it from the other md and update store, or, if it
-                         * is now empty, move it into the archive */
-                        omd->domains = md_array_str_remove(ptemp, omd->domains, common, 0);
-                        if (apr_is_empty_array(omd->domains)) {
-                            md_log_perror(MD_LOG_MARK, MD_LOG_WARNING, rv, p, 
-                                          "All domains of the MD %s have moved elsewhere, "
-                                          " moving it to the archive. ", omd->name);
-                            md_reg_remove(reg, ptemp, omd->name, 1); /* best effort */
-                        }
-                        else {
-                            rv = md_reg_update(reg, ptemp, omd->name, omd, MD_UPD_DOMAINS);
-                        }
-                    }
-                }
-
-                if (MD_SVAL_UPDATE(md, smd, ca_url)) {
-                    smd->ca_url = md->ca_url;
-                    fields |= MD_UPD_CA_URL;
-                }
-                if (MD_SVAL_UPDATE(md, smd, ca_proto)) {
-                    smd->ca_proto = md->ca_proto;
-                    fields |= MD_UPD_CA_PROTO;
-                }
-                if (MD_SVAL_UPDATE(md, smd, ca_agreement)) {
-                    smd->ca_agreement = md->ca_agreement;
-                    fields |= MD_UPD_AGREEMENT;
-                }
-                if (MD_VAL_UPDATE(md, smd, transitive)) {
-                    smd->transitive = md->transitive;
-                    fields |= MD_UPD_TRANSITIVE;
-                }
-                if (MD_VAL_UPDATE(md, smd, drive_mode)) {
-                    smd->drive_mode = md->drive_mode;
-                    fields |= MD_UPD_DRIVE_MODE;
-                }
-                if (!apr_is_empty_array(md->contacts) 
-                    && !md_array_str_eq(md->contacts, smd->contacts, 0)) {
-                    smd->contacts = md->contacts;
-                    fields |= MD_UPD_CONTACTS;
-                }
-                if (MD_VAL_UPDATE(md, smd, renew_window) 
-                    || MD_VAL_UPDATE(md, smd, renew_norm)) {
-                    md_log_perror(MD_LOG_MARK, MD_LOG_DEBUG, rv, p, 
-                                  "%s: update renew norm=%ld, window=%ld", 
-                                  smd->name, (long)md->renew_norm, (long)md->renew_window);
-                    smd->renew_norm = md->renew_norm;
-                    smd->renew_window = md->renew_window;
-                    fields |= MD_UPD_RENEW_WINDOW;
-                }
-                if (md->ca_challenges) {
-                    md->ca_challenges = md_array_str_compact(p, md->ca_challenges, 0);
-                    if (!smd->ca_challenges 
-                        || !md_array_str_eq(md->ca_challenges, smd->ca_challenges, 0)) {
-                        smd->ca_challenges = apr_array_copy(ptemp, md->ca_challenges);
-                        fields |= MD_UPD_CA_CHALLENGES;
-                    }
-                }
-                else if (smd->ca_challenges) {
-                    smd->ca_challenges = NULL;
-                    fields |= MD_UPD_CA_CHALLENGES;
-                }
-                if (!md_pkey_spec_eq(md->pkey_spec, smd->pkey_spec)) {
-                    fields |= MD_UPD_PKEY_SPEC;
-                    smd->pkey_spec = NULL;
-                    if (md->pkey_spec) {
-                        smd->pkey_spec = apr_pmemdup(p, md->pkey_spec, sizeof(md_pkey_spec_t));
-                    }
-                }
-                if (MD_VAL_UPDATE(md, smd, require_https)) {
-                    smd->require_https = md->require_https;
-                    fields |= MD_UPD_REQUIRE_HTTPS;
-                }
-                if (MD_VAL_UPDATE(md, smd, must_staple)) {
-                    smd->must_staple = md->must_staple;
-                    fields |= MD_UPD_MUST_STAPLE;
-                }
-                
-                if (fields) {
-                    rv = md_reg_update(reg, ptemp, smd->name, smd, fields);
-                    md_log_perror(MD_LOG_MARK, MD_LOG_DEBUG, rv, p, "md %s updated", smd->name);
-                }
-            }
-            else {
-                /* new managed domain */
-                rv = md_reg_add(reg, md, ptemp);
-                md_log_perror(MD_LOG_MARK, MD_LOG_DEBUG, rv, p, "new md %s added", md->name);
+    md_log_perror(MD_LOG_MARK, MD_LOG_DEBUG, 0, p, "sync MDs, start");
+     
+    ctx.p = p;
+    ctx.master_mds = master_mds;
+    ctx.store_names = apr_array_make(p, master_mds->nelts + 100, sizeof(const char*));
+    ctx.maybe_new_mds = apr_array_make(p, master_mds->nelts, sizeof(md_t*));
+    ctx.new_mds = apr_array_make(p, master_mds->nelts, sizeof(md_t*));
+    ctx.unassigned_mds = apr_array_make(p, master_mds->nelts, sizeof(md_t*));
+    
+    rv = md_store_iter_names(iter_add_name, &ctx, reg->store, p, MD_SG_DOMAINS, "*");
+    if (APR_SUCCESS != rv) {
+        md_log_perror(MD_LOG_MARK, MD_LOG_ERR, rv, p, "listing existing store MD names"); 
+        goto leave;
+    }
+    
+    /* Get all MDs that are not already present in store */
+    for (i = 0; i < ctx.master_mds->nelts; ++i) {
+        md = APR_ARRAY_IDX(ctx.master_mds, i, md_t*);
+        idx = md_array_str_index(ctx.store_names, md->name, 0, 1);
+        if (idx < 0) {
+            APR_ARRAY_PUSH(ctx.maybe_new_mds, md_t*) = md;
+            md_array_remove_at(ctx.store_names, idx);
+        }
+    }
+    
+    if (ctx.maybe_new_mds->nelts == 0) goto leave; /* none new */
+    if (ctx.store_names->nelts == 0) goto leave;   /* all new */
+    
+    md_log_perror(MD_LOG_MARK, MD_LOG_DEBUG, 0, p, 
+                  "sync MDs, %d potentially new MDs detected, looking for renames among "
+                  "the %d unassigned store domains", (int)ctx.maybe_new_mds->nelts,
+                  (int)ctx.store_names->nelts);
+    for (i = 0; i < ctx.store_names->nelts; ++i) {
+        name = APR_ARRAY_IDX(ctx.store_names, i, const char*);
+        if (APR_SUCCESS == md_load(reg->store, MD_SG_DOMAINS, name, &md, p)) {
+            APR_ARRAY_PUSH(ctx.unassigned_mds, md_t*) = md;
+        } 
+    }
+    
+    md_log_perror(MD_LOG_MARK, MD_LOG_DEBUG, 0, p, 
+                  "sync MDs, %d MDs maybe new, checking store", (int)ctx.maybe_new_mds->nelts);
+    for (i = 0; i < ctx.maybe_new_mds->nelts; ++i) {
+        md = APR_ARRAY_IDX(ctx.maybe_new_mds, i, md_t*);
+        oldmd = find_closest_match(ctx.unassigned_mds, md);
+        if (oldmd) {
+            /* found the rename, move the domains and possible staging directory */
+            md_log_perror(MD_LOG_MARK, MD_LOG_DEBUG, 0, p, 
+                          "sync MDs, found MD %s under previous name %s", md->name, oldmd->name);
+            rv = md_store_rename(reg->store, p, MD_SG_DOMAINS, oldmd->name, md->name);
+            if (APR_SUCCESS != rv) {
+                md_log_perror(MD_LOG_MARK, MD_LOG_ERR, 0, p, 
+                              "sync MDs, renaming MD %s to %s failed", oldmd->name, md->name);
+                /* ignore it? */
             }
+            md_store_rename(reg->store, p, MD_SG_STAGING, oldmd->name, md->name);
+            md_array_remove(ctx.unassigned_mds, oldmd);
+        }
+        else {
+            APR_ARRAY_PUSH(ctx.new_mds, md_t*) = md;
         }
     }
-    else {
-        md_log_perror(MD_LOG_MARK, MD_LOG_ERR, rv, p, "loading mds");
+
+leave:
+    md_log_perror(MD_LOG_MARK, MD_LOG_DEBUG, 0, p, 
+                  "sync MDs, %d existing, %d moved, %d new.", 
+                  (int)ctx.master_mds->nelts - ctx.maybe_new_mds->nelts,
+                  (int)ctx.maybe_new_mds->nelts - ctx.new_mds->nelts,
+                  (int)ctx.new_mds->nelts);
+    return rv;
+}
+
+/** 
+ * Finish syncing an MD with the store. 
+ * 1. if there are changed properties (or if the MD is new), save it.
+ * 2. read any existing certificate and init the state of the memory MD
+ */
+apr_status_t md_reg_sync_finish(md_reg_t *reg, md_t *md, apr_pool_t *p, apr_pool_t *ptemp)
+{
+    md_t *old;
+    apr_status_t rv;
+    int changed = 1;
+    md_proto_t *proto;
+    
+    if (!md->ca_proto) {
+        md->ca_proto = MD_PROTO_ACME;
     }
+    proto = apr_hash_get(reg->protos, md->ca_proto, (apr_ssize_t)strlen(md->ca_proto));
+    if (!proto) {
+        rv = APR_ENOTIMPL;
+        md_log_perror(MD_LOG_MARK, MD_LOG_ERR, rv, ptemp,
+                      "[%s] uses unknown CA protocol '%s'",
+                      md->name, md->ca_proto);
+        goto leave;
+    }
+    rv = proto->complete_md(md, p);
+    if (APR_SUCCESS != rv) goto leave;
+
+    rv = state_init(reg, p, md);
+    if (APR_SUCCESS != rv) goto leave;
     
+    md_log_perror(MD_LOG_MARK, MD_LOG_DEBUG, rv, ptemp, "loading md %s", md->name);
+    if (APR_SUCCESS == md_load(reg->store, MD_SG_DOMAINS, md->name, &old, ptemp)) {
+        md_log_perror(MD_LOG_MARK, MD_LOG_DEBUG, rv, ptemp, "loaded md %s", md->name);
+        /* Some parts are kept from old, lacking new values */
+        if ((!md->contacts || apr_is_empty_array(md->contacts)) && old->contacts) {
+            md->contacts = md_array_str_clone(p, old->contacts);
+        } 
+        if (md->ca_challenges && old->ca_challenges) {
+            if (!md_array_str_eq(md->ca_challenges, old->ca_challenges, 0)) {
+                md->ca_challenges = md_array_str_compact(p, md->ca_challenges, 0);
+            }
+        }
+        if (!md->ca_effective && old->ca_effective) {
+            md->ca_effective = apr_pstrdup(p, old->ca_effective);
+        }
+        if (!md->ca_account && old->ca_account) {
+            md->ca_account = apr_pstrdup(p, old->ca_account);
+        }
+        
+        /* if everything remains the same, spare the write back */
+        if (!MD_VAL_UPDATE(md, old, state)
+            && md_array_str_eq(md->ca_urls, old->ca_urls, 0)
+            && !MD_SVAL_UPDATE(md, old, ca_proto)
+            && !MD_SVAL_UPDATE(md, old, ca_agreement)
+            && !MD_VAL_UPDATE(md, old, transitive)
+            && md_equal_domains(md, old, 1)
+            && !MD_VAL_UPDATE(md, old, renew_mode)
+            && md_timeslice_eq(md->renew_window, old->renew_window)
+            && md_timeslice_eq(md->warn_window, old->warn_window)
+            && md_pkeys_spec_eq(md->pks, old->pks)
+            && !MD_VAL_UPDATE(md, old, require_https)
+            && !MD_VAL_UPDATE(md, old, must_staple)
+            && md_array_str_eq(md->acme_tls_1_domains, old->acme_tls_1_domains, 0)
+            && !MD_VAL_UPDATE(md, old, stapling)
+            && md_array_str_eq(md->contacts, old->contacts, 0)
+            && md_array_str_eq(md->cert_files, old->cert_files, 0)
+            && md_array_str_eq(md->pkey_files, old->pkey_files, 0)
+            && md_array_str_eq(md->ca_challenges, old->ca_challenges, 0)) {
+            changed = 0;
+        }
+    }
+    if (changed) {
+        md_log_perror(MD_LOG_MARK, MD_LOG_DEBUG, rv, ptemp, "saving md %s", md->name);
+        rv = md_save(reg->store, ptemp, MD_SG_DOMAINS, md, 0);
+    }
+leave:
+    md_log_perror(MD_LOG_MARK, MD_LOG_DEBUG, rv, ptemp, "sync MDs, finish done");
     return rv;
 }
 
 apr_status_t md_reg_remove(md_reg_t *reg, apr_pool_t *p, const char *name, int archive)
 {
+    if (reg->domains_frozen) return APR_EACCES; 
     return md_store_move(reg->store, p, MD_SG_DOMAINS, MD_SG_ARCHIVE, name, archive);
 }
 
+typedef struct {
+    md_reg_t *reg;
+    apr_pool_t *p;
+    apr_array_header_t *mds;
+} cleanup_challenge_ctx;
+ 
+static apr_status_t cleanup_challenge_inspector(void *baton, const char *dir, const char *name, 
+                                                md_store_vtype_t vtype, void *value, 
+                                                apr_pool_t *ptemp)
+{
+    cleanup_challenge_ctx *ctx = baton;
+    const md_t *md;
+    int i, used;
+    apr_status_t rv;
+    
+    (void)value;
+    (void)vtype;
+    (void)dir;
+    for (used = 0, i = 0; i < ctx->mds->nelts && !used; ++i) {
+        md = APR_ARRAY_IDX(ctx->mds, i, const md_t *);
+        used = !strcmp(name, md->name);
+    }
+    if (!used) {
+        md_log_perror(MD_LOG_MARK, MD_LOG_DEBUG, 0, ptemp, 
+                      "challenges/%s: not in use, purging", name);
+        rv = md_store_purge(ctx->reg->store, ctx->p, MD_SG_CHALLENGES, name);
+        if (APR_SUCCESS != rv) {
+            md_log_perror(MD_LOG_MARK, MD_LOG_WARNING, rv, ptemp, 
+                          "challenges/%s: unable to purge", name);
+        }
+    }
+    return APR_SUCCESS;
+}
+
+apr_status_t md_reg_cleanup_challenges(md_reg_t *reg, apr_pool_t *p, apr_pool_t *ptemp, 
+                                       apr_array_header_t *mds)
+{
+    apr_status_t rv;
+    cleanup_challenge_ctx ctx;
+
+    (void)p;
+    ctx.reg = reg;
+    ctx.p = ptemp;
+    ctx.mds = mds;
+    rv = md_store_iter_names(cleanup_challenge_inspector, &ctx, reg->store, ptemp, 
+                             MD_SG_CHALLENGES, "*");
+    return rv;
+}
+
 
 /**************************************************************************************************/
 /* driving */
 
-static apr_status_t init_proto_driver(md_proto_driver_t *driver, const md_proto_t *proto, 
-                                      md_reg_t *reg, const md_t *md, 
-                                      const char *challenge, int reset, apr_pool_t *p) 
+static apr_status_t run_init(void *baton, apr_pool_t *p, ...)
 {
-    apr_status_t rv = APR_SUCCESS;
+    va_list ap;
+    md_reg_t *reg = baton;
+    const md_t *md;
+    md_proto_driver_t *driver, **pdriver;
+    md_result_t *result;
+    apr_table_t *env;
+    const char *s;
+    int preload;
+    
+    (void)p;
+    va_start(ap, p);
+    pdriver = va_arg(ap, md_proto_driver_t **);
+    md = va_arg(ap, const md_t *);
+    preload = va_arg(ap, int);
+    env = va_arg(ap, apr_table_t *);
+    result = va_arg(ap, md_result_t *); 
+    va_end(ap);
+    
+    *pdriver = driver = apr_pcalloc(p, sizeof(*driver));
 
-    /* If this registry instance was not synched before (and obtained server
-     * properties that way), read them from the store.
-     */
-    driver->proto = proto;
     driver->p = p;
-    driver->challenge = challenge;
-    driver->can_http = reg->can_http;
-    driver->can_https = reg->can_https;
+    driver->env = env? apr_table_copy(p, env) : apr_table_make(p, 10);
     driver->reg = reg;
     driver->store = md_reg_store_get(reg);
     driver->proxy_url = reg->proxy_url;
+    driver->ca_file = reg->ca_file;
     driver->md = md;
-    driver->reset = reset;
+    driver->can_http = reg->can_http;
+    driver->can_https = reg->can_https;
+    
+    s = apr_table_get(driver->env, MD_KEY_ACTIVATION_DELAY);
+    if (!s || APR_SUCCESS != md_duration_parse(&driver->activation_delay, s, "d")) {
+        driver->activation_delay = 0;
+    }
 
-    return rv;
+    if (!md->ca_proto) {
+        md_result_printf(result, APR_EGENERAL, "CA protocol is not defined"); 
+        md_log_perror(MD_LOG_MARK, MD_LOG_WARNING, 0, p, "md[%s]: %s", md->name, result->detail);
+        goto leave;
+    }
+    
+    driver->proto = apr_hash_get(reg->protos, md->ca_proto, (apr_ssize_t)strlen(md->ca_proto));
+    if (!driver->proto) {
+        md_result_printf(result, APR_EGENERAL, "Unknown CA protocol '%s'", md->ca_proto); 
+        goto leave;
+    }
+    
+    if (preload) {
+        result->status = driver->proto->init_preload(driver, result);
+    }
+    else {
+        result->status = driver->proto->init(driver, result);
+    }
+
+leave:
+    if (APR_SUCCESS != result->status) {
+        md_log_perror(MD_LOG_MARK, MD_LOG_WARNING, result->status, p, "md[%s]: %s", md->name, 
+                      result->detail? result->detail : "<see error log for details>");
+    }
+    else {
+        md_log_perror(MD_LOG_MARK, MD_LOG_DEBUG, 0, p, "%s: init done", md->name);
+    }
+    return result->status;
+}
+
+static apr_status_t run_test_init(void *baton, apr_pool_t *p, apr_pool_t *ptemp, va_list ap)
+{
+    const md_t *md;
+    apr_table_t *env;
+    md_result_t *result;
+    md_proto_driver_t *driver;
+    
+    (void)p;
+    md = va_arg(ap, const md_t *);
+    env = va_arg(ap, apr_table_t *);
+    result = va_arg(ap, md_result_t *); 
+
+    return run_init(baton, ptemp, &driver, md, 0, env, result, NULL);
 }
 
-static apr_status_t run_stage(void *baton, apr_pool_t *p, apr_pool_t *ptemp, va_list ap)
+apr_status_t md_reg_test_init(md_reg_t *reg, const md_t *md, struct apr_table_t *env, 
+                              md_result_t *result, apr_pool_t *p)
+{
+    return md_util_pool_vdo(run_test_init, reg, p, md, env, result, NULL);
+}
+
+static apr_status_t run_renew(void *baton, apr_pool_t *p, apr_pool_t *ptemp, va_list ap)
 {
     md_reg_t *reg = baton;
-    const md_proto_t *proto;
     const md_t *md;
-    int reset;
+    int reset, attempt;
     md_proto_driver_t *driver;
-    const char *challenge;
-    apr_time_t *pvalid_from;
+    apr_table_t *env;
     apr_status_t rv;
+    md_result_t *result;
     
     (void)p;
-    proto = va_arg(ap, const md_proto_t *);
     md = va_arg(ap, const md_t *);
-    challenge = va_arg(ap, const char *);
+    env = va_arg(ap, apr_table_t *);
     reset = va_arg(ap, int); 
-    pvalid_from = va_arg(ap, apr_time_t*);
-    
-    driver = apr_pcalloc(ptemp, sizeof(*driver));
-    rv = init_proto_driver(driver, proto, reg, md, challenge, reset, ptemp);
-    if (APR_SUCCESS == rv && 
-        APR_SUCCESS == (rv = proto->init(driver))) {
-        
-        md_log_perror(MD_LOG_MARK, MD_LOG_DEBUG, 0, ptemp, "%s: run staging", md->name);
-        rv = proto->stage(driver);
+    attempt = va_arg(ap, int);
+    result = va_arg(ap, md_result_t *);
 
-        if (APR_SUCCESS == rv && pvalid_from) {
-            *pvalid_from = driver->stage_valid_from;
-        }
+    rv = run_init(reg, ptemp, &driver, md, 0, env, result, NULL);
+    if (APR_SUCCESS == rv) { 
+        md_log_perror(MD_LOG_MARK, MD_LOG_DEBUG, 0, ptemp, "%s: run staging", md->name);
+        driver->reset = reset;
+        driver->attempt = attempt;
+        driver->retry_failover = reg->retry_failover;
+        rv = driver->proto->renew(driver, result);
     }
     md_log_perror(MD_LOG_MARK, MD_LOG_DEBUG, rv, ptemp, "%s: staging done", md->name);
     return rv;
 }
 
-apr_status_t md_reg_stage(md_reg_t *reg, const md_t *md, const char *challenge, 
-                          int reset, apr_time_t *pvalid_from, apr_pool_t *p)
+apr_status_t md_reg_renew(md_reg_t *reg, const md_t *md, apr_table_t *env, 
+                          int reset, int attempt,
+                          md_result_t *result, apr_pool_t *p)
 {
-    const md_proto_t *proto;
-    
-    if (!md->ca_proto) {
-        md_log_perror(MD_LOG_MARK, MD_LOG_WARNING, 0, p, "md %s has no CA protocol", md->name);
-        ((md_t *)md)->state = MD_S_ERROR;
-        return APR_SUCCESS;
-    }
-    
-    proto = apr_hash_get(reg->protos, md->ca_proto, (apr_ssize_t)strlen(md->ca_proto));
-    if (!proto) {
-        md_log_perror(MD_LOG_MARK, MD_LOG_WARNING, 0, p, 
-                      "md %s has unknown CA protocol: %s", md->name, md->ca_proto);
-        ((md_t *)md)->state = MD_S_ERROR;
-        return APR_EINVAL;
-    }
-    
-    return md_util_pool_vdo(run_stage, reg, p, proto, md, challenge, reset, pvalid_from, NULL);
+    return md_util_pool_vdo(run_renew, reg, p, md, env, reset, attempt, result, NULL);
 }
 
-static apr_status_t run_load(void *baton, apr_pool_t *p, apr_pool_t *ptemp, va_list ap)
+static apr_status_t run_load_staging(void *baton, apr_pool_t *p, apr_pool_t *ptemp, va_list ap)
 {
     md_reg_t *reg = baton;
-    const char *name;
-    const md_proto_t *proto;
-    const md_t *md, *nmd;
+    const md_t *md;
     md_proto_driver_t *driver;
+    md_result_t *result;
+    apr_table_t *env;
+    md_job_t *job;
     apr_status_t rv;
     
-    name = va_arg(ap, const char *);
+    /* For the MD,  check if something is in the STAGING area. If none is there, 
+     * return that status. Otherwise ask the protocol driver to preload it into
+     * a new, temporary area. 
+     * If that succeeds, we move the TEMP area over the DOMAINS (causing the 
+     * existing one go to ARCHIVE).
+     * Finally, we clean up the data from CHALLENGES and STAGING.
+     */
+    md = va_arg(ap, const md_t*);
+    env =  va_arg(ap, apr_table_t*);
+    result =  va_arg(ap, md_result_t*);
     
-    if (APR_STATUS_IS_ENOENT(rv = md_load(reg->store, MD_SG_STAGING, name, NULL, ptemp))) {
-        md_log_perror(MD_LOG_MARK, MD_LOG_TRACE3, rv, ptemp, "%s: nothing staged", name);
-        return APR_ENOENT;
+    if (APR_STATUS_IS_ENOENT(rv = md_load(reg->store, MD_SG_STAGING, md->name, NULL, ptemp))) {
+        md_log_perror(MD_LOG_MARK, MD_LOG_TRACE2, 0, ptemp, "%s: nothing staged", md->name);
+        goto out;
     }
     
-    md = md_reg_get(reg, name, p);
-    if (!md) {
-        return APR_ENOENT;
-    }
+    rv = run_init(baton, ptemp, &driver, md, 1, env, result, NULL);
+    if (APR_SUCCESS != rv) goto out;
     
-    if (!md->ca_proto) {
-        md_log_perror(MD_LOG_MARK, MD_LOG_WARNING, 0, p, "md %s has no CA protocol", name);
-        ((md_t *)md)->state = MD_S_ERROR;
-        return APR_EINVAL;
+    apr_hash_set(reg->certs, md->name, (apr_ssize_t)strlen(md->name), NULL);
+    md_result_activity_setn(result, "preloading staged to tmp");
+    rv = driver->proto->preload(driver, MD_SG_TMP, result);
+    if (APR_SUCCESS != rv) goto out;
+
+    /* If we had a job saved in STAGING, copy it over too */
+    job = md_reg_job_make(reg, md->name, ptemp);
+    if (APR_SUCCESS == md_job_load(job)) {
+        md_job_set_group(job, MD_SG_TMP);
+        md_job_save(job, NULL, ptemp);
     }
     
-    proto = apr_hash_get(reg->protos, md->ca_proto, (apr_ssize_t)strlen(md->ca_proto));
-    if (!proto) {
-        md_log_perror(MD_LOG_MARK, MD_LOG_WARNING, 0, p, 
-                      "md %s has unknown CA protocol: %s", md->name, md->ca_proto);
-        ((md_t *)md)->state = MD_S_ERROR;
-        return APR_EINVAL;
+    /* swap */
+    md_result_activity_setn(result, "moving tmp to become new domains");
+    rv = md_store_move(reg->store, p, MD_SG_TMP, MD_SG_DOMAINS, md->name, 1);
+    if (APR_SUCCESS != rv) {
+        md_result_set(result, rv, NULL);
+        goto out;
     }
     
-    driver = apr_pcalloc(ptemp, sizeof(*driver));
-    init_proto_driver(driver, proto, reg, md, NULL, 0, ptemp);
+    md_store_purge(reg->store, p, MD_SG_STAGING, md->name);
+    md_store_purge(reg->store, p, MD_SG_CHALLENGES, md->name);
+    md_result_set(result, APR_SUCCESS, "new certificate successfully saved in domains");
+    md_event_holler("installed", md->name, job, result, ptemp);
+    if (job->dirty) md_job_save(job, result, ptemp);
+    
+out:
+    if (!APR_STATUS_IS_ENOENT(rv)) {
+        md_log_perror(MD_LOG_MARK, MD_LOG_TRACE1, rv, ptemp, "%s: load done", md->name);
+    }
+    return rv;
+}
 
-    if (APR_SUCCESS == (rv = proto->init(driver))) {
-        md_log_perror(MD_LOG_MARK, MD_LOG_DEBUG, 0, ptemp, "%s: run load", md->name);
-        
-        if (APR_SUCCESS == (rv = proto->preload(driver, MD_SG_TMP))) {
-            /* swap */
-            rv = md_store_move(reg->store, p, MD_SG_TMP, MD_SG_DOMAINS, md->name, 1);
-            if (APR_SUCCESS == rv) {
-                /* load again */
-                nmd = md_reg_get(reg, md->name, p);
-                if (!nmd) {
-                    rv = APR_ENOENT;
-                    md_log_perror(MD_LOG_MARK, MD_LOG_ERR, rv, p, "loading md after staging");
-                }
-                else if (nmd->state != MD_S_COMPLETE) {
-                    md_log_perror(MD_LOG_MARK, MD_LOG_WARNING, rv, p, 
-                                  "md has state %d after load", nmd->state);
-                }
-                
-                md_store_purge(reg->store, p, MD_SG_STAGING, md->name);
-                md_store_purge(reg->store, p, MD_SG_CHALLENGES, md->name);
-            }
+apr_status_t md_reg_load_staging(md_reg_t *reg, const md_t *md, apr_table_t *env, 
+                                 md_result_t *result, apr_pool_t *p)
+{
+    if (reg->domains_frozen) return APR_EACCES;
+    return md_util_pool_vdo(run_load_staging, reg, p, md, env, result, NULL);
+}
+
+apr_status_t md_reg_load_stagings(md_reg_t *reg, apr_array_header_t *mds,
+                                  apr_table_t *env, apr_pool_t *p)
+{
+    apr_status_t rv = APR_SUCCESS;
+    md_t *md;
+    md_result_t *result;
+    int i;
+
+    for (i = 0; i < mds->nelts; ++i) {
+        md = APR_ARRAY_IDX(mds, i, md_t *);
+        result = md_result_md_make(p, md->name);
+        rv = md_reg_load_staging(reg, md, env, result, p);
+        if (APR_SUCCESS == rv) {
+            md_log_perror(MD_LOG_MARK, MD_LOG_INFO, rv, p, APLOGNO(10068)
+                          "%s: staged set activated", md->name);
+        }
+        else if (!APR_STATUS_IS_ENOENT(rv)) {
+            md_log_perror(MD_LOG_MARK, MD_LOG_ERR, rv, p, APLOGNO(10069)
+                          "%s: error loading staged set", md->name);
+        }
+    }
+
+    return rv;
+}
+
+apr_status_t md_reg_lock_global(md_reg_t *reg, apr_pool_t *p)
+{
+    apr_status_t rv = APR_SUCCESS;
+
+    if (reg->use_store_locks) {
+        rv = md_store_lock_global(reg->store, p, reg->lock_wait_timeout);
+        if (APR_SUCCESS != rv) {
+            md_log_perror(MD_LOG_MARK, MD_LOG_DEBUG, rv, p,
+                          "unable to acquire global store lock");
+        }
+    }
+    return rv;
+}
+
+void md_reg_unlock_global(md_reg_t *reg, apr_pool_t *p)
+{
+    if (reg->use_store_locks) {
+        md_store_unlock_global(reg->store, p);
+    }
+}
+
+apr_status_t md_reg_freeze_domains(md_reg_t *reg, apr_array_header_t *mds)
+{
+    apr_status_t rv = APR_SUCCESS;
+    md_t *md;
+    const md_pubcert_t *pubcert;
+    int i, j;
+    
+    assert(!reg->domains_frozen);
+    /* prefill the certs cache for all mds */
+    for (i = 0; i < mds->nelts; ++i) {
+        md = APR_ARRAY_IDX(mds, i, md_t*);
+        for (j = 0; j < md_cert_count(md); ++j) {
+            rv = md_reg_get_pubcert(&pubcert, reg, md, i, reg->p);
+            if (APR_SUCCESS != rv && !APR_STATUS_IS_ENOENT(rv)) goto leave;
         }
     }
-    md_log_perror(MD_LOG_MARK, MD_LOG_DEBUG, rv, ptemp, "%s: load done", md->name);
+    reg->domains_frozen = 1;
+leave:
     return rv;
 }
 
-apr_status_t md_reg_load(md_reg_t *reg, const char *name, apr_pool_t *p)
+void md_reg_set_renew_window_default(md_reg_t *reg, md_timeslice_t *renew_window)
 {
-    return md_util_pool_vdo(run_load, reg, p, name, NULL);
+    *reg->renew_window = *renew_window;
 }
 
+void md_reg_set_warn_window_default(md_reg_t *reg, md_timeslice_t *warn_window)
+{
+    *reg->warn_window = *warn_window;
+}
+
+md_job_t *md_reg_job_make(md_reg_t *reg, const char *mdomain, apr_pool_t *p)
+{
+    return md_job_make(p, reg->store, MD_SG_STAGING, mdomain, reg->min_delay);
+}