Adding upstream version 256.upstream/256

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

1 files changed, 656 insertions, 436 deletions

diff --git a/src/network/networkd-nexthop.c b/src/network/networkd-nexthop.c
index e2ded28..1b44ef3 100644
--- a/src/network/networkd-nexthop.c
+++ b/src/network/networkd-nexthop.c
@@ -2,6 +2,7 @@
  * Copyright © 2019 VMware, Inc.
  */
 
+/* Make sure the net/if.h header is included before any linux/ one */
 #include <net/if.h>
 #include <linux/nexthop.h>
 
@@ -12,53 +13,125 @@
 #include "networkd-network.h"
 #include "networkd-nexthop.h"
 #include "networkd-queue.h"
+#include "networkd-route.h"
 #include "networkd-route-util.h"
 #include "parse-util.h"
 #include "set.h"
 #include "stdio-util.h"
 #include "string-util.h"
 
-NextHop *nexthop_free(NextHop *nexthop) {
-        if (!nexthop)
-                return NULL;
+static void nexthop_detach_from_group_members(NextHop *nexthop) {
+        assert(nexthop);
+        assert(nexthop->manager);
+        assert(nexthop->id > 0);
 
-        if (nexthop->network) {
-                assert(nexthop->section);
-                hashmap_remove(nexthop->network->nexthops_by_section, nexthop->section);
+        struct nexthop_grp *nhg;
+        HASHMAP_FOREACH(nhg, nexthop->group) {
+                NextHop *nh;
+
+                if (nexthop_get_by_id(nexthop->manager, nhg->id, &nh) < 0)
+                        continue;
+
+                set_remove(nh->nexthops, UINT32_TO_PTR(nexthop->id));
         }
+}
 
-        config_section_free(nexthop->section);
+static void nexthop_attach_to_group_members(NextHop *nexthop) {
+        int r;
 
-        if (nexthop->link) {
-                set_remove(nexthop->link->nexthops, nexthop);
+        assert(nexthop);
+        assert(nexthop->manager);
+        assert(nexthop->id > 0);
+
+        struct nexthop_grp *nhg;
+        HASHMAP_FOREACH(nhg, nexthop->group) {
+                NextHop *nh;
+
+                r = nexthop_get_by_id(nexthop->manager, nhg->id, &nh);
+                if (r < 0) {
+                        if (nexthop->manager->manage_foreign_nexthops)
+                                log_debug_errno(r, "Nexthop (id=%"PRIu32") has unknown group member (%"PRIu32"), ignoring.",
+                                                nexthop->id, nhg->id);
+                        continue;
+                }
+
+                r = set_ensure_put(&nh->nexthops, NULL, UINT32_TO_PTR(nexthop->id));
+                if (r < 0)
+                        log_debug_errno(r, "Failed to save nexthop ID (%"PRIu32") to group member (%"PRIu32"), ignoring: %m",
+                                        nexthop->id, nhg->id);
+        }
+}
+
+static NextHop* nexthop_detach_impl(NextHop *nexthop) {
+        assert(nexthop);
+        assert(!nexthop->manager || !nexthop->network);
 
-                if (nexthop->link->manager && nexthop->id > 0)
-                        hashmap_remove(nexthop->link->manager->nexthops_by_id, UINT32_TO_PTR(nexthop->id));
+        if (nexthop->network) {
+                assert(nexthop->section);
+                ordered_hashmap_remove(nexthop->network->nexthops_by_section, nexthop->section);
+                nexthop->network = NULL;
+                return nexthop;
         }
 
         if (nexthop->manager) {
-                set_remove(nexthop->manager->nexthops, nexthop);
+                assert(nexthop->id > 0);
 
-                if (nexthop->id > 0)
-                        hashmap_remove(nexthop->manager->nexthops_by_id, UINT32_TO_PTR(nexthop->id));
+                nexthop_detach_from_group_members(nexthop);
+
+                hashmap_remove(nexthop->manager->nexthops_by_id, UINT32_TO_PTR(nexthop->id));
+                nexthop->manager = NULL;
+                return nexthop;
         }
 
+        return NULL;
+}
+
+static void nexthop_detach(NextHop *nexthop) {
+        nexthop_unref(nexthop_detach_impl(nexthop));
+}
+
+static NextHop* nexthop_free(NextHop *nexthop) {
+        if (!nexthop)
+                return NULL;
+
+        nexthop_detach_impl(nexthop);
+
+        config_section_free(nexthop->section);
         hashmap_free_free(nexthop->group);
+        set_free(nexthop->nexthops);
+        set_free(nexthop->routes);
 
         return mfree(nexthop);
 }
 
-DEFINE_SECTION_CLEANUP_FUNCTIONS(NextHop, nexthop_free);
+DEFINE_TRIVIAL_REF_UNREF_FUNC(NextHop, nexthop, nexthop_free);
+DEFINE_SECTION_CLEANUP_FUNCTIONS(NextHop, nexthop_unref);
+
+DEFINE_PRIVATE_HASH_OPS_WITH_VALUE_DESTRUCTOR(
+                nexthop_hash_ops,
+                void,
+                trivial_hash_func,
+                trivial_compare_func,
+                NextHop,
+                nexthop_detach);
+
+DEFINE_PRIVATE_HASH_OPS_WITH_VALUE_DESTRUCTOR(
+                nexthop_section_hash_ops,
+                ConfigSection,
+                config_section_hash_func,
+                config_section_compare_func,
+                NextHop,
+                nexthop_detach);
 
 static int nexthop_new(NextHop **ret) {
-        _cleanup_(nexthop_freep) NextHop *nexthop = NULL;
+        _cleanup_(nexthop_unrefp) NextHop *nexthop = NULL;
 
         nexthop = new(NextHop, 1);
         if (!nexthop)
                 return -ENOMEM;
 
         *nexthop = (NextHop) {
-                .family = AF_UNSPEC,
+                .n_ref = 1,
                 .onlink = -1,
         };
 
@@ -69,7 +142,7 @@ static int nexthop_new(NextHop **ret) {
 
 static int nexthop_new_static(Network *network, const char *filename, unsigned section_line, NextHop **ret) {
         _cleanup_(config_section_freep) ConfigSection *n = NULL;
-        _cleanup_(nexthop_freep) NextHop *nexthop = NULL;
+        _cleanup_(nexthop_unrefp) NextHop *nexthop = NULL;
         int r;
 
         assert(network);
@@ -81,7 +154,7 @@ static int nexthop_new_static(Network *network, const char *filename, unsigned s
         if (r < 0)
                 return r;
 
-        nexthop = hashmap_get(network->nexthops_by_section, n);
+        nexthop = ordered_hashmap_get(network->nexthops_by_section, n);
         if (nexthop) {
                 *ret = TAKE_PTR(nexthop);
                 return 0;
@@ -96,7 +169,7 @@ static int nexthop_new_static(Network *network, const char *filename, unsigned s
         nexthop->section = TAKE_PTR(n);
         nexthop->source = NETWORK_CONFIG_SOURCE_STATIC;
 
-        r = hashmap_ensure_put(&network->nexthops_by_section, &config_section_hash_ops, nexthop->section, nexthop);
+        r = ordered_hashmap_ensure_put(&network->nexthops_by_section, &nexthop_section_hash_ops, nexthop->section, nexthop);
         if (r < 0)
                 return r;
 
@@ -106,35 +179,54 @@ static int nexthop_new_static(Network *network, const char *filename, unsigned s
 
 static void nexthop_hash_func(const NextHop *nexthop, struct siphash *state) {
         assert(nexthop);
+        assert(state);
 
-        siphash24_compress(&nexthop->protocol, sizeof(nexthop->protocol), state);
-        siphash24_compress(&nexthop->id, sizeof(nexthop->id), state);
-        siphash24_compress(&nexthop->blackhole, sizeof(nexthop->blackhole), state);
-        siphash24_compress(&nexthop->family, sizeof(nexthop->family), state);
+        siphash24_compress_typesafe(nexthop->id, state);
+}
 
-        switch (nexthop->family) {
-        case AF_INET:
-        case AF_INET6:
-                siphash24_compress(&nexthop->gw, FAMILY_ADDRESS_SIZE(nexthop->family), state);
+static int nexthop_compare_func(const NextHop *a, const NextHop *b) {
+        assert(a);
+        assert(b);
 
-                break;
-        default:
-                /* treat any other address family as AF_UNSPEC */
-                break;
-        }
+        return CMP(a->id, b->id);
 }
 
-static int nexthop_compare_func(const NextHop *a, const NextHop *b) {
+static int nexthop_compare_full(const NextHop *a, const NextHop *b) {
         int r;
 
+        assert(a);
+        assert(b);
+
+        /* This compares detailed configs, except for ID and ifindex. */
+
         r = CMP(a->protocol, b->protocol);
         if (r != 0)
                 return r;
 
-        r = CMP(a->id, b->id);
+        r = CMP(a->flags, b->flags);
+        if (r != 0)
+                return r;
+
+        r = CMP(hashmap_size(a->group), hashmap_size(b->group));
         if (r != 0)
                 return r;
 
+        if (!hashmap_isempty(a->group)) {
+                struct nexthop_grp *ga;
+
+                HASHMAP_FOREACH(ga, a->group) {
+                        struct nexthop_grp *gb;
+
+                        gb = hashmap_get(b->group, UINT32_TO_PTR(ga->id));
+                        if (!gb)
+                                return CMP(ga, gb);
+
+                        r = CMP(ga->weight, gb->weight);
+                        if (r != 0)
+                                return r;
+                }
+        }
+
         r = CMP(a->blackhole, b->blackhole);
         if (r != 0)
                 return r;
@@ -143,31 +235,17 @@ static int nexthop_compare_func(const NextHop *a, const NextHop *b) {
         if (r != 0)
                 return r;
 
-        if (IN_SET(a->family, AF_INET, AF_INET6))
-                return memcmp(&a->gw, &b->gw, FAMILY_ADDRESS_SIZE(a->family));
+        if (IN_SET(a->family, AF_INET, AF_INET6)) {
+                r = memcmp(&a->gw, &b->gw, FAMILY_ADDRESS_SIZE(a->family));
+                if (r != 0)
+                        return r;
+        }
 
         return 0;
 }
 
-DEFINE_PRIVATE_HASH_OPS_WITH_KEY_DESTRUCTOR(
-                nexthop_hash_ops,
-                NextHop,
-                nexthop_hash_func,
-                nexthop_compare_func,
-                nexthop_free);
-
-static bool nexthop_equal(const NextHop *a, const NextHop *b) {
-        if (a == b)
-                return true;
-
-        if (!a || !b)
-                return false;
-
-        return nexthop_compare_func(a, b) == 0;
-}
-
 static int nexthop_dup(const NextHop *src, NextHop **ret) {
-        _cleanup_(nexthop_freep) NextHop *dest = NULL;
+        _cleanup_(nexthop_unrefp) NextHop *dest = NULL;
         struct nexthop_grp *nhg;
         int r;
 
@@ -178,9 +256,9 @@ static int nexthop_dup(const NextHop *src, NextHop **ret) {
         if (!dest)
                 return -ENOMEM;
 
-        /* unset all pointers */
+        /* clear the reference counter and all pointers */
+        dest->n_ref = 1;
         dest->manager = NULL;
-        dest->link = NULL;
         dest->network = NULL;
         dest->section = NULL;
         dest->group = NULL;
@@ -203,7 +281,12 @@ static int nexthop_dup(const NextHop *src, NextHop **ret) {
         return 0;
 }
 
-int manager_get_nexthop_by_id(Manager *manager, uint32_t id, NextHop **ret) {
+static bool nexthop_bound_to_link(const NextHop *nexthop) {
+        assert(nexthop);
+        return !nexthop->blackhole && hashmap_isempty(nexthop->group);
+}
+
+int nexthop_get_by_id(Manager *manager, uint32_t id, NextHop **ret) {
         NextHop *nh;
 
         assert(manager);
@@ -220,143 +303,175 @@ int manager_get_nexthop_by_id(Manager *manager, uint32_t id, NextHop **ret) {
         return 0;
 }
 
-static bool nexthop_owned_by_link(const NextHop *nexthop) {
-        return !nexthop->blackhole && hashmap_isempty(nexthop->group);
-}
-
-static int nexthop_get(Manager *manager, Link *link, NextHop *in, NextHop **ret) {
+static int nexthop_get(Link *link, const NextHop *in, NextHop **ret) {
         NextHop *nexthop;
-        Set *nexthops;
+        int ifindex;
 
+        assert(link);
+        assert(link->manager);
         assert(in);
 
-        if (nexthop_owned_by_link(in)) {
-                if (!link)
-                        return -ENOENT;
+        if (in->id > 0)
+                return nexthop_get_by_id(link->manager, in->id, ret);
 
-                nexthops = link->nexthops;
-        } else {
-                if (!manager)
-                        return -ENOENT;
+        /* If ManageForeignNextHops=no, nexthop with id == 0 should be already filtered by
+         * nexthop_section_verify(). */
+        assert(link->manager->manage_foreign_nexthops);
 
-                nexthops = manager->nexthops;
-        }
+        ifindex = nexthop_bound_to_link(in) ? link->ifindex : 0;
+
+        HASHMAP_FOREACH(nexthop, link->manager->nexthops_by_id) {
+                if (nexthop->ifindex != ifindex)
+                        continue;
+                if (nexthop_compare_full(nexthop, in) != 0)
+                        continue;
+
+                /* Even if the configuration matches, it may be configured with another [NextHop] section
+                 * that has an explicit ID. If so, the assigned nexthop is not the one we are looking for. */
+                if (set_contains(link->manager->nexthop_ids, UINT32_TO_PTR(nexthop->id)))
+                        continue;
 
-        nexthop = set_get(nexthops, in);
-        if (nexthop) {
                 if (ret)
                         *ret = nexthop;
                 return 0;
         }
 
-        if (in->id > 0)
+        return -ENOENT;
+}
+
+static int nexthop_get_request_by_id(Manager *manager, uint32_t id, Request **ret) {
+        Request *req;
+
+        assert(manager);
+
+        if (id == 0)
+                return -EINVAL;
+
+        req = ordered_set_get(
+                        manager->request_queue,
+                        &(Request) {
+                                .type = REQUEST_TYPE_NEXTHOP,
+                                .userdata = (void*) &(const NextHop) { .id = id },
+                                .hash_func = (hash_func_t) nexthop_hash_func,
+                                .compare_func = (compare_func_t) nexthop_compare_func,
+                        });
+        if (!req)
                 return -ENOENT;
 
-        /* Also find nexthop configured without ID. */
-        SET_FOREACH(nexthop, nexthops) {
-                uint32_t id;
-                bool found;
+        if (ret)
+                *ret = req;
+        return 0;
+}
 
-                id = nexthop->id;
-                nexthop->id = 0;
-                found = nexthop_equal(nexthop, in);
-                nexthop->id = id;
+static int nexthop_get_request(Link *link, const NextHop *in, Request **ret) {
+        Request *req;
+        int ifindex;
+
+        assert(link);
+        assert(link->manager);
+        assert(in);
+
+        if (in->id > 0)
+                return nexthop_get_request_by_id(link->manager, in->id, ret);
 
-                if (!found)
+        /* If ManageForeignNextHops=no, nexthop with id == 0 should be already filtered by
+         * nexthop_section_verify(). */
+        assert(link->manager->manage_foreign_nexthops);
+
+        ifindex = nexthop_bound_to_link(in) ? link->ifindex : 0;
+
+        ORDERED_SET_FOREACH(req, link->manager->request_queue) {
+                if (req->type != REQUEST_TYPE_NEXTHOP)
+                        continue;
+
+                NextHop *nexthop = ASSERT_PTR(req->userdata);
+                if (nexthop->ifindex != ifindex)
+                        continue;
+                if (nexthop_compare_full(nexthop, in) != 0)
+                        continue;
+
+                /* Even if the configuration matches, it may be requested by another [NextHop] section
+                 * that has an explicit ID. If so, the request is not the one we are looking for. */
+                if (set_contains(link->manager->nexthop_ids, UINT32_TO_PTR(nexthop->id)))
                         continue;
 
                 if (ret)
-                        *ret = nexthop;
+                        *ret = req;
                 return 0;
         }
 
         return -ENOENT;
 }
 
-static int nexthop_add(Manager *manager, Link *link, NextHop *nexthop) {
+static int nexthop_add_new(Manager *manager, uint32_t id, NextHop **ret) {
+        _cleanup_(nexthop_unrefp) NextHop *nexthop = NULL;
         int r;
 
-        assert(nexthop);
-        assert(nexthop->id > 0);
-
-        if (nexthop_owned_by_link(nexthop)) {
-                assert(link);
+        assert(manager);
+        assert(id > 0);
 
-                r = set_ensure_put(&link->nexthops, &nexthop_hash_ops, nexthop);
-                if (r < 0)
-                        return r;
-                if (r == 0)
-                        return -EEXIST;
+        r = nexthop_new(&nexthop);
+        if (r < 0)
+                return r;
 
-                nexthop->link = link;
+        nexthop->id = id;
 
-                manager = link->manager;
-        } else {
-                assert(manager);
+        r = hashmap_ensure_put(&manager->nexthops_by_id, &nexthop_hash_ops, UINT32_TO_PTR(nexthop->id), nexthop);
+        if (r < 0)
+                return r;
+        if (r == 0)
+                return -EEXIST;
 
-                r = set_ensure_put(&manager->nexthops, &nexthop_hash_ops, nexthop);
-                if (r < 0)
-                        return r;
-                if (r == 0)
-                        return -EEXIST;
+        nexthop->manager = manager;
 
-                nexthop->manager = manager;
-        }
+        if (ret)
+                *ret = nexthop;
 
-        return hashmap_ensure_put(&manager->nexthops_by_id, NULL, UINT32_TO_PTR(nexthop->id), nexthop);
+        TAKE_PTR(nexthop);
+        return 0;
 }
 
 static int nexthop_acquire_id(Manager *manager, NextHop *nexthop) {
-        _cleanup_set_free_ Set *ids = NULL;
-        Network *network;
-        uint32_t id;
-        int r;
-
         assert(manager);
         assert(nexthop);
 
         if (nexthop->id > 0)
                 return 0;
 
-        /* Find the lowest unused ID. */
-
-        ORDERED_HASHMAP_FOREACH(network, manager->networks) {
-                NextHop *tmp;
-
-                HASHMAP_FOREACH(tmp, network->nexthops_by_section) {
-                        if (tmp->id == 0)
-                                continue;
+        /* If ManageForeignNextHops=no, nexthop with id == 0 should be already filtered by
+         * nexthop_section_verify(). */
+        assert(manager->manage_foreign_nexthops);
 
-                        r = set_ensure_put(&ids, NULL, UINT32_TO_PTR(tmp->id));
-                        if (r < 0)
-                                return r;
-                }
-        }
+        /* Find the lowest unused ID. */
 
-        for (id = 1; id < UINT32_MAX; id++) {
-                if (manager_get_nexthop_by_id(manager, id, NULL) >= 0)
+        for (uint32_t id = 1; id < UINT32_MAX; id++) {
+                if (nexthop_get_by_id(manager, id, NULL) >= 0)
                         continue;
-                if (set_contains(ids, UINT32_TO_PTR(id)))
+                if (nexthop_get_request_by_id(manager, id, NULL) >= 0)
                         continue;
-                break;
+                if (set_contains(manager->nexthop_ids, UINT32_TO_PTR(id)))
+                        continue;
+
+                nexthop->id = id;
+                return 0;
         }
 
-        nexthop->id = id;
-        return 0;
+        return -EBUSY;
 }
 
-static void log_nexthop_debug(const NextHop *nexthop, const char *str, const Link *link) {
+static void log_nexthop_debug(const NextHop *nexthop, const char *str, Manager *manager) {
         _cleanup_free_ char *state = NULL, *group = NULL, *flags = NULL;
         struct nexthop_grp *nhg;
+        Link *link = NULL;
 
         assert(nexthop);
         assert(str);
-
-        /* link may be NULL. */
+        assert(manager);
 
         if (!DEBUG_LOGGING)
                 return;
 
+        (void) link_get_by_index(manager, nexthop->ifindex, &link);
         (void) network_config_state_to_string_alloc(nexthop->state, &state);
         (void) route_flags_to_string_alloc(nexthop->flags, &flags);
 
@@ -370,42 +485,81 @@ static void log_nexthop_debug(const NextHop *nexthop, const char *str, const Lin
                        yes_no(nexthop->blackhole), strna(group), strna(flags));
 }
 
-static int nexthop_remove_handler(sd_netlink *rtnl, sd_netlink_message *m, Link *link) {
+static int nexthop_remove_dependents(NextHop *nexthop, Manager *manager) {
+        int r = 0;
+
+        assert(nexthop);
+        assert(manager);
+
+        /* If a nexthop is removed, the kernel silently removes nexthops and routes that depend on the
+         * removed nexthop. Let's remove them for safety (though, they are already removed in the kernel,
+         * hence that should fail), and forget them. */
+
+        void *id;
+        SET_FOREACH(id, nexthop->nexthops) {
+                NextHop *nh;
+
+                if (nexthop_get_by_id(manager, PTR_TO_UINT32(id), &nh) < 0)
+                        continue;
+
+                RET_GATHER(r, nexthop_remove(nh, manager));
+        }
+
+        Route *route;
+        SET_FOREACH(route, nexthop->routes)
+                RET_GATHER(r, route_remove(route, manager));
+
+        return r;
+}
+
+static int nexthop_remove_handler(sd_netlink *rtnl, sd_netlink_message *m, RemoveRequest *rreq) {
         int r;
 
         assert(m);
+        assert(rreq);
 
-        /* link may be NULL. */
-
-        if (link && IN_SET(link->state, LINK_STATE_FAILED, LINK_STATE_LINGER))
-                return 1;
+        Manager *manager = ASSERT_PTR(rreq->manager);
+        NextHop *nexthop = ASSERT_PTR(rreq->userdata);
 
         r = sd_netlink_message_get_errno(m);
-        if (r < 0 && r != -ENOENT)
-                log_link_message_warning_errno(link, m, r, "Could not drop nexthop, ignoring");
+        if (r < 0) {
+                log_message_full_errno(m,
+                                       (r == -ENOENT || !nexthop->manager) ? LOG_DEBUG : LOG_WARNING,
+                                       r, "Could not drop nexthop, ignoring");
+
+                (void) nexthop_remove_dependents(nexthop, manager);
+
+                if (nexthop->manager) {
+                        /* If the nexthop cannot be removed, then assume the nexthop is already removed. */
+                        log_nexthop_debug(nexthop, "Forgetting", manager);
+
+                        Request *req;
+                        if (nexthop_get_request_by_id(manager, nexthop->id, &req) >= 0)
+                                nexthop_enter_removed(req->userdata);
+
+                        nexthop_detach(nexthop);
+                }
+        }
 
         return 1;
 }
 
-static int nexthop_remove(NextHop *nexthop) {
+int nexthop_remove(NextHop *nexthop, Manager *manager) {
         _cleanup_(sd_netlink_message_unrefp) sd_netlink_message *m = NULL;
-        Manager *manager;
-        Link *link;
+        Link *link = NULL;
         int r;
 
         assert(nexthop);
-        assert(nexthop->manager || (nexthop->link && nexthop->link->manager));
+        assert(nexthop->id > 0);
+        assert(manager);
 
-        /* link may be NULL. */
-        link = nexthop->link;
-        manager = nexthop->manager ?: nexthop->link->manager;
+        /* If the nexthop is remembered, then use the remembered object. */
+        (void) nexthop_get_by_id(manager, PTR_TO_UINT32(nexthop->id), &nexthop);
 
-        if (nexthop->id == 0) {
-                log_link_debug(link, "Cannot remove nexthop without valid ID, ignoring.");
-                return 0;
-        }
+        /* link may be NULL. */
+        (void) link_get_by_index(manager, nexthop->ifindex, &link);
 
-        log_nexthop_debug(nexthop, "Removing", link);
+        log_nexthop_debug(nexthop, "Removing", manager);
 
         r = sd_rtnl_message_new_nexthop(manager->rtnl, &m, RTM_DELNEXTHOP, AF_UNSPEC, RTPROT_UNSPEC);
         if (r < 0)
@@ -415,12 +569,9 @@ static int nexthop_remove(NextHop *nexthop) {
         if (r < 0)
                 return log_link_error_errno(link, r, "Could not append NHA_ID attribute: %m");
 
-        r = netlink_call_async(manager->rtnl, NULL, m, nexthop_remove_handler,
-                               link ? link_netlink_destroy_callback : NULL, link);
+        r = manager_remove_request_add(manager, nexthop, nexthop, manager->rtnl, m, nexthop_remove_handler);
         if (r < 0)
-                return log_link_error_errno(link, r, "Could not send rtnetlink message: %m");
-
-        link_ref(link); /* link may be NULL, link_ref() is OK with that */
+                return log_link_error_errno(link, r, "Could not queue rtnetlink message: %m");
 
         nexthop_enter_removing(nexthop);
         return 0;
@@ -431,6 +582,7 @@ static int nexthop_configure(NextHop *nexthop, Link *link, Request *req) {
         int r;
 
         assert(nexthop);
+        assert(nexthop->id > 0);
         assert(IN_SET(nexthop->family, AF_UNSPEC, AF_INET, AF_INET6));
         assert(link);
         assert(link->manager);
@@ -438,17 +590,15 @@ static int nexthop_configure(NextHop *nexthop, Link *link, Request *req) {
         assert(link->ifindex > 0);
         assert(req);
 
-        log_nexthop_debug(nexthop, "Configuring", link);
+        log_nexthop_debug(nexthop, "Configuring", link->manager);
 
         r = sd_rtnl_message_new_nexthop(link->manager->rtnl, &m, RTM_NEWNEXTHOP, nexthop->family, nexthop->protocol);
         if (r < 0)
                 return r;
 
-        if (nexthop->id > 0) {
-                r = sd_netlink_message_append_u32(m, NHA_ID, nexthop->id);
-                if (r < 0)
-                        return r;
-        }
+        r = sd_netlink_message_append_u32(m, NHA_ID, nexthop->id);
+        if (r < 0)
+                return r;
 
         if (!hashmap_isempty(nexthop->group)) {
                 _cleanup_free_ struct nexthop_grp *group = NULL;
@@ -471,7 +621,9 @@ static int nexthop_configure(NextHop *nexthop, Link *link, Request *req) {
                 if (r < 0)
                         return r;
         } else {
-                r = sd_netlink_message_append_u32(m, NHA_OIF, link->ifindex);
+                assert(nexthop->ifindex == link->ifindex);
+
+                r = sd_netlink_message_append_u32(m, NHA_OIF, nexthop->ifindex);
                 if (r < 0)
                         return r;
 
@@ -511,16 +663,49 @@ static int static_nexthop_handler(sd_netlink *rtnl, sd_netlink_message *m, Reque
         return 1;
 }
 
+int nexthop_is_ready(Manager *manager, uint32_t id, NextHop **ret) {
+        NextHop *nexthop;
+
+        assert(manager);
+
+        if (id == 0)
+                return -EINVAL;
+
+        if (nexthop_get_request_by_id(manager, id, NULL) >= 0)
+                goto not_ready;
+
+        if (nexthop_get_by_id(manager, id, &nexthop) < 0)
+                goto not_ready;
+
+        if (!nexthop_exists(nexthop))
+                goto not_ready;
+
+        if (ret)
+                *ret = nexthop;
+
+        return true;
+
+not_ready:
+        if (ret)
+                *ret = NULL;
+
+        return false;
+}
+
 static bool nexthop_is_ready_to_configure(Link *link, const NextHop *nexthop) {
         struct nexthop_grp *nhg;
+        int r;
 
         assert(link);
         assert(nexthop);
+        assert(nexthop->id > 0);
 
         if (!link_is_ready_to_configure(link, false))
                 return false;
 
-        if (nexthop_owned_by_link(nexthop)) {
+        if (nexthop_bound_to_link(nexthop)) {
+                assert(nexthop->ifindex == link->ifindex);
+
                 /* TODO: fdb nexthop does not require IFF_UP. The conditions below needs to be updated
                  * when fdb nexthop support is added. See rtm_to_nh_config() in net/ipv4/nexthop.c of
                  * kernel. */
@@ -532,34 +717,21 @@ static bool nexthop_is_ready_to_configure(Link *link, const NextHop *nexthop) {
 
         /* All group members must be configured first. */
         HASHMAP_FOREACH(nhg, nexthop->group) {
-                NextHop *g;
-
-                if (manager_get_nexthop_by_id(link->manager, nhg->id, &g) < 0)
-                        return false;
-
-                if (!nexthop_exists(g))
-                        return false;
-        }
-
-        if (nexthop->id == 0) {
-                Request *req;
-
-                ORDERED_SET_FOREACH(req, link->manager->request_queue) {
-                        if (req->type != REQUEST_TYPE_NEXTHOP)
-                                continue;
-                        if (((NextHop*) req->userdata)->id != 0)
-                                return false; /* first configure nexthop with id. */
-                }
+                r = nexthop_is_ready(link->manager, nhg->id, NULL);
+                if (r <= 0)
+                        return r;
         }
 
         return gateway_is_ready(link, FLAGS_SET(nexthop->flags, RTNH_F_ONLINK), nexthop->family, &nexthop->gw);
 }
 
 static int nexthop_process_request(Request *req, Link *link, NextHop *nexthop) {
+        NextHop *existing;
         int r;
 
         assert(req);
         assert(link);
+        assert(link->manager);
         assert(nexthop);
 
         if (!nexthop_is_ready_to_configure(link, nexthop))
@@ -570,39 +742,49 @@ static int nexthop_process_request(Request *req, Link *link, NextHop *nexthop) {
                 return log_link_warning_errno(link, r, "Failed to configure nexthop");
 
         nexthop_enter_configuring(nexthop);
+        if (nexthop_get_by_id(link->manager, nexthop->id, &existing) >= 0)
+                nexthop_enter_configuring(existing);
+
         return 1;
 }
 
-static int link_request_nexthop(Link *link, NextHop *nexthop) {
-        NextHop *existing;
+static int link_request_nexthop(Link *link, const NextHop *nexthop) {
+        _cleanup_(nexthop_unrefp) NextHop *tmp = NULL;
+        NextHop *existing = NULL;
         int r;
 
         assert(link);
+        assert(link->manager);
         assert(nexthop);
         assert(nexthop->source != NETWORK_CONFIG_SOURCE_FOREIGN);
 
-        if (nexthop_get(link->manager, link, nexthop, &existing) < 0) {
-                _cleanup_(nexthop_freep) NextHop *tmp = NULL;
+        if (nexthop_get_request(link, nexthop, NULL) >= 0)
+                return 0; /* already requested, skipping. */
 
-                r = nexthop_dup(nexthop, &tmp);
-                if (r < 0)
-                        return r;
+        r = nexthop_dup(nexthop, &tmp);
+        if (r < 0)
+                return r;
 
+        if (nexthop_get(link, nexthop, &existing) < 0) {
                 r = nexthop_acquire_id(link->manager, tmp);
                 if (r < 0)
                         return r;
+        } else {
+                /* Copy ID */
+                assert(tmp->id == 0 || tmp->id == existing->id);
+                tmp->id = existing->id;
 
-                r = nexthop_add(link->manager, link, tmp);
-                if (r < 0)
-                        return r;
+                /* Copy state for logging below. */
+                tmp->state = existing->state;
+        }
 
-                existing = TAKE_PTR(tmp);
-        } else
-                existing->source = nexthop->source;
+        if (nexthop_bound_to_link(tmp))
+                tmp->ifindex = link->ifindex;
 
-        log_nexthop_debug(existing, "Requesting", link);
+        log_nexthop_debug(tmp, "Requesting", link->manager);
         r = link_queue_request_safe(link, REQUEST_TYPE_NEXTHOP,
-                                    existing, NULL,
+                                    tmp,
+                                    nexthop_unref,
                                     nexthop_hash_func,
                                     nexthop_compare_func,
                                     nexthop_process_request,
@@ -612,7 +794,11 @@ static int link_request_nexthop(Link *link, NextHop *nexthop) {
         if (r <= 0)
                 return r;
 
-        nexthop_enter_requesting(existing);
+        nexthop_enter_requesting(tmp);
+        if (existing)
+                nexthop_enter_requesting(existing);
+
+        TAKE_PTR(tmp);
         return 1;
 }
 
@@ -625,7 +811,7 @@ int link_request_static_nexthops(Link *link, bool only_ipv4) {
 
         link->static_nexthops_configured = false;
 
-        HASHMAP_FOREACH(nh, link->network->nexthops_by_section) {
+        ORDERED_HASHMAP_FOREACH(nh, link->network->nexthops_by_section) {
                 if (only_ipv4 && nh->family != AF_INET)
                         continue;
 
@@ -645,162 +831,182 @@ int link_request_static_nexthops(Link *link, bool only_ipv4) {
         return 0;
 }
 
-static void manager_mark_nexthops(Manager *manager, bool foreign, const Link *except) {
+static bool nexthop_can_update(const NextHop *assigned_nexthop, const NextHop *requested_nexthop) {
+        assert(assigned_nexthop);
+        assert(assigned_nexthop->manager);
+        assert(requested_nexthop);
+        assert(requested_nexthop->network);
+
+        /* A group nexthop cannot be replaced with a non-group nexthop, and vice versa.
+         * See replace_nexthop_grp() and replace_nexthop_single() in net/ipv4/nexthop.c of the kernel. */
+        if (hashmap_isempty(assigned_nexthop->group) != hashmap_isempty(requested_nexthop->group))
+                return false;
+
+        /* There are several more conditions if we can replace a group nexthop, e.g. hash threshold and
+         * resilience. But, currently we do not support to modify that. Let's add checks for them in the
+         * future when we support to configure them.*/
+
+        /* When a nexthop is replaced with a blackhole nexthop, and a group nexthop has multiple nexthops
+         * including this nexthop, then the kernel refuses to replace the existing nexthop.
+         * So, here, for simplicity, let's unconditionally refuse to replace a non-blackhole nexthop with
+         * a blackhole nexthop. See replace_nexthop() in net/ipv4/nexthop.c of the kernel. */
+        if (!assigned_nexthop->blackhole && requested_nexthop->blackhole)
+                return false;
+
+        return true;
+}
+
+static void link_mark_nexthops(Link *link, bool foreign) {
         NextHop *nexthop;
-        Link *link;
+        Link *other;
 
-        assert(manager);
+        assert(link);
+        assert(link->manager);
 
         /* First, mark all nexthops. */
-        SET_FOREACH(nexthop, manager->nexthops) {
+        HASHMAP_FOREACH(nexthop, link->manager->nexthops_by_id) {
                 /* do not touch nexthop created by the kernel */
                 if (nexthop->protocol == RTPROT_KERNEL)
                         continue;
 
                 /* When 'foreign' is true, mark only foreign nexthops, and vice versa. */
-                if (foreign != (nexthop->source == NETWORK_CONFIG_SOURCE_FOREIGN))
+                if (nexthop->source != (foreign ? NETWORK_CONFIG_SOURCE_FOREIGN : NETWORK_CONFIG_SOURCE_STATIC))
                         continue;
 
                 /* Ignore nexthops not assigned yet or already removed. */
                 if (!nexthop_exists(nexthop))
                         continue;
 
+                /* Ignore nexthops bound to other links. */
+                if (nexthop->ifindex > 0 && nexthop->ifindex != link->ifindex)
+                        continue;
+
                 nexthop_mark(nexthop);
         }
 
         /* Then, unmark all nexthops requested by active links. */
-        HASHMAP_FOREACH(link, manager->links_by_index) {
-                if (link == except)
+        HASHMAP_FOREACH(other, link->manager->links_by_index) {
+                if (!foreign && other == link)
                         continue;
 
-                if (!IN_SET(link->state, LINK_STATE_CONFIGURING, LINK_STATE_CONFIGURED))
+                if (!IN_SET(other->state, LINK_STATE_CONFIGURING, LINK_STATE_CONFIGURED))
                         continue;
 
-                HASHMAP_FOREACH(nexthop, link->network->nexthops_by_section) {
+                ORDERED_HASHMAP_FOREACH(nexthop, other->network->nexthops_by_section) {
                         NextHop *existing;
 
-                        if (nexthop_get(manager, NULL, nexthop, &existing) >= 0)
-                                nexthop_unmark(existing);
+                        if (nexthop_get(other, nexthop, &existing) < 0)
+                                continue;
+
+                        if (!nexthop_can_update(existing, nexthop))
+                                continue;
+
+                        /* Found matching static configuration. Keep the existing nexthop. */
+                        nexthop_unmark(existing);
                 }
         }
 }
 
-static int manager_drop_marked_nexthops(Manager *manager) {
+int link_drop_nexthops(Link *link, bool foreign) {
         NextHop *nexthop;
         int r = 0;
 
-        assert(manager);
+        assert(link);
+        assert(link->manager);
 
-        SET_FOREACH(nexthop, manager->nexthops) {
+        link_mark_nexthops(link, foreign);
+
+        HASHMAP_FOREACH(nexthop, link->manager->nexthops_by_id) {
                 if (!nexthop_is_marked(nexthop))
                         continue;
 
-                RET_GATHER(r, nexthop_remove(nexthop));
+                RET_GATHER(r, nexthop_remove(nexthop, link->manager));
         }
 
         return r;
 }
 
-int link_drop_foreign_nexthops(Link *link) {
+void link_foreignize_nexthops(Link *link) {
         NextHop *nexthop;
-        int r = 0;
 
         assert(link);
         assert(link->manager);
-        assert(link->network);
 
-        /* First, mark all nexthops. */
-        SET_FOREACH(nexthop, link->nexthops) {
-                /* do not touch nexthop created by the kernel */
-                if (nexthop->protocol == RTPROT_KERNEL)
-                        continue;
-
-                /* Do not remove nexthops we configured. */
-                if (nexthop->source != NETWORK_CONFIG_SOURCE_FOREIGN)
-                        continue;
+        link_mark_nexthops(link, /* foreign = */ false);
 
-                /* Ignore nexthops not assigned yet or already removed. */
-                if (!nexthop_exists(nexthop))
+        HASHMAP_FOREACH(nexthop, link->manager->nexthops_by_id) {
+                if (!nexthop_is_marked(nexthop))
                         continue;
 
-                nexthop_mark(nexthop);
+                nexthop->source = NETWORK_CONFIG_SOURCE_FOREIGN;
         }
+}
 
-        /* Then, unmark all nexthops requested by active links. */
-        HASHMAP_FOREACH(nexthop, link->network->nexthops_by_section) {
-                NextHop *existing;
-
-                if (nexthop_get(NULL, link, nexthop, &existing) >= 0)
-                        nexthop_unmark(existing);
-        }
+static int nexthop_update_group(NextHop *nexthop, sd_netlink_message *message) {
+        _cleanup_hashmap_free_free_ Hashmap *h = NULL;
+        _cleanup_free_ struct nexthop_grp *group = NULL;
+        size_t size = 0, n_group;
+        int r;
 
-        /* Finally, remove all marked rules. */
-        SET_FOREACH(nexthop, link->nexthops) {
-                if (!nexthop_is_marked(nexthop))
-                        continue;
+        assert(nexthop);
+        assert(message);
 
-                RET_GATHER(r, nexthop_remove(nexthop));
-        }
+        r = sd_netlink_message_read_data(message, NHA_GROUP, &size, (void**) &group);
+        if (r < 0 && r != -ENODATA)
+                return log_debug_errno(r, "rtnl: could not get NHA_GROUP attribute, ignoring: %m");
 
-        manager_mark_nexthops(link->manager, /* foreign = */ true, NULL);
+        nexthop_detach_from_group_members(nexthop);
 
-        return RET_GATHER(r, manager_drop_marked_nexthops(link->manager));
-}
+        if (size % sizeof(struct nexthop_grp) != 0)
+                return log_debug_errno(SYNTHETIC_ERRNO(EINVAL),
+                                       "rtnl: received nexthop message with invalid nexthop group size, ignoring.");
 
-int link_drop_managed_nexthops(Link *link) {
-        NextHop *nexthop;
-        int r = 0;
+        if ((uintptr_t) group % alignof(struct nexthop_grp) != 0)
+                return log_debug_errno(SYNTHETIC_ERRNO(EINVAL),
+                                       "rtnl: received nexthop message with invalid alignment, ignoring.");
 
-        assert(link);
-        assert(link->manager);
+        n_group = size / sizeof(struct nexthop_grp);
+        for (size_t i = 0; i < n_group; i++) {
+                _cleanup_free_ struct nexthop_grp *nhg = NULL;
 
-        SET_FOREACH(nexthop, link->nexthops) {
-                /* do not touch nexthop created by the kernel */
-                if (nexthop->protocol == RTPROT_KERNEL)
+                if (group[i].id == 0) {
+                        log_debug("rtnl: received nexthop message with invalid ID in group, ignoring.");
                         continue;
+                }
 
-                /* Do not touch addresses managed by kernel or other tools. */
-                if (nexthop->source == NETWORK_CONFIG_SOURCE_FOREIGN)
+                if (group[i].weight > 254) {
+                        log_debug("rtnl: received nexthop message with invalid weight in group, ignoring.");
                         continue;
+                }
 
-                /* Ignore nexthops not assigned yet or already removing. */
-                if (!nexthop_exists(nexthop))
-                        continue;
+                nhg = newdup(struct nexthop_grp, group + i, 1);
+                if (!nhg)
+                        return log_oom();
 
-                RET_GATHER(r, nexthop_remove(nexthop));
+                r = hashmap_ensure_put(&h, NULL, UINT32_TO_PTR(nhg->id), nhg);
+                if (r == -ENOMEM)
+                        return log_oom();
+                if (r < 0) {
+                        log_debug_errno(r, "Failed to store nexthop group, ignoring: %m");
+                        continue;
+                }
+                if (r > 0)
+                        TAKE_PTR(nhg);
         }
 
-        manager_mark_nexthops(link->manager, /* foreign = */ false, link);
-
-        return RET_GATHER(r, manager_drop_marked_nexthops(link->manager));
-}
-
-void link_foreignize_nexthops(Link *link) {
-        NextHop *nexthop;
-
-        assert(link);
-
-        SET_FOREACH(nexthop, link->nexthops)
-                nexthop->source = NETWORK_CONFIG_SOURCE_FOREIGN;
-
-        manager_mark_nexthops(link->manager, /* foreign = */ false, link);
-
-        SET_FOREACH(nexthop, link->manager->nexthops) {
-                if (!nexthop_is_marked(nexthop))
-                        continue;
+        hashmap_free_free(nexthop->group);
+        nexthop->group = TAKE_PTR(h);
 
-                nexthop->source = NETWORK_CONFIG_SOURCE_FOREIGN;
-        }
+        nexthop_attach_to_group_members(nexthop);
+        return 0;
 }
 
 int manager_rtnl_process_nexthop(sd_netlink *rtnl, sd_netlink_message *message, Manager *m) {
-        _cleanup_(nexthop_freep) NextHop *tmp = NULL;
-        _cleanup_free_ void *raw_group = NULL;
-        NextHop *nexthop = NULL;
-        size_t raw_group_size;
-        uint32_t ifindex;
         uint16_t type;
-        Link *link = NULL;
+        uint32_t id, ifindex;
+        NextHop *nexthop = NULL;
+        Request *req = NULL;
+        bool is_new = false;
         int r;
 
         assert(rtnl);
@@ -824,163 +1030,102 @@ int manager_rtnl_process_nexthop(sd_netlink *rtnl, sd_netlink_message *message,
                 return 0;
         }
 
-        r = sd_netlink_message_read_u32(message, NHA_OIF, &ifindex);
-        if (r < 0 && r != -ENODATA) {
-                log_warning_errno(r, "rtnl: could not get NHA_OIF attribute, ignoring: %m");
+        r = sd_netlink_message_read_u32(message, NHA_ID, &id);
+        if (r == -ENODATA) {
+                log_warning_errno(r, "rtnl: received nexthop message without NHA_ID attribute, ignoring: %m");
                 return 0;
-        } else if (r >= 0) {
-                if (ifindex <= 0) {
-                        log_warning("rtnl: received nexthop message with invalid ifindex %"PRIu32", ignoring.", ifindex);
-                        return 0;
-                }
-
-                r = link_get_by_index(m, ifindex, &link);
-                if (r < 0) {
-                        if (!m->enumerating)
-                                log_warning("rtnl: received nexthop message for link (%"PRIu32") we do not know about, ignoring", ifindex);
-                        return 0;
-                }
-        }
-
-        r = nexthop_new(&tmp);
-        if (r < 0)
-                return log_oom();
-
-        r = sd_rtnl_message_get_family(message, &tmp->family);
-        if (r < 0) {
-                log_link_warning_errno(link, r, "rtnl: could not get nexthop family, ignoring: %m");
+        } else if (r < 0) {
+                log_warning_errno(r, "rtnl: could not get NHA_ID attribute, ignoring: %m");
                 return 0;
-        } else if (!IN_SET(tmp->family, AF_UNSPEC, AF_INET, AF_INET6)) {
-                log_link_debug(link, "rtnl: received nexthop message with invalid family %d, ignoring.", tmp->family);
+        } else if (id == 0) {
+                log_warning("rtnl: received nexthop message with invalid nexthop ID, ignoring: %m");
                 return 0;
         }
 
-        r = sd_rtnl_message_nexthop_get_protocol(message, &tmp->protocol);
-        if (r < 0) {
-                log_link_warning_errno(link, r, "rtnl: could not get nexthop protocol, ignoring: %m");
-                return 0;
-        }
+        (void) nexthop_get_by_id(m, id, &nexthop);
+        (void) nexthop_get_request_by_id(m, id, &req);
 
-        r = sd_rtnl_message_nexthop_get_flags(message, &tmp->flags);
-        if (r < 0) {
-                log_link_warning_errno(link, r, "rtnl: could not get nexthop flags, ignoring: %m");
-                return 0;
-        }
+        if (type == RTM_DELNEXTHOP) {
+                if (nexthop) {
+                        nexthop_enter_removed(nexthop);
+                        log_nexthop_debug(nexthop, "Forgetting removed", m);
+                        (void) nexthop_remove_dependents(nexthop, m);
+                        nexthop_detach(nexthop);
+                } else
+                        log_nexthop_debug(&(const NextHop) { .id = id }, "Kernel removed unknown", m);
+
+                if (req)
+                        nexthop_enter_removed(req->userdata);
 
-        r = sd_netlink_message_read_data(message, NHA_GROUP, &raw_group_size, &raw_group);
-        if (r < 0 && r != -ENODATA) {
-                log_link_warning_errno(link, r, "rtnl: could not get NHA_GROUP attribute, ignoring: %m");
                 return 0;
-        } else if (r >= 0) {
-                struct nexthop_grp *group = raw_group;
-                size_t n_group;
+        }
 
-                if (raw_group_size == 0 || raw_group_size % sizeof(struct nexthop_grp) != 0) {
-                        log_link_warning(link, "rtnl: received nexthop message with invalid nexthop group size, ignoring.");
+        /* If we did not know the nexthop, then save it. */
+        if (!nexthop) {
+                r = nexthop_add_new(m, id, &nexthop);
+                if (r < 0) {
+                        log_warning_errno(r, "Failed to add received nexthop, ignoring: %m");
                         return 0;
                 }
 
-                assert((uintptr_t) group % alignof(struct nexthop_grp) == 0);
+                is_new = true;
+        }
 
-                n_group = raw_group_size / sizeof(struct nexthop_grp);
-                for (size_t i = 0; i < n_group; i++) {
-                        _cleanup_free_ struct nexthop_grp *nhg = NULL;
+        /* Also update information that cannot be obtained through netlink notification. */
+        if (req && req->waiting_reply) {
+                NextHop *n = ASSERT_PTR(req->userdata);
 
-                        if (group[i].id == 0) {
-                                log_link_warning(link, "rtnl: received nexthop message with invalid ID in group, ignoring.");
-                                return 0;
-                        }
-                        if (group[i].weight > 254) {
-                                log_link_warning(link, "rtnl: received nexthop message with invalid weight in group, ignoring.");
-                                return 0;
-                        }
+                nexthop->source = n->source;
+        }
 
-                        nhg = newdup(struct nexthop_grp, group + i, 1);
-                        if (!nhg)
-                                return log_oom();
+        r = sd_rtnl_message_get_family(message, &nexthop->family);
+        if (r < 0)
+                log_debug_errno(r, "rtnl: could not get nexthop family, ignoring: %m");
 
-                        r = hashmap_ensure_put(&tmp->group, NULL, UINT32_TO_PTR(nhg->id), nhg);
-                        if (r == -ENOMEM)
-                                return log_oom();
-                        if (r < 0) {
-                                log_link_warning_errno(link, r, "Failed to store nexthop group, ignoring: %m");
-                                return 0;
-                        }
-                        if (r > 0)
-                                TAKE_PTR(nhg);
-                }
-        }
+        r = sd_rtnl_message_nexthop_get_protocol(message, &nexthop->protocol);
+        if (r < 0)
+                log_debug_errno(r, "rtnl: could not get nexthop protocol, ignoring: %m");
 
-        if (tmp->family != AF_UNSPEC) {
-                r = netlink_message_read_in_addr_union(message, NHA_GATEWAY, tmp->family, &tmp->gw);
-                if (r < 0 && r != -ENODATA) {
-                        log_link_warning_errno(link, r, "rtnl: could not get NHA_GATEWAY attribute, ignoring: %m");
-                        return 0;
-                }
+        r = sd_rtnl_message_nexthop_get_flags(message, &nexthop->flags);
+        if (r < 0)
+                log_debug_errno(r, "rtnl: could not get nexthop flags, ignoring: %m");
+
+        (void) nexthop_update_group(nexthop, message);
+
+        if (nexthop->family != AF_UNSPEC) {
+                r = netlink_message_read_in_addr_union(message, NHA_GATEWAY, nexthop->family, &nexthop->gw);
+                if (r == -ENODATA)
+                        nexthop->gw = IN_ADDR_NULL;
+                else if (r < 0)
+                        log_debug_errno(r, "rtnl: could not get NHA_GATEWAY attribute, ignoring: %m");
         }
 
         r = sd_netlink_message_has_flag(message, NHA_BLACKHOLE);
-        if (r < 0) {
-                log_link_warning_errno(link, r, "rtnl: could not get NHA_BLACKHOLE attribute, ignoring: %m");
-                return 0;
-        }
-        tmp->blackhole = r;
+        if (r < 0)
+                log_debug_errno(r, "rtnl: could not get NHA_BLACKHOLE attribute, ignoring: %m");
+        else
+                nexthop->blackhole = r;
 
-        r = sd_netlink_message_read_u32(message, NHA_ID, &tmp->id);
-        if (r == -ENODATA) {
-                log_link_warning_errno(link, r, "rtnl: received nexthop message without NHA_ID attribute, ignoring: %m");
-                return 0;
-        } else if (r < 0) {
-                log_link_warning_errno(link, r, "rtnl: could not get NHA_ID attribute, ignoring: %m");
-                return 0;
-        } else if (tmp->id == 0) {
-                log_link_warning(link, "rtnl: received nexthop message with invalid nexthop ID, ignoring: %m");
-                return 0;
-        }
+        r = sd_netlink_message_read_u32(message, NHA_OIF, &ifindex);
+        if (r == -ENODATA)
+                nexthop->ifindex = 0;
+        else if (r < 0)
+                log_debug_errno(r, "rtnl: could not get NHA_OIF attribute, ignoring: %m");
+        else if (ifindex > INT32_MAX)
+                log_debug_errno(r, "rtnl: received invalid NHA_OIF attribute, ignoring: %m");
+        else
+                nexthop->ifindex = (int) ifindex;
 
         /* All blackhole or group nexthops are managed by Manager. Note that the linux kernel does not
          * set NHA_OID attribute when NHA_BLACKHOLE or NHA_GROUP is set. Just for safety. */
-        if (!nexthop_owned_by_link(tmp))
-                link = NULL;
-
-        (void) nexthop_get(m, link, tmp, &nexthop);
-
-        switch (type) {
-        case RTM_NEWNEXTHOP:
-                if (nexthop) {
-                        nexthop->flags = tmp->flags;
-                        nexthop_enter_configured(nexthop);
-                        log_nexthop_debug(tmp, "Received remembered", link);
-                } else {
-                        nexthop_enter_configured(tmp);
-                        log_nexthop_debug(tmp, "Remembering", link);
-
-                        r = nexthop_add(m, link, tmp);
-                        if (r < 0) {
-                                log_link_warning_errno(link, r, "Could not remember foreign nexthop, ignoring: %m");
-                                return 0;
-                        }
-
-                        TAKE_PTR(tmp);
-                }
+        if (!nexthop_bound_to_link(nexthop))
+                nexthop->ifindex = 0;
 
-                break;
-        case RTM_DELNEXTHOP:
-                if (nexthop) {
-                        nexthop_enter_removed(nexthop);
-                        if (nexthop->state == 0) {
-                                log_nexthop_debug(nexthop, "Forgetting", link);
-                                nexthop_free(nexthop);
-                        } else
-                                log_nexthop_debug(nexthop, "Removed", link);
-                } else
-                        log_nexthop_debug(tmp, "Kernel removed unknown", link);
-                break;
-
-        default:
-                assert_not_reached();
-        }
+        nexthop_enter_configured(nexthop);
+        if (req)
+                nexthop_enter_configured(req->userdata);
 
+        log_nexthop_debug(nexthop, is_new ? "Remembering" : "Received remembered", m);
         return 1;
 }
 
@@ -988,6 +1133,13 @@ static int nexthop_section_verify(NextHop *nh) {
         if (section_is_invalid(nh->section))
                 return -EINVAL;
 
+        if (!nh->network->manager->manage_foreign_nexthops && nh->id == 0)
+                return log_warning_errno(SYNTHETIC_ERRNO(EINVAL),
+                                         "%s: [NextHop] section without specifying Id= is not supported "
+                                         "if ManageForeignNextHops=no is set in networkd.conf. "
+                                         "Ignoring [NextHop] section from line %u.",
+                                         nh->section->filename, nh->section->line);
+
         if (!hashmap_isempty(nh->group)) {
                 if (in_addr_is_set(nh->family, &nh->gw))
                         return log_warning_errno(SYNTHETIC_ERRNO(EINVAL),
@@ -1001,20 +1153,34 @@ static int nexthop_section_verify(NextHop *nh) {
                                                  "Ignoring [NextHop] section from line %u.",
                                                  nh->section->filename, nh->section->line);
 
-                if (nh->blackhole && in_addr_is_set(nh->family, &nh->gw))
+                if (nh->blackhole)
                         return log_warning_errno(SYNTHETIC_ERRNO(EINVAL),
                                                  "%s: nexthop group cannot be a blackhole. "
                                                  "Ignoring [NextHop] section from line %u.",
                                                  nh->section->filename, nh->section->line);
+
+                if (nh->onlink > 0)
+                        return log_warning_errno(SYNTHETIC_ERRNO(EINVAL),
+                                                 "%s: nexthop group cannot have on-link flag. "
+                                                 "Ignoring [NextHop] section from line %u.",
+                                                 nh->section->filename, nh->section->line);
         } else if (nh->family == AF_UNSPEC)
                 /* When neither Family=, Gateway=, nor Group= is specified, assume IPv4. */
                 nh->family = AF_INET;
 
-        if (nh->blackhole && in_addr_is_set(nh->family, &nh->gw))
-                return log_warning_errno(SYNTHETIC_ERRNO(EINVAL),
-                                         "%s: blackhole nexthop cannot have gateway address. "
-                                         "Ignoring [NextHop] section from line %u.",
-                                         nh->section->filename, nh->section->line);
+        if (nh->blackhole) {
+                if (in_addr_is_set(nh->family, &nh->gw))
+                        return log_warning_errno(SYNTHETIC_ERRNO(EINVAL),
+                                                 "%s: blackhole nexthop cannot have gateway address. "
+                                                 "Ignoring [NextHop] section from line %u.",
+                                                 nh->section->filename, nh->section->line);
+
+                if (nh->onlink > 0)
+                        return log_warning_errno(SYNTHETIC_ERRNO(EINVAL),
+                                                 "%s: blackhole nexthop cannot have on-link flag. "
+                                                 "Ignoring [NextHop] section from line %u.",
+                                                 nh->section->filename, nh->section->line);
+        }
 
         if (nh->onlink < 0 && in_addr_is_set(nh->family, &nh->gw) &&
             ordered_hashmap_isempty(nh->network->addresses_by_section)) {
@@ -1032,14 +1198,68 @@ static int nexthop_section_verify(NextHop *nh) {
         return 0;
 }
 
-void network_drop_invalid_nexthops(Network *network) {
+int network_drop_invalid_nexthops(Network *network) {
+        _cleanup_hashmap_free_ Hashmap *nexthops = NULL;
         NextHop *nh;
+        int r;
 
         assert(network);
 
-        HASHMAP_FOREACH(nh, network->nexthops_by_section)
-                if (nexthop_section_verify(nh) < 0)
-                        nexthop_free(nh);
+        ORDERED_HASHMAP_FOREACH(nh, network->nexthops_by_section) {
+                if (nexthop_section_verify(nh) < 0) {
+                        nexthop_detach(nh);
+                        continue;
+                }
+
+                if (nh->id == 0)
+                        continue;
+
+                /* Always use the setting specified later. So, remove the previously assigned setting. */
+                NextHop *dup = hashmap_remove(nexthops, UINT32_TO_PTR(nh->id));
+                if (dup) {
+                        log_warning("%s: Duplicated nexthop settings for ID %"PRIu32" is specified at line %u and %u, "
+                                    "dropping the nexthop setting specified at line %u.",
+                                    dup->section->filename,
+                                    nh->id, nh->section->line,
+                                    dup->section->line, dup->section->line);
+                        /* nexthop_detach() will drop the nexthop from nexthops_by_section. */
+                        nexthop_detach(dup);
+                }
+
+                r = hashmap_ensure_put(&nexthops, NULL, UINT32_TO_PTR(nh->id), nh);
+                if (r < 0)
+                        return log_oom();
+                assert(r > 0);
+        }
+
+        return 0;
+}
+
+int manager_build_nexthop_ids(Manager *manager) {
+        Network *network;
+        int r;
+
+        assert(manager);
+
+        if (!manager->manage_foreign_nexthops)
+                return 0;
+
+        manager->nexthop_ids = set_free(manager->nexthop_ids);
+
+        ORDERED_HASHMAP_FOREACH(network, manager->networks) {
+                NextHop *nh;
+
+                ORDERED_HASHMAP_FOREACH(nh, network->nexthops_by_section) {
+                        if (nh->id == 0)
+                                continue;
+
+                        r = set_ensure_put(&manager->nexthop_ids, NULL, UINT32_TO_PTR(nh->id));
+                        if (r < 0)
+                                return r;
+                }
+        }
+
+        return 0;
 }
 
 int config_parse_nexthop_id(
@@ -1054,7 +1274,7 @@ int config_parse_nexthop_id(
                 void *data,
                 void *userdata) {
 
-        _cleanup_(nexthop_free_or_set_invalidp) NextHop *n = NULL;
+        _cleanup_(nexthop_unref_or_set_invalidp) NextHop *n = NULL;
         Network *network = userdata;
         uint32_t id;
         int r;
@@ -1104,7 +1324,7 @@ int config_parse_nexthop_gateway(
                 void *data,
                 void *userdata) {
 
-        _cleanup_(nexthop_free_or_set_invalidp) NextHop *n = NULL;
+        _cleanup_(nexthop_unref_or_set_invalidp) NextHop *n = NULL;
         Network *network = userdata;
         int r;
 
@@ -1149,7 +1369,7 @@ int config_parse_nexthop_family(
                 void *data,
                 void *userdata) {
 
-        _cleanup_(nexthop_free_or_set_invalidp) NextHop *n = NULL;
+        _cleanup_(nexthop_unref_or_set_invalidp) NextHop *n = NULL;
         Network *network = userdata;
         AddressFamily a;
         int r;
@@ -1215,7 +1435,7 @@ int config_parse_nexthop_onlink(
                 void *data,
                 void *userdata) {
 
-        _cleanup_(nexthop_free_or_set_invalidp) NextHop *n = NULL;
+        _cleanup_(nexthop_unref_or_set_invalidp) NextHop *n = NULL;
         Network *network = userdata;
         int r;
 
@@ -1252,7 +1472,7 @@ int config_parse_nexthop_blackhole(
                 void *data,
                 void *userdata) {
 
-        _cleanup_(nexthop_free_or_set_invalidp) NextHop *n = NULL;
+        _cleanup_(nexthop_unref_or_set_invalidp) NextHop *n = NULL;
         Network *network = userdata;
         int r;
 
@@ -1291,7 +1511,7 @@ int config_parse_nexthop_group(
                 void *data,
                 void *userdata) {
 
-        _cleanup_(nexthop_free_or_set_invalidp) NextHop *n = NULL;
+        _cleanup_(nexthop_unref_or_set_invalidp) NextHop *n = NULL;
         Network *network = userdata;
         int r;