Adding upstream version 2.1.148.upstream/2.1.148upstream

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

1 files changed, 332 insertions, 126 deletions

diff --git a/src/tree_data_new.c b/src/tree_data_new.c
index 752b181..5d9f429 100644
--- a/src/tree_data_new.c
+++ b/src/tree_data_new.c
@@ -51,7 +51,7 @@
 #include "xpath.h"
 
 LY_ERR
-lyd_create_term(const struct lysc_node *schema, const char *value, size_t value_len, ly_bool *dynamic,
+lyd_create_term(const struct lysc_node *schema, const char *value, size_t value_len, ly_bool is_utf8, ly_bool *dynamic,
         LY_VALUE_FORMAT format, void *prefix_data, uint32_t hints, ly_bool *incomplete, struct lyd_node **node)
 {
     LY_ERR ret;
@@ -68,7 +68,7 @@ lyd_create_term(const struct lysc_node *schema, const char *value, size_t value_
 
     LOG_LOCSET(schema, NULL, NULL, NULL);
     ret = lyd_value_store(schema->module->ctx, &term->value, ((struct lysc_node_leaf *)term->schema)->type, value,
-            value_len, dynamic, format, prefix_data, hints, schema, incomplete);
+            value_len, is_utf8, dynamic, format, prefix_data, hints, schema, incomplete);
     LOG_LOCBACK(1, 0, 0, 0);
     LY_CHECK_ERR_RET(ret, free(term), ret);
     lyd_hash(&term->node);
@@ -134,10 +134,14 @@ lyd_create_inner(const struct lysc_node *schema, struct lyd_node **node)
 }
 
 LY_ERR
-lyd_create_list(const struct lysc_node *schema, const struct ly_path_predicate *predicates, struct lyd_node **node)
+lyd_create_list(const struct lysc_node *schema, const struct ly_path_predicate *predicates, const struct lyxp_var *vars,
+        struct lyd_node **node)
 {
     LY_ERR ret = LY_SUCCESS;
     struct lyd_node *list = NULL, *key;
+    const struct lyd_value *value;
+    struct lyd_value val = {0};
+    struct lyxp_var *var;
     LY_ARRAY_COUNT_TYPE u;
 
     assert((schema->nodetype == LYS_LIST) && !(schema->flags & LYS_KEYLESS));
@@ -149,7 +153,31 @@ lyd_create_list(const struct lysc_node *schema, const struct ly_path_predicate *
 
     /* create and insert all the keys */
     LY_ARRAY_FOR(predicates, u) {
-        LY_CHECK_GOTO(ret = lyd_create_term2(predicates[u].key, &predicates[u].value, &key), cleanup);
+        if (predicates[u].type == LY_PATH_PREDTYPE_LIST_VAR) {
+            /* find the var */
+            if ((ret = lyxp_vars_find(schema->module->ctx, vars, predicates[u].variable, 0, &var))) {
+                goto cleanup;
+            }
+
+            /* store the value */
+            LOG_LOCSET(predicates[u].key, NULL, NULL, NULL);
+            ret = lyd_value_store(schema->module->ctx, &val, ((struct lysc_node_leaf *)predicates[u].key)->type,
+                    var->value, strlen(var->value), 0, NULL, LY_VALUE_JSON, NULL, LYD_HINT_DATA, predicates[u].key, NULL);
+            LOG_LOCBACK(1, 0, 0, 0);
+            LY_CHECK_GOTO(ret, cleanup);
+
+            value = &val;
+        } else {
+            assert(predicates[u].type == LY_PATH_PREDTYPE_LIST);
+            value = &predicates[u].value;
+        }
+
+        ret = lyd_create_term2(predicates[u].key, value, &key);
+        if (val.realtype) {
+            val.realtype->plugin->free(schema->module->ctx, &val);
+            memset(&val, 0, sizeof val);
+        }
+        LY_CHECK_GOTO(ret, cleanup);
         lyd_insert_node(list, NULL, key, 0);
     }
 
@@ -172,7 +200,6 @@ lyd_create_list2(const struct lysc_node *schema, const char *keys, size_t keys_l
     LY_ERR ret = LY_SUCCESS;
     struct lyxp_expr *expr = NULL;
     uint32_t exp_idx = 0;
-    enum ly_path_pred_type pred_type = 0;
     struct ly_path_predicate *predicates = NULL;
 
     LOG_LOCSET(schema, NULL, NULL, NULL);
@@ -183,100 +210,111 @@ lyd_create_list2(const struct lysc_node *schema, const char *keys, size_t keys_l
 
     /* compile them */
     LY_CHECK_GOTO(ret = ly_path_compile_predicate(schema->module->ctx, NULL, NULL, schema, expr, &exp_idx, LY_VALUE_JSON,
-            NULL, &predicates, &pred_type), cleanup);
+            NULL, &predicates), cleanup);
 
     /* create the list node */
-    LY_CHECK_GOTO(ret = lyd_create_list(schema, predicates, node), cleanup);
+    LY_CHECK_GOTO(ret = lyd_create_list(schema, predicates, NULL, node), cleanup);
 
 cleanup:
     LOG_LOCBACK(1, 0, 0, 0);
     lyxp_expr_free(schema->module->ctx, expr);
-    ly_path_predicates_free(schema->module->ctx, pred_type, predicates);
+    ly_path_predicates_free(schema->module->ctx, predicates);
     return ret;
 }
 
 /**
- * @brief Convert an anydata value into a datatree.
+ * @brief Learn actual any value type in case it is currently ::LYD_ANYDATA_STRING.
+ *
+ * @param[in] value Any value.
+ * @param[out] value_type Detected value type.
+ */
+static void
+lyd_create_any_string_valtype(const void *value, LYD_ANYDATA_VALUETYPE *value_type)
+{
+    /* detect format */
+    if (!value) {
+        /* interpret it as an empty data tree */
+        *value_type = LYD_ANYDATA_DATATREE;
+    } else if (((char *)value)[0] == '<') {
+        *value_type = LYD_ANYDATA_XML;
+    } else if (((char *)value)[0] == '{') {
+        *value_type = LYD_ANYDATA_JSON;
+    } else if (!strncmp(value, "lyb", 3)) {
+        *value_type = LYD_ANYDATA_LYB;
+    } else {
+        /* really just some string */
+        *value_type = LYD_ANYDATA_STRING;
+    }
+}
+
+/**
+ * @brief Convert an any value into a datatree.
  *
  * @param[in] ctx libyang context.
- * @param[in] value Anydata value.
- * @param[in] value_type Anydata @p value type.
+ * @param[in] value_in Any value as an input.
+ * @param[in] value_type Any @p value type.
+ * @param[in] log Whether parsing errors should be logged.
  * @param[out] tree Parsed data tree.
  * @return LY_ERR value.
  */
 static LY_ERR
-lyd_create_anydata_datatree(const struct ly_ctx *ctx, const void *value, LYD_ANYDATA_VALUETYPE value_type,
+lyd_create_any_datatree(const struct ly_ctx *ctx, struct ly_in *value_in, LYD_ANYDATA_VALUETYPE value_type, ly_bool log,
         struct lyd_node **tree)
 {
-    LY_ERR r;
-    struct ly_in *in = NULL;
+    LY_ERR rc = LY_SUCCESS;
     struct lyd_ctx *lydctx = NULL;
-    uint32_t parse_opts, int_opts;
+    uint32_t parse_opts, int_opts, log_opts = 0;
 
     *tree = NULL;
 
-    if (!value) {
-        /* empty data tree no matter the value type */
-        return LY_SUCCESS;
-    }
-
-    if (value_type == LYD_ANYDATA_STRING) {
-        /* detect format */
-        if (((char *)value)[0] == '<') {
-            value_type = LYD_ANYDATA_XML;
-        } else if (((char *)value)[0] == '{') {
-            value_type = LYD_ANYDATA_JSON;
-        } else if (!strncmp(value, "lyb", 3)) {
-            value_type = LYD_ANYDATA_LYB;
-        } else {
-            LOGERR(ctx, LY_EINVAL, "Invalid string value of an anydata node.");
-            return LY_EINVAL;
-        }
-    }
-
-    /* create input */
-    LY_CHECK_RET(ly_in_new_memory(value, &in));
-
     /* set options */
     parse_opts = LYD_PARSE_ONLY | LYD_PARSE_OPAQ;
     int_opts = LYD_INTOPT_ANY | LYD_INTOPT_WITH_SIBLINGS;
 
+    if (!log) {
+        /* no logging */
+        ly_temp_log_options(&log_opts);
+    }
+
     switch (value_type) {
     case LYD_ANYDATA_DATATREE:
     case LYD_ANYDATA_STRING:
         /* unreachable */
-        ly_in_free(in, 0);
         LOGINT_RET(ctx);
     case LYD_ANYDATA_XML:
-        r = lyd_parse_xml(ctx, NULL, NULL, tree, in, parse_opts, 0, int_opts, NULL, NULL, &lydctx);
+        rc = lyd_parse_xml(ctx, NULL, NULL, tree, value_in, parse_opts, 0, int_opts, NULL, NULL, &lydctx);
         break;
     case LYD_ANYDATA_JSON:
-        r = lyd_parse_json(ctx, NULL, NULL, tree, in, parse_opts, 0, int_opts, NULL, NULL, &lydctx);
+        rc = lyd_parse_json(ctx, NULL, NULL, tree, value_in, parse_opts, 0, int_opts, NULL, NULL, &lydctx);
         break;
     case LYD_ANYDATA_LYB:
-        r = lyd_parse_lyb(ctx, NULL, NULL, tree, in, parse_opts | LYD_PARSE_STRICT, 0, int_opts, NULL, NULL, &lydctx);
+        rc = lyd_parse_lyb(ctx, NULL, NULL, tree, value_in, parse_opts | LYD_PARSE_STRICT, 0, int_opts, NULL, NULL, &lydctx);
         break;
     }
     if (lydctx) {
         lydctx->free(lydctx);
     }
-    ly_in_free(in, 0);
 
-    if (r) {
-        LOGERR(ctx, LY_EINVAL, "Failed to parse anydata content into a data tree.");
-        return LY_EINVAL;
+    if (!log) {
+        /* restore logging */
+        ly_temp_log_options(NULL);
     }
-    return LY_SUCCESS;
+    if (rc && *tree) {
+        lyd_free_siblings(*tree);
+        *tree = NULL;
+    }
+    return rc;
 }
 
 LY_ERR
 lyd_create_any(const struct lysc_node *schema, const void *value, LYD_ANYDATA_VALUETYPE value_type, ly_bool use_value,
         struct lyd_node **node)
 {
-    LY_ERR ret;
+    LY_ERR rc = LY_SUCCESS, r;
     struct lyd_node *tree;
     struct lyd_node_any *any = NULL;
     union lyd_any_value any_val;
+    struct ly_in *in = NULL;
 
     assert(schema->nodetype & LYD_NODE_ANY);
 
@@ -287,15 +325,70 @@ lyd_create_any(const struct lysc_node *schema, const void *value, LYD_ANYDATA_VA
     any->prev = &any->node;
     any->flags = LYD_NEW;
 
-    if ((schema->nodetype == LYS_ANYDATA) && (value_type != LYD_ANYDATA_DATATREE)) {
-        /* only a data tree can be stored */
-        LY_CHECK_GOTO(ret = lyd_create_anydata_datatree(schema->module->ctx, value, value_type, &tree), error);
-        if (use_value) {
-            free((void *)value);
+    if (schema->nodetype == LYS_ANYDATA) {
+        /* anydata */
+        if (value_type == LYD_ANYDATA_STRING) {
+            /* detect value type */
+            lyd_create_any_string_valtype(value, &value_type);
+        }
+
+        if (value_type != LYD_ANYDATA_DATATREE) {
+            /* create input */
+            assert(value);
+            LY_CHECK_GOTO(rc = ly_in_new_memory(value, &in), cleanup);
+
+            /* parse as a data tree */
+            if ((r = lyd_create_any_datatree(schema->module->ctx, in, value_type, 1, &tree))) {
+                LOGERR(schema->module->ctx, rc, "Failed to parse any content into a data tree.");
+                rc = r;
+                goto cleanup;
+            }
+
+            /* use the parsed data tree */
+            if (use_value) {
+                free((void *)value);
+            }
+            use_value = 1;
+            value = tree;
+            value_type = LYD_ANYDATA_DATATREE;
+        }
+    } else {
+        /* anyxml */
+        switch (value_type) {
+        case LYD_ANYDATA_DATATREE:
+            /* fine, just use the value */
+            break;
+        case LYD_ANYDATA_STRING:
+            /* detect value type */
+            lyd_create_any_string_valtype(value, &value_type);
+            if ((value_type == LYD_ANYDATA_DATATREE) || (value_type == LYD_ANYDATA_STRING)) {
+                break;
+            }
+        /* fallthrough */
+        case LYD_ANYDATA_XML:
+        case LYD_ANYDATA_JSON:
+        case LYD_ANYDATA_LYB:
+            if (!value) {
+                /* nothing to parse */
+                break;
+            }
+
+            /* create input */
+            LY_CHECK_GOTO(rc = ly_in_new_memory(value, &in), cleanup);
+
+            /* try to parse as a data tree */
+            r = lyd_create_any_datatree(schema->module->ctx, in, value_type, 0, &tree);
+            if (!r) {
+                /* use the parsed data tree */
+                if (use_value) {
+                    free((void *)value);
+                }
+                use_value = 1;
+                value = tree;
+                value_type = LYD_ANYDATA_DATATREE;
+            }
+            break;
         }
-        use_value = 1;
-        value = tree;
-        value_type = LYD_ANYDATA_DATATREE;
     }
 
     if (use_value) {
@@ -306,7 +399,7 @@ lyd_create_any(const struct lysc_node *schema, const void *value, LYD_ANYDATA_VA
         case LYD_ANYDATA_STRING:
         case LYD_ANYDATA_XML:
         case LYD_ANYDATA_JSON:
-            LY_CHECK_GOTO(ret = lydict_insert_zc(schema->module->ctx, (void *)value, &any->value.str), error);
+            LY_CHECK_GOTO(rc = lydict_insert_zc(schema->module->ctx, (void *)value, &any->value.str), cleanup);
             break;
         case LYD_ANYDATA_LYB:
             any->value.mem = (void *)value;
@@ -315,16 +408,18 @@ lyd_create_any(const struct lysc_node *schema, const void *value, LYD_ANYDATA_VA
         any->value_type = value_type;
     } else {
         any_val.str = value;
-        LY_CHECK_GOTO(ret = lyd_any_copy_value(&any->node, &any_val, value_type), error);
+        LY_CHECK_GOTO(rc = lyd_any_copy_value(&any->node, &any_val, value_type), cleanup);
     }
     lyd_hash(&any->node);
 
-    *node = &any->node;
-    return LY_SUCCESS;
-
-error:
-    free(any);
-    return ret;
+cleanup:
+    if (rc) {
+        lyd_free_tree(&any->node);
+    } else {
+        *node = &any->node;
+    }
+    ly_in_free(in, 0);
+    return rc;
 }
 
 LY_ERR
@@ -444,33 +539,25 @@ lyd_new_ext_inner(const struct lysc_ext_instance *ext, const char *name, struct
 }
 
 /**
- * @brief Create a new list node in the data tree.
+ * @brief Create a new lits node instance without its keys.
  *
+ * @param[in] ctx Context to use for logging.
  * @param[in] parent Parent node for the node being created. NULL in case of creating a top level element.
  * @param[in] module Module of the node being created. If NULL, @p parent module will be used.
  * @param[in] name Schema node name of the new data node. The node must be #LYS_LIST.
- * @param[in] format Format of key values.
  * @param[in] output Flag in case the @p parent is RPC/Action. If value is 0, the input's data nodes of the RPC/Action are
  * taken into consideration. Otherwise, the output's data node is going to be created.
- * @param[out] node Optional created node.
- * @param[in] ap Ordered key values of the new list instance, all must be set. For ::LY_VALUE_LYB, every value must
- * be followed by the value length.
+ * @param[out] node Created node.
  * @return LY_ERR value.
  */
 static LY_ERR
-_lyd_new_list(struct lyd_node *parent, const struct lys_module *module, const char *name, LY_VALUE_FORMAT format,
-        ly_bool output, struct lyd_node **node, va_list ap)
+_lyd_new_list_node(const struct ly_ctx *ctx, const struct lyd_node *parent, const struct lys_module *module,
+        const char *name, ly_bool output, struct lyd_node **node)
 {
-    struct lyd_node *ret = NULL, *key;
-    const struct lysc_node *schema, *key_s;
+    struct lyd_node *ret = NULL;
+    const struct lysc_node *schema;
     struct lysc_ext_instance *ext = NULL;
-    const struct ly_ctx *ctx = parent ? LYD_CTX(parent) : (module ? module->ctx : NULL);
-    const void *key_val;
-    uint32_t key_len;
-    LY_ERR r, rc = LY_SUCCESS;
-
-    LY_CHECK_ARG_RET(ctx, parent || module, parent || node, name, LY_EINVAL);
-    LY_CHECK_CTX_EQUAL_RET(parent ? LYD_CTX(parent) : NULL, module ? module->ctx : NULL, LY_EINVAL);
+    LY_ERR r;
 
     if (!module) {
         module = parent->schema->module;
@@ -487,8 +574,47 @@ _lyd_new_list(struct lyd_node *parent, const struct lys_module *module, const ch
     /* create list inner node */
     LY_CHECK_RET(lyd_create_inner(schema, &ret));
 
+    if (ext) {
+        ret->flags |= LYD_EXT;
+    }
+
+    *node = ret;
+    return LY_SUCCESS;
+}
+
+/**
+ * @brief Create a new list node in the data tree.
+ *
+ * @param[in] parent Parent node for the node being created. NULL in case of creating a top level element.
+ * @param[in] module Module of the node being created. If NULL, @p parent module will be used.
+ * @param[in] name Schema node name of the new data node. The node must be #LYS_LIST.
+ * @param[in] format Format of key values.
+ * @param[in] output Flag in case the @p parent is RPC/Action. If value is 0, the input's data nodes of the RPC/Action are
+ * taken into consideration. Otherwise, the output's data node is going to be created.
+ * @param[out] node Optional created node.
+ * @param[in] ap Ordered key values of the new list instance, all must be set. For ::LY_VALUE_LYB, every value must
+ * be followed by the value length.
+ * @return LY_ERR value.
+ */
+static LY_ERR
+_lyd_new_list(struct lyd_node *parent, const struct lys_module *module, const char *name, LY_VALUE_FORMAT format,
+        ly_bool output, struct lyd_node **node, va_list ap)
+{
+    struct lyd_node *ret = NULL, *key;
+    const struct lysc_node *key_s;
+    const struct ly_ctx *ctx = parent ? LYD_CTX(parent) : (module ? module->ctx : NULL);
+    const void *key_val;
+    uint32_t key_len;
+    LY_ERR rc = LY_SUCCESS;
+
+    LY_CHECK_ARG_RET(ctx, parent || module, parent || node, name, LY_EINVAL);
+    LY_CHECK_CTX_EQUAL_RET(parent ? LYD_CTX(parent) : NULL, module ? module->ctx : NULL, LY_EINVAL);
+
+    /* create the list node */
+    LY_CHECK_RET(_lyd_new_list_node(ctx, parent, module, name, output, &ret));
+
     /* create and insert all the keys */
-    for (key_s = lysc_node_child(schema); key_s && (key_s->flags & LYS_KEY); key_s = key_s->next) {
+    for (key_s = lysc_node_child(ret->schema); key_s && (key_s->flags & LYS_KEY); key_s = key_s->next) {
         if (format == LY_VALUE_LYB) {
             key_val = va_arg(ap, const void *);
             key_len = va_arg(ap, uint32_t);
@@ -497,14 +623,11 @@ _lyd_new_list(struct lyd_node *parent, const struct lys_module *module, const ch
             key_len = key_val ? strlen((char *)key_val) : 0;
         }
 
-        rc = lyd_create_term(key_s, key_val, key_len, NULL, format, NULL, LYD_HINT_DATA, NULL, &key);
+        rc = lyd_create_term(key_s, key_val, key_len, 0, NULL, format, NULL, LYD_HINT_DATA, NULL, &key);
         LY_CHECK_GOTO(rc, cleanup);
         lyd_insert_node(ret, NULL, key, 1);
     }
 
-    if (ext) {
-        ret->flags |= LYD_EXT;
-    }
     if (parent) {
         lyd_insert_node(parent, NULL, ret, 0);
     }
@@ -595,7 +718,7 @@ lyd_new_ext_list(const struct lysc_ext_instance *ext, const char *name, struct l
     for (key_s = lysc_node_child(schema); key_s && (key_s->flags & LYS_KEY); key_s = key_s->next) {
         key_val = va_arg(ap, const char *);
 
-        rc = lyd_create_term(key_s, key_val, key_val ? strlen(key_val) : 0, NULL, LY_VALUE_JSON, NULL, LYD_HINT_DATA,
+        rc = lyd_create_term(key_s, key_val, key_val ? strlen(key_val) : 0, 0, NULL, LY_VALUE_JSON, NULL, LYD_HINT_DATA,
                 NULL, &key);
         LY_CHECK_GOTO(rc, cleanup);
         lyd_insert_node(ret, NULL, key, 1);
@@ -661,6 +784,84 @@ lyd_new_list2(struct lyd_node *parent, const struct lys_module *module, const ch
 }
 
 /**
+ * @brief Create a new list node in the data tree.
+ *
+ * @param[in] parent Parent node for the node being created. NULL in case of creating a top level element.
+ * @param[in] module Module of the node being created. If NULL, @p parent module will be used.
+ * @param[in] name Schema node name of the new data node. The node must be #LYS_LIST.
+ * @param[in] format Format of key values.
+ * @param[in] key_values Ordered key values of the new list instance ended with NULL, all must be set.
+ * @param[in] value_lengths Lengths of @p key_values, required for ::LY_VALUE_LYB, optional otherwise.
+ * @param[in] output Flag in case the @p parent is RPC/Action. If value is 0, the input's data nodes of the RPC/Action are
+ * taken into consideration. Otherwise, the output's data node is going to be created.
+ * @param[out] node Optional created node.
+ * @return LY_ERR value.
+ */
+static LY_ERR
+_lyd_new_list3(struct lyd_node *parent, const struct lys_module *module, const char *name, LY_VALUE_FORMAT format,
+        const void **key_values, uint32_t *value_lengths, ly_bool output, struct lyd_node **node)
+{
+    struct lyd_node *ret = NULL, *key;
+    const struct lysc_node *key_s;
+    const struct ly_ctx *ctx = parent ? LYD_CTX(parent) : (module ? module->ctx : NULL);
+    const void *key_val;
+    uint32_t key_len, i;
+    LY_ERR rc = LY_SUCCESS;
+
+    LY_CHECK_ARG_RET(ctx, parent || module, parent || node, name, key_values, (format != LY_VALUE_LYB) || value_lengths,
+            LY_EINVAL);
+    LY_CHECK_CTX_EQUAL_RET(parent ? LYD_CTX(parent) : NULL, module ? module->ctx : NULL, LY_EINVAL);
+
+    /* create the list node */
+    LY_CHECK_RET(_lyd_new_list_node(ctx, parent, module, name, output, &ret));
+
+    /* create and insert all the keys */
+    i = 0;
+    for (key_s = lysc_node_child(ret->schema); key_s && (key_s->flags & LYS_KEY); key_s = key_s->next) {
+        key_val = key_values[i] ? key_values[i] : "";
+        key_len = value_lengths ? value_lengths[i] : strlen(key_val);
+
+        rc = lyd_create_term(key_s, key_val, key_len, 0, NULL, format, NULL, LYD_HINT_DATA, NULL, &key);
+        LY_CHECK_GOTO(rc, cleanup);
+        lyd_insert_node(ret, NULL, key, 1);
+    }
+
+    if (parent) {
+        lyd_insert_node(parent, NULL, ret, 0);
+    }
+
+cleanup:
+    if (rc) {
+        lyd_free_tree(ret);
+        ret = NULL;
+    } else if (node) {
+        *node = ret;
+    }
+    return rc;
+}
+
+LIBYANG_API_DEF LY_ERR
+lyd_new_list3(struct lyd_node *parent, const struct lys_module *module, const char *name, const char **key_values,
+        uint32_t *value_lengths, ly_bool output, struct lyd_node **node)
+{
+    return _lyd_new_list3(parent, module, name, LY_VALUE_JSON, (const void **)key_values, value_lengths, output, node);
+}
+
+LIBYANG_API_DEF LY_ERR
+lyd_new_list3_bin(struct lyd_node *parent, const struct lys_module *module, const char *name, const void **key_values,
+        uint32_t *value_lengths, ly_bool output, struct lyd_node **node)
+{
+    return _lyd_new_list3(parent, module, name, LY_VALUE_LYB, key_values, value_lengths, output, node);
+}
+
+LIBYANG_API_DEF LY_ERR
+lyd_new_list3_canon(struct lyd_node *parent, const struct lys_module *module, const char *name, const char **key_values,
+        uint32_t *value_lengths, ly_bool output, struct lyd_node **node)
+{
+    return _lyd_new_list3(parent, module, name, LY_VALUE_CANON, (const void **)key_values, value_lengths, output, node);
+}
+
+/**
  * @brief Create a new term node in the data tree.
  *
  * @param[in] parent Parent node for the node being created. NULL in case of creating a top level element.
@@ -699,7 +900,7 @@ _lyd_new_term(struct lyd_node *parent, const struct lys_module *module, const ch
     }
     LY_CHECK_ERR_RET(!schema, LOGERR(ctx, LY_EINVAL, "Term node \"%s\" not found.", name), LY_ENOTFOUND);
 
-    LY_CHECK_RET(lyd_create_term(schema, value, value_len, NULL, format, NULL, LYD_HINT_DATA, NULL, &ret));
+    LY_CHECK_RET(lyd_create_term(schema, value, value_len, 0, NULL, format, NULL, LYD_HINT_DATA, NULL, &ret));
     if (ext) {
         ret->flags |= LYD_EXT;
     }
@@ -754,7 +955,8 @@ lyd_new_ext_term(const struct lysc_ext_instance *ext, const char *name, const ch
         }
         return LY_ENOTFOUND;
     }
-    rc = lyd_create_term(schema, val_str, val_str ? strlen(val_str) : 0, NULL, LY_VALUE_JSON, NULL, LYD_HINT_DATA, NULL, &ret);
+    rc = lyd_create_term(schema, val_str, val_str ? strlen(val_str) : 0, 0, NULL, LY_VALUE_JSON, NULL, LYD_HINT_DATA,
+            NULL, &ret);
     LY_CHECK_RET(rc);
 
     *node = ret;
@@ -772,7 +974,8 @@ lyd_new_any(struct lyd_node *parent, const struct lys_module *module, const char
     struct lysc_ext_instance *ext = NULL;
     const struct ly_ctx *ctx = parent ? LYD_CTX(parent) : (module ? module->ctx : NULL);
 
-    LY_CHECK_ARG_RET(ctx, parent || module, parent || node, name, LY_EINVAL);
+    LY_CHECK_ARG_RET(ctx, parent || module, parent || node, name,
+            (value_type == LYD_ANYDATA_DATATREE) || (value_type == LYD_ANYDATA_STRING) || value, LY_EINVAL);
     LY_CHECK_CTX_EQUAL_RET(parent ? LYD_CTX(parent) : NULL, module ? module->ctx : NULL, LY_EINVAL);
 
     if (!module) {
@@ -867,7 +1070,7 @@ lyd_new_meta(const struct ly_ctx *ctx, struct lyd_node *parent, const struct lys
         val_str = "";
     }
 
-    return lyd_create_meta(parent, meta, module, name, name_len, val_str, strlen(val_str), NULL, LY_VALUE_JSON,
+    return lyd_create_meta(parent, meta, module, name, name_len, val_str, strlen(val_str), 0, NULL, LY_VALUE_JSON,
             NULL, LYD_HINT_DATA, parent ? parent->schema : NULL, clear_dflt, NULL);
 }
 
@@ -881,7 +1084,8 @@ lyd_new_meta2(const struct ly_ctx *ctx, struct lyd_node *parent, ly_bool clear_d
     LY_CHECK_CTX_EQUAL_RET(ctx, parent ? LYD_CTX(parent) : NULL, LY_EINVAL);
 
     if (parent && !parent->schema) {
-        LOGERR(ctx, LY_EINVAL, "Cannot add metadata to an opaque node \"%s\".", ((struct lyd_node_opaq *)parent)->name);
+        LOGERR(ctx, LY_EINVAL, "Cannot add metadata to an opaque node \"%s\".",
+                ((struct lyd_node_opaq *)parent)->name.name);
         return LY_EINVAL;
     }
     if (meta) {
@@ -908,7 +1112,7 @@ lyd_new_meta2(const struct ly_ctx *ctx, struct lyd_node *parent, ly_bool clear_d
     }
 
     return lyd_create_meta(parent, meta, mod, attr->name.name, strlen(attr->name.name), attr->value, strlen(attr->value),
-            NULL, attr->format, attr->val_prefix_data, attr->hints, parent ? parent->schema : NULL, clear_dflt, NULL);
+            0, NULL, attr->format, attr->val_prefix_data, attr->hints, parent ? parent->schema : NULL, clear_dflt, NULL);
 }
 
 LIBYANG_API_DEF LY_ERR
@@ -1094,7 +1298,8 @@ _lyd_change_term(struct lyd_node *term, const void *value, size_t value_len, LY_
 
     /* parse the new value */
     LOG_LOCSET(term->schema, term, NULL, NULL);
-    ret = lyd_value_store(LYD_CTX(term), &val, type, value, value_len, NULL, format, NULL, LYD_HINT_DATA, term->schema, NULL);
+    ret = lyd_value_store(LYD_CTX(term), &val, type, value, value_len, 0, NULL, format, NULL, LYD_HINT_DATA,
+            term->schema, NULL);
     LOG_LOCBACK(term->schema ? 1 : 0, 1, 0, 0);
     LY_CHECK_GOTO(ret, cleanup);
 
@@ -1195,7 +1400,7 @@ lyd_change_meta(struct lyd_meta *meta, const char *val_str)
 
     /* parse the new value into a new meta structure */
     ret = lyd_create_meta(NULL, &m2, meta->annotation->module, meta->name, strlen(meta->name), val_str, strlen(val_str),
-            NULL, LY_VALUE_JSON, NULL, LYD_HINT_DATA, meta->parent ? meta->parent->schema : NULL, 0, NULL);
+            0, NULL, LY_VALUE_JSON, NULL, LYD_HINT_DATA, meta->parent ? meta->parent->schema : NULL, 0, NULL);
     LY_CHECK_GOTO(ret, cleanup);
 
     /* compare original and new value */
@@ -1320,18 +1525,20 @@ lyd_new_path_check_find_lypath(struct ly_path *path, const char *str_path, const
         schema = path[u].node;
 
         if (lysc_is_dup_inst_list(schema)) {
-            if (path[u].pred_type == LY_PATH_PREDTYPE_NONE) {
+            if (!path[u].predicates ||
+                    ((schema->nodetype == LYS_LEAFLIST) && (path[u].predicates[0].type == LY_PATH_PREDTYPE_LEAFLIST))) {
                 /* creating a new key-less list or state leaf-list instance */
                 create = 1;
                 new_count = u;
-            } else if (path[u].pred_type != LY_PATH_PREDTYPE_POSITION) {
+            } else if (path[u].predicates[0].type != LY_PATH_PREDTYPE_POSITION) {
                 LOG_LOCSET(schema, NULL, NULL, NULL);
-                LOGVAL(schema->module->ctx, LYVE_XPATH, "Invalid predicate for %s \"%s\" in path \"%s\".",
+                LOGVAL(schema->module->ctx, LYVE_XPATH, "Invalid predicate for state %s \"%s\" in path \"%s\".",
                         lys_nodetype2str(schema->nodetype), schema->name, str_path);
                 LOG_LOCBACK(1, 0, 0, 0);
                 return LY_EINVAL;
             }
-        } else if ((schema->nodetype == LYS_LIST) && (path[u].pred_type != LY_PATH_PREDTYPE_LIST)) {
+        } else if ((schema->nodetype == LYS_LIST) &&
+                (!path[u].predicates || (path[u].predicates[0].type != LY_PATH_PREDTYPE_LIST))) {
             if ((u < LY_ARRAY_COUNT(path) - 1) || !(options & LYD_NEW_PATH_OPAQ)) {
                 LOG_LOCSET(schema, NULL, NULL, NULL);
                 LOGVAL(schema->module->ctx, LYVE_XPATH, "Predicate missing for %s \"%s\" in path \"%s\".",
@@ -1339,7 +1546,8 @@ lyd_new_path_check_find_lypath(struct ly_path *path, const char *str_path, const
                 LOG_LOCBACK(1, 0, 0, 0);
                 return LY_EINVAL;
             } /* else creating an opaque list */
-        } else if ((schema->nodetype == LYS_LEAFLIST) && (path[u].pred_type != LY_PATH_PREDTYPE_LEAFLIST)) {
+        } else if ((schema->nodetype == LYS_LEAFLIST) &&
+                (!path[u].predicates || (path[u].predicates[0].type != LY_PATH_PREDTYPE_LEAFLIST))) {
             r = LY_SUCCESS;
             if (options & LYD_NEW_PATH_OPAQ) {
                 r = lyd_value_validate(NULL, schema, value, value_len, NULL, NULL, NULL);
@@ -1347,12 +1555,12 @@ lyd_new_path_check_find_lypath(struct ly_path *path, const char *str_path, const
             if (!r) {
                 /* try to store the value */
                 LY_CHECK_RET(lyd_value_store(schema->module->ctx, &val, ((struct lysc_node_leaflist *)schema)->type,
-                        value, value_len, NULL, format, NULL, LYD_HINT_DATA, schema, NULL));
+                        value, value_len, 0, NULL, format, NULL, LYD_HINT_DATA, schema, NULL));
                 ++((struct lysc_type *)val.realtype)->refcount;
 
                 /* store the new predicate so that it is used when searching for this instance */
-                path[u].pred_type = LY_PATH_PREDTYPE_LEAFLIST;
                 LY_ARRAY_NEW_RET(schema->module->ctx, path[u].predicates, pred, LY_EMEM);
+                pred->type = LY_PATH_PREDTYPE_LEAFLIST;
                 pred->value = val;
             } /* else we have opaq flag and the value is not valid, leave no predicate and then create an opaque node */
         }
@@ -1429,7 +1637,7 @@ lyd_new_path_(struct lyd_node *parent, const struct ly_ctx *ctx, const struct ly
     }
 
     /* parse path */
-    LY_CHECK_GOTO(ret = ly_path_parse(ctx, NULL, path, strlen(path), 0, LY_PATH_BEGIN_EITHER, LY_PATH_PREFIX_OPTIONAL,
+    LY_CHECK_GOTO(ret = ly_path_parse(ctx, NULL, path, strlen(path), 0, LY_PATH_BEGIN_EITHER, LY_PATH_PREFIX_FIRST,
             LY_PATH_PRED_SIMPLE, &exp), cleanup);
 
     /* compile path */
@@ -1442,32 +1650,36 @@ lyd_new_path_(struct lyd_node *parent, const struct ly_ctx *ctx, const struct ly
 
     /* try to find any existing nodes in the path */
     if (parent) {
-        ret = ly_path_eval_partial(p, parent, &path_idx, &node);
-        if (ret == LY_SUCCESS) {
+        r = ly_path_eval_partial(p, parent, NULL, options & LYD_NEW_PATH_WITH_OPAQ, &path_idx, &node);
+        if (r == LY_SUCCESS) {
             if (orig_count == LY_ARRAY_COUNT(p)) {
                 /* the node exists, are we supposed to update it or is it just a default? */
                 if (!(options & LYD_NEW_PATH_UPDATE) && !(node->flags & LYD_DEFAULT)) {
                     LOG_LOCSET(NULL, node, NULL, NULL);
-                    LOGVAL(ctx, LYVE_REFERENCE, "Path \"%s\" already exists", path);
+                    LOGVAL(ctx, LYVE_REFERENCE, "Path \"%s\" already exists.", path);
                     LOG_LOCBACK(0, 1, 0, 0);
                     ret = LY_EEXIST;
                     goto cleanup;
+                } else if ((options & LYD_NEW_PATH_UPDATE) && lysc_is_key(node->schema)) {
+                    /* fine, the key value must not be changed and has to be in the predicate to be found */
+                    goto cleanup;
                 }
 
                 /* update the existing node */
                 ret = lyd_new_path_update(node, value, value_len, value_type, format, &nparent, &nnode);
                 goto cleanup;
             } /* else we were not searching for the whole path */
-        } else if (ret == LY_EINCOMPLETE) {
+        } else if (r == LY_EINCOMPLETE) {
             /* some nodes were found, adjust the iterator to the next segment */
             ++path_idx;
-        } else if (ret == LY_ENOTFOUND) {
+        } else if (r == LY_ENOTFOUND) {
             /* we will create the nodes from top-level, default behavior (absolute path), or from the parent (relative path) */
             if (lysc_data_parent(p[0].node)) {
                 node = parent;
             }
         } else {
             /* error */
+            ret = r;
             goto cleanup;
         }
     }
@@ -1487,15 +1699,14 @@ lyd_new_path_(struct lyd_node *parent, const struct ly_ctx *ctx, const struct ly
             if (lysc_is_dup_inst_list(schema)) {
                 /* create key-less list instance */
                 LY_CHECK_GOTO(ret = lyd_create_inner(schema, &node), cleanup);
-            } else if ((options & LYD_NEW_PATH_OPAQ) && (p[path_idx].pred_type == LY_PATH_PREDTYPE_NONE)) {
+            } else if ((options & LYD_NEW_PATH_OPAQ) && !p[path_idx].predicates) {
                 /* creating opaque list without keys */
                 LY_CHECK_GOTO(ret = lyd_create_opaq(ctx, schema->name, strlen(schema->name), NULL, 0,
                         schema->module->name, strlen(schema->module->name), NULL, 0, NULL, LY_VALUE_JSON, NULL,
                         LYD_NODEHINT_LIST, &node), cleanup);
             } else {
                 /* create standard list instance */
-                assert(p[path_idx].pred_type == LY_PATH_PREDTYPE_LIST);
-                LY_CHECK_GOTO(ret = lyd_create_list(schema, p[path_idx].predicates, &node), cleanup);
+                LY_CHECK_GOTO(ret = lyd_create_list(schema, p[path_idx].predicates, NULL, &node), cleanup);
             }
             break;
         case LYS_CONTAINER:
@@ -1505,7 +1716,8 @@ lyd_new_path_(struct lyd_node *parent, const struct ly_ctx *ctx, const struct ly
             LY_CHECK_GOTO(ret = lyd_create_inner(schema, &node), cleanup);
             break;
         case LYS_LEAFLIST:
-            if ((options & LYD_NEW_PATH_OPAQ) && (p[path_idx].pred_type != LY_PATH_PREDTYPE_LEAFLIST)) {
+            if ((options & LYD_NEW_PATH_OPAQ) &&
+                    (!p[path_idx].predicates || (p[path_idx].predicates[0].type != LY_PATH_PREDTYPE_LEAFLIST))) {
                 /* we have not checked this only for dup-inst lists, otherwise it must be opaque */
                 r = LY_EVALID;
                 if (lysc_is_dup_inst_list(schema)) {
@@ -1522,15 +1734,15 @@ lyd_new_path_(struct lyd_node *parent, const struct ly_ctx *ctx, const struct ly
             }
 
             /* get value to set */
-            if (p[path_idx].pred_type == LY_PATH_PREDTYPE_LEAFLIST) {
+            if (p[path_idx].predicates && (p[path_idx].predicates[0].type == LY_PATH_PREDTYPE_LEAFLIST)) {
                 val = &p[path_idx].predicates[0].value;
             }
 
             /* create a leaf-list instance */
             if (val) {
-                LY_CHECK_GOTO(ret = lyd_create_term2(schema, &p[path_idx].predicates[0].value, &node), cleanup);
+                LY_CHECK_GOTO(ret = lyd_create_term2(schema, val, &node), cleanup);
             } else {
-                LY_CHECK_GOTO(ret = lyd_create_term(schema, value, value_len, NULL, format, NULL, LYD_HINT_DATA,
+                LY_CHECK_GOTO(ret = lyd_create_term(schema, value, value_len, 0, NULL, format, NULL, LYD_HINT_DATA,
                         NULL, &node), cleanup);
             }
             break;
@@ -1560,7 +1772,7 @@ lyd_new_path_(struct lyd_node *parent, const struct ly_ctx *ctx, const struct ly
             }
 
             /* create a leaf instance */
-            LY_CHECK_GOTO(ret = lyd_create_term(schema, value, value_len, NULL, format, NULL, LYD_HINT_DATA, NULL,
+            LY_CHECK_GOTO(ret = lyd_create_term(schema, value, value_len, 0, NULL, format, NULL, LYD_HINT_DATA, NULL,
                     &node), cleanup);
             break;
         case LYS_ANYDATA:
@@ -1804,9 +2016,7 @@ lyd_new_implicit_tree(struct lyd_node *tree, uint32_t implicit_options, struct l
     }
 
     LYD_TREE_DFS_BEGIN(tree, node) {
-        /* skip added default nodes */
-        if (((node->flags & (LYD_DEFAULT | LYD_NEW)) != (LYD_DEFAULT | LYD_NEW)) &&
-                (node->schema->nodetype & LYD_NODE_INNER)) {
+        if (node->schema->nodetype & LYD_NODE_INNER) {
             LY_CHECK_GOTO(ret = lyd_new_implicit_r(node, lyd_node_child_p(node), NULL, NULL, &node_when, NULL,
                     NULL, implicit_options, diff), cleanup);
         }
@@ -1891,16 +2101,12 @@ lyd_new_implicit_module(struct lyd_node **tree, const struct lys_module *module,
 
     /* process nested nodes */
     LY_LIST_FOR(*tree, root) {
-        /* skip added default nodes */
-        if ((root->flags & (LYD_DEFAULT | LYD_NEW)) != (LYD_DEFAULT | LYD_NEW)) {
-            LY_CHECK_GOTO(ret = lyd_new_implicit_tree(root, implicit_options, diff ? &d : NULL), cleanup);
-
-            if (d) {
-                /* merge into one diff */
-                lyd_insert_sibling(*diff, d, diff);
+        LY_CHECK_GOTO(ret = lyd_new_implicit_tree(root, implicit_options, diff ? &d : NULL), cleanup);
 
-                d = NULL;
-            }
+        if (d) {
+            /* merge into one diff */
+            lyd_insert_sibling(*diff, d, diff);
+            d = NULL;
         }
     }