From 12e8343068b906f8b2afddc5569968a8a91fa5b0 Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Mon, 29 Apr 2024 06:24:24 +0200 Subject: Adding upstream version 2.1.0. Signed-off-by: Daniel Baumann --- markdown_it/rules_core/normalize.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 markdown_it/rules_core/normalize.py (limited to 'markdown_it/rules_core/normalize.py') diff --git a/markdown_it/rules_core/normalize.py b/markdown_it/rules_core/normalize.py new file mode 100644 index 0000000..bf16fd7 --- /dev/null +++ b/markdown_it/rules_core/normalize.py @@ -0,0 +1,19 @@ +"""Normalize input string.""" +import re + +from .state_core import StateCore + +# https://spec.commonmark.org/0.29/#line-ending +NEWLINES_RE = re.compile(r"\r\n?|\n") +NULL_RE = re.compile(r"\0") + + +def normalize(state: StateCore) -> None: + + # Normalize newlines + string = NEWLINES_RE.sub("\n", state.src) + + # Replace NULL characters + string = NULL_RE.sub("\uFFFD", string) + + state.src = string -- cgit v1.2.3