diff options
author | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-04-29 04:24:24 +0000 |
---|---|---|
committer | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-04-29 04:24:24 +0000 |
commit | 12e8343068b906f8b2afddc5569968a8a91fa5b0 (patch) | |
tree | 75cc5e05a4392ea0292251898f992a15a16b172b /markdown_it/rules_core/normalize.py | |
parent | Initial commit. (diff) | |
download | markdown-it-py-12e8343068b906f8b2afddc5569968a8a91fa5b0.tar.xz markdown-it-py-12e8343068b906f8b2afddc5569968a8a91fa5b0.zip |
Adding upstream version 2.1.0.upstream/2.1.0upstream
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to '')
-rw-r--r-- | markdown_it/rules_core/normalize.py | 19 |
1 files changed, 19 insertions, 0 deletions
diff --git a/markdown_it/rules_core/normalize.py b/markdown_it/rules_core/normalize.py new file mode 100644 index 0000000..bf16fd7 --- /dev/null +++ b/markdown_it/rules_core/normalize.py @@ -0,0 +1,19 @@ +"""Normalize input string.""" +import re + +from .state_core import StateCore + +# https://spec.commonmark.org/0.29/#line-ending +NEWLINES_RE = re.compile(r"\r\n?|\n") +NULL_RE = re.compile(r"\0") + + +def normalize(state: StateCore) -> None: + + # Normalize newlines + string = NEWLINES_RE.sub("\n", state.src) + + # Replace NULL characters + string = NULL_RE.sub("\uFFFD", string) + + state.src = string |