1 files changed, 19 insertions, 0 deletions
diff --git a/markdown_it/rules_core/normalize.py b/markdown_it/rules_core/normalize.py
new file mode 100644
index 0000000..bf16fd7
--- /dev/null
+++ b/markdown_it/rules_core/normalize.py
@@ -0,0 +1,19 @@
+"""Normalize input string."""
+import re
+
+from .state_core import StateCore
+
+# https://spec.commonmark.org/0.29/#line-ending
+NEWLINES_RE = re.compile(r"\r\n?|\n")
+NULL_RE = re.compile(r"\0")
+
+
+def normalize(state: StateCore) -> None:
+
+    # Normalize newlines
+    string = NEWLINES_RE.sub("\n", state.src)
+
+    # Replace NULL characters
+    string = NULL_RE.sub("\uFFFD", string)
+
+    state.src = string