diff options
| author | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-04-28 14:29:10 +0000 |
|---|---|---|
| committer | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-04-28 14:29:10 +0000 |
| commit | 2aa4a82499d4becd2284cdb482213d541b8804dd (patch) | |
| tree | b80bf8bf13c3766139fbacc530efd0dd9d54394c /third_party/rust/fluent-syntax | |
| parent | Initial commit. (diff) | |
| download | firefox-2aa4a82499d4becd2284cdb482213d541b8804dd.tar.xz firefox-2aa4a82499d4becd2284cdb482213d541b8804dd.zip | |
Adding upstream version 86.0.1.upstream/86.0.1upstream
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'third_party/rust/fluent-syntax')
19 files changed, 2387 insertions, 0 deletions
diff --git a/third_party/rust/fluent-syntax/.cargo-checksum.json b/third_party/rust/fluent-syntax/.cargo-checksum.json new file mode 100644 index 0000000000..1c9404e2de --- /dev/null +++ b/third_party/rust/fluent-syntax/.cargo-checksum.json @@ -0,0 +1 @@ +{"files":{"Cargo.lock":"7ef89d17cfb66296e8599334e0b2b7a5c339351156df223ec0de011dc17551cc","Cargo.toml":"35327710f5722106689ab30c0140eae6f75fc4893e72ccc5c8e57cc6643e155a","README.md":"dcab9c1dfa8dde8d660fc4eadbbc90793180c9d3bc198dd81a2441e8610b89b5","benches/contexts/README.md":"562d317f507caedd62cbe00e6b2bb350cb970168e5ebd5b3c497e68696e21e8b","benches/parser.rs":"50e1fe08dabffeb379fd208ac88260d35852ff6a2a72701b1902b315bc5c18a3","src/ast/helper.rs":"39187bbc97823ba0f3a9baadd76ddc59140bf09cddeb92997dd872e41b305375","src/ast/mod.rs":"86b203d6537a4fe0c761d9180d4663352fe14e55aca5a34c0571e1798693b343","src/bin/parser.rs":"08f4092a9fc8b7346d3935cbf90a663d5ba2ace593f3becf4622bc2e08db43f4","src/bin/update_fixtures.rs":"b4e25f1dbe7b6289d7cdab8114bdd11553787eabf570d3fc04c9ba87da6cee98","src/lib.rs":"ccbc6ff63ef90c280755bf8ef7d056ff3f586708e418e290d3faab0fc4c37889","src/parser/comment.rs":"1ac8500651f812c3187326687cb5f445e5e686d586bc32732ddb0f31560dc71e","src/parser/errors.rs":"1e2e90b873d2c92011d883b8ca93a2a0eda2e1c576b3d402e35902e9ffad848e","src/parser/expression.rs":"aa4a37d0f894907b4a61b61fc076e31d327aac66193c9f08586a1d03fd2d803a","src/parser/helper.rs":"71fb35290cba6b5e5b05cbc77aaadd0cc15e1b38984c83f51ac8c4d594d53ae3","src/parser/mod.rs":"02028af70c2d355699a2df2bd5b390d933df81f8d33a6f656cf3f62257202665","src/parser/pattern.rs":"3be83a1123488f7fe2d960c9f6aac72160f3305cfa509a81e1f37df84ca12522","src/parser/slice.rs":"aad45bc35ecc3ff68bcd5b2671fecb134a4535d6ee463092422dfc5cc8b25a1d","src/unicode.rs":"1736a89cb8827a4e49883c0c33f3dd89f37bb90cc92152364578fe2c3a3a1de1"},"package":"edb1016e8c600060e0099218442fff329a204f6316d6ec974d590d3281517a52"}
\ No newline at end of file diff --git a/third_party/rust/fluent-syntax/Cargo.lock b/third_party/rust/fluent-syntax/Cargo.lock new file mode 100644 index 0000000000..9cafc988ff --- /dev/null +++ b/third_party/rust/fluent-syntax/Cargo.lock @@ -0,0 +1,625 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +[[package]] +name = "atty" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" +dependencies = [ + "hermit-abi", + "libc", + "winapi", +] + +[[package]] +name = "autocfg" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a" + +[[package]] +name = "bitflags" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693" + +[[package]] +name = "bstr" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "473fc6b38233f9af7baa94fb5852dca389e3d95b8e21c8e3719301462c5d9faf" +dependencies = [ + "lazy_static", + "memchr", + "regex-automata", + "serde", +] + +[[package]] +name = "bumpalo" +version = "3.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e8c087f005730276d1096a652e92a8bacee2e2472bcc9715a74d2bec38b5820" + +[[package]] +name = "byteorder" +version = "1.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08c48aae112d48ed9f069b33538ea9e3e90aa263cfa3d1c24309612b1f7472de" + +[[package]] +name = "cast" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b9434b9a5aa1450faa3f9cb14ea0e8c53bb5d2b3c1bfd1ab4fc03e9f33fbfb0" +dependencies = [ + "rustc_version", +] + +[[package]] +name = "cfg-if" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822" + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "clap" +version = "2.33.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37e58ac78573c40708d45522f0d80fa2f01cc4f9b4e2bf749807255454312002" +dependencies = [ + "bitflags", + "textwrap", + "unicode-width", +] + +[[package]] +name = "const_fn" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd51eab21ab4fd6a3bf889e2d0958c0a6e3a61ad04260325e919e652a2a62826" + +[[package]] +name = "criterion" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70daa7ceec6cf143990669a04c7df13391d55fb27bd4079d252fca774ba244d8" +dependencies = [ + "atty", + "cast", + "clap", + "criterion-plot", + "csv", + "itertools", + "lazy_static", + "num-traits", + "oorandom", + "plotters", + "rayon", + "regex", + "serde", + "serde_cbor", + "serde_derive", + "serde_json", + "tinytemplate", + "walkdir", +] + +[[package]] +name = "criterion-plot" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e022feadec601fba1649cfa83586381a4ad31c6bf3a9ab7d408118b05dd9889d" +dependencies = [ + "cast", + "itertools", +] + +[[package]] +name = "crossbeam-channel" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dca26ee1f8d361640700bde38b2c37d8c22b3ce2d360e1fc1c74ea4b0aa7d775" +dependencies = [ + "cfg-if 1.0.0", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94af6efb46fef72616855b036a624cf27ba656ffc9be1b9a3c931cfc7749a9a9" +dependencies = [ + "cfg-if 1.0.0", + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1aaa739f95311c2c7887a76863f500026092fb1dce0161dab577e559ef3569d" +dependencies = [ + "cfg-if 1.0.0", + "const_fn", + "crossbeam-utils", + "lazy_static", + "memoffset", + "scopeguard", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02d96d1e189ef58269ebe5b97953da3274d83a93af647c2ddd6f9dab28cedb8d" +dependencies = [ + "autocfg", + "cfg-if 1.0.0", + "lazy_static", +] + +[[package]] +name = "csv" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9d58633299b24b515ac72a3f869f8b91306a3cec616a602843a383acd6f9e97" +dependencies = [ + "bstr", + "csv-core", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "csv-core" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b2466559f260f48ad25fe6317b3c8dac77b5bdb5763ac7d9d6103530663bc90" +dependencies = [ + "memchr", +] + +[[package]] +name = "either" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e78d4f1cc4ae33bbfc157ed5d5a5ef3bc29227303d595861deb238fcec4e9457" + +[[package]] +name = "fluent-syntax" +version = "0.10.1" +dependencies = [ + "criterion", + "glob", + "serde", + "serde_json", +] + +[[package]] +name = "glob" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b919933a397b79c37e33b77bb2aa3dc8eb6e165ad809e58ff75bc7db2e34574" + +[[package]] +name = "half" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d36fab90f82edc3c747f9d438e06cf0a491055896f2a279638bb5beed6c40177" + +[[package]] +name = "hermit-abi" +version = "0.1.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5aca5565f760fb5b220e499d72710ed156fdb74e631659e99377d9ebfbd13ae8" +dependencies = [ + "libc", +] + +[[package]] +name = "itertools" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "284f18f85651fe11e8a991b2adb42cb078325c996ed026d994719efcfca1d54b" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd25036021b0de88a0aff6b850051563c6516d0bf53f8638938edbb9de732736" + +[[package]] +name = "js-sys" +version = "0.3.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf3d7383929f7c9c7c2d0fa596f325832df98c3704f2c60553080f7127a58175" +dependencies = [ + "wasm-bindgen", +] + +[[package]] +name = "lazy_static" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" + +[[package]] +name = "libc" +version = "0.2.81" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1482821306169ec4d07f6aca392a4681f66c75c9918aa49641a2595db64053cb" + +[[package]] +name = "log" +version = "0.4.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fabed175da42fed1fa0746b0ea71f412aa9d35e76e95e59b192c64b9dc2bf8b" +dependencies = [ + "cfg-if 0.1.10", +] + +[[package]] +name = "memchr" +version = "2.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ee1c47aaa256ecabcaea351eae4a9b01ef39ed810004e298d2511ed284b1525" + +[[package]] +name = "memoffset" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "157b4208e3059a8f9e78d559edc658e13df41410cb3ae03979c83130067fdd87" +dependencies = [ + "autocfg", +] + +[[package]] +name = "num-traits" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a64b1ec5cda2586e284722486d802acf1f7dbdc623e2bfc57e65ca1cd099290" +dependencies = [ + "autocfg", +] + +[[package]] +name = "num_cpus" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05499f3756671c15885fee9034446956fff3f243d6077b91e5767df161f766b3" +dependencies = [ + "hermit-abi", + "libc", +] + +[[package]] +name = "oorandom" +version = "11.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ab1bc2a289d34bd04a330323ac98a1b4bc82c9d9fcb1e66b63caa84da26b575" + +[[package]] +name = "plotters" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d1685fbe7beba33de0330629da9d955ac75bd54f33d7b79f9a895590124f6bb" +dependencies = [ + "js-sys", + "num-traits", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "proc-macro2" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e0704ee1a7e00d7bb417d0770ea303c1bccbabf0ef1667dae92b5967f5f8a71" +dependencies = [ + "unicode-xid", +] + +[[package]] +name = "quote" +version = "1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "991431c3519a3f36861882da93630ce66b52918dcf1b8e2fd66b397fc96f28df" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rayon" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b0d8e0819fadc20c74ea8373106ead0600e3a67ef1fe8da56e39b9ae7275674" +dependencies = [ + "autocfg", + "crossbeam-deque", + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ab346ac5921dc62ffa9f89b7a773907511cdfa5490c572ae9be1be33e8afa4a" +dependencies = [ + "crossbeam-channel", + "crossbeam-deque", + "crossbeam-utils", + "lazy_static", + "num_cpus", +] + +[[package]] +name = "regex" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38cf2c13ed4745de91a5eb834e11c00bcc3709e773173b2ce4c56c9fbde04b9c" +dependencies = [ + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae1ded71d66a4a97f5e961fd0cb25a5f366a42a41570d16a763a69c092c26ae4" +dependencies = [ + "byteorder", +] + +[[package]] +name = "regex-syntax" +version = "0.6.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b181ba2dcf07aaccad5448e8ead58db5b742cf85dfe035e2227f137a539a189" + +[[package]] +name = "rustc_version" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a" +dependencies = [ + "semver", +] + +[[package]] +name = "ryu" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71d301d4193d031abdd79ff7e3dd721168a9572ef3fe51a1517aba235bd8f86e" + +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "scopeguard" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" + +[[package]] +name = "semver" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403" +dependencies = [ + "semver-parser", +] + +[[package]] +name = "semver-parser" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" + +[[package]] +name = "serde" +version = "1.0.118" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06c64263859d87aa2eb554587e2d23183398d617427327cf2b3d0ed8c69e4800" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_cbor" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e18acfa2f90e8b735b2836ab8d538de304cbb6729a7360729ea5a895d15a622" +dependencies = [ + "half", + "serde", +] + +[[package]] +name = "serde_derive" +version = "1.0.118" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c84d3526699cd55261af4b941e4e725444df67aa4f9e6a3564f18030d12672df" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.61" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fceb2595057b6891a4ee808f70054bd2d12f0e97f1cbb78689b59f676df325a" +dependencies = [ + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "syn" +version = "1.0.57" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4211ce9909eb971f111059df92c45640aad50a619cf55cd76476be803c4c68e6" +dependencies = [ + "proc-macro2", + "quote", + "unicode-xid", +] + +[[package]] +name = "textwrap" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060" +dependencies = [ + "unicode-width", +] + +[[package]] +name = "tinytemplate" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2ada8616fad06a2d0c455adc530de4ef57605a8120cc65da9653e0e9623ca74" +dependencies = [ + "serde", + "serde_json", +] + +[[package]] +name = "unicode-width" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9337591893a19b88d8d87f2cec1e73fad5cdfd10e5a6f349f498ad6ea2ffb1e3" + +[[package]] +name = "unicode-xid" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7fe0bb3479651439c9112f72b6c505038574c9fbb575ed1bf3b797fa39dd564" + +[[package]] +name = "walkdir" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "777182bc735b6424e1a57516d35ed72cb8019d85c8c9bf536dccb3445c1a2f7d" +dependencies = [ + "same-file", + "winapi", + "winapi-util", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3cd364751395ca0f68cafb17666eee36b63077fb5ecd972bbcd74c90c4bf736e" +dependencies = [ + "cfg-if 1.0.0", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1114f89ab1f4106e5b55e688b828c0ab0ea593a1ea7c094b141b14cbaaec2d62" +dependencies = [ + "bumpalo", + "lazy_static", + "log", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a6ac8995ead1f084a8dea1e65f194d0973800c7f571f6edd70adf06ecf77084" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5a48c72f299d80557c7c62e37e7225369ecc0c963964059509fbafe917c7549" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e7811dd7f9398f14cc76efd356f98f03aa30419dea46aa810d71e819fc97158" + +[[package]] +name = "web-sys" +version = "0.3.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "222b1ef9334f92a21d3fb53dc3fd80f30836959a90f9274a626d7e06315ba3c3" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-util" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178" +dependencies = [ + "winapi", +] + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" diff --git a/third_party/rust/fluent-syntax/Cargo.toml b/third_party/rust/fluent-syntax/Cargo.toml new file mode 100644 index 0000000000..cedd23dd6f --- /dev/null +++ b/third_party/rust/fluent-syntax/Cargo.toml @@ -0,0 +1,67 @@ +# THIS FILE IS AUTOMATICALLY GENERATED BY CARGO +# +# When uploading crates to the registry Cargo will automatically +# "normalize" Cargo.toml files for maximal compatibility +# with all versions of Cargo and also rewrite `path` dependencies +# to registry (e.g., crates.io) dependencies +# +# If you believe there's an error in this file please file an +# issue against the rust-lang/cargo repository. If you're +# editing this file be aware that the upstream Cargo.toml +# will likely look very different (and much more reasonable) + +[package] +edition = "2018" +name = "fluent-syntax" +version = "0.10.1" +authors = ["Zibi Braniecki <gandalf@mozilla.com>", "Staś Małolepszy <stas@mozilla.com>"] +include = ["src/**/*", "benches/*.rs", "Cargo.toml", "README.md"] +description = "Parser/Serializer tools for Fluent Syntax. \n" +homepage = "http://www.projectfluent.org" +readme = "README.md" +keywords = ["localization", "l10n", "i18n", "intl", "internationalization"] +categories = ["localization", "internationalization"] +license = "Apache-2.0/MIT" +repository = "https://github.com/projectfluent/fluent-rs" + +[[bin]] +name = "parser" +path = "src/bin/parser.rs" + +[[bin]] +name = "update_fixtures" +path = "src/bin/update_fixtures.rs" +required-features = ["json"] + +[[test]] +name = "parser_fixtures" +path = "tests/parser_fixtures.rs" +required-features = ["json"] + +[[bench]] +name = "parser" +harness = false +[dependencies.serde] +version = "1.0" +features = ["derive"] +optional = true + +[dependencies.serde_json] +version = "1.0" +optional = true +[dev-dependencies.criterion] +version = "0.3" + +[dev-dependencies.glob] +version = "0.3" + +[dev-dependencies.serde] +version = "1.0" +features = ["derive"] + +[dev-dependencies.serde_json] +version = "1.0" + +[features] +default = [] +json = ["serde", "serde_json"] diff --git a/third_party/rust/fluent-syntax/README.md b/third_party/rust/fluent-syntax/README.md new file mode 100644 index 0000000000..fd00855c24 --- /dev/null +++ b/third_party/rust/fluent-syntax/README.md @@ -0,0 +1,63 @@ +# Fluent Syntax + +`fluent-syntax` is a parser/serializer API for the Fluent Syntax, part of the [Project Fluent](https://projectfluent.org/), a localization +framework designed to unleash the entire expressive power of natural language translations. + +[](https://crates.io/crates/fluent-syntax) +[](https://travis-ci.org/projectfluent/fluent-rs) +[](https://coveralls.io/github/projectfluent/fluent-rs?branch=master) + +Status +------ + +The crate currently provides just a parser, which is tracking Fluent Syntax on its way to 1.0. + +Local Development +----------------- + + cargo build + cargo test + cargo bench + +When submitting a PR please use [`cargo fmt`][] (nightly). + +[`cargo fmt`]: https://github.com/rust-lang-nursery/rustfmt + + +Learn the FTL syntax +-------------------- + +FTL is a localization file format used for describing translation resources. +FTL stands for _Fluent Translation List_. + +FTL is designed to be simple to read, but at the same time allows to represent +complex concepts from natural languages like gender, plurals, conjugations, and +others. + + hello-user = Hello, { $username }! + +[Read the Fluent Syntax Guide][] in order to learn more about the syntax. If +you're a tool author you may be interested in the formal [EBNF grammar][]. + +[Read the Fluent Syntax Guide]: http://projectfluent.org/fluent/guide/ +[EBNF grammar]: https://github.com/projectfluent/fluent/tree/master/spec + + +Get Involved +------------ + +`fluent-rs` is open-source, licensed under the Apache License, Version 2.0. We +encourage everyone to take a look at our code and we'll listen to your +feedback. + + +Discuss +------- + +We'd love to hear your thoughts on Project Fluent! Whether you're a localizer +looking for a better way to express yourself in your language, or a developer +trying to make your app localizable and multilingual, or a hacker looking for +a project to contribute to, please do get in touch on discourse and the IRC channel. + + - Discourse: https://discourse.mozilla.org/c/fluent + - IRC channel: [irc://irc.mozilla.org/l20n](irc://irc.mozilla.org/l20n) diff --git a/third_party/rust/fluent-syntax/benches/contexts/README.md b/third_party/rust/fluent-syntax/benches/contexts/README.md new file mode 100644 index 0000000000..7d37ac0fe5 --- /dev/null +++ b/third_party/rust/fluent-syntax/benches/contexts/README.md @@ -0,0 +1,4 @@ +The following context is extracted from +the `browser.xhtml` localization context +from mozilla-central rev 51efc4b931f7 +from 2020-03-03. diff --git a/third_party/rust/fluent-syntax/benches/parser.rs b/third_party/rust/fluent-syntax/benches/parser.rs new file mode 100644 index 0000000000..d81b6bc1d2 --- /dev/null +++ b/third_party/rust/fluent-syntax/benches/parser.rs @@ -0,0 +1,125 @@ +use criterion::criterion_group; +use criterion::criterion_main; +use criterion::Criterion; +use std::collections::HashMap; +use std::fs; +use std::io; +use std::io::Read; + +use fluent_syntax::parser::Parser; +use fluent_syntax::unicode::{unescape_unicode, unescape_unicode_to_string}; + +fn read_file(path: &str) -> Result<String, io::Error> { + let mut f = fs::File::open(path)?; + let mut s = String::new(); + f.read_to_string(&mut s)?; + Ok(s) +} + +fn get_strings(tests: &[&'static str]) -> HashMap<&'static str, String> { + let mut ftl_strings = HashMap::new(); + for test in tests { + let path = format!("./benches/{}.ftl", test); + ftl_strings.insert(*test, read_file(&path).expect("Couldn't load file")); + } + return ftl_strings; +} + +fn get_ctxs(tests: &[&'static str]) -> HashMap<&'static str, Vec<String>> { + let mut ftl_strings = HashMap::new(); + for test in tests { + let paths = fs::read_dir(format!("./benches/contexts/{}", test)).unwrap(); + let strings = paths + .into_iter() + .map(|p| { + let p = p.unwrap().path(); + let path = p.to_str().unwrap(); + read_file(path).unwrap() + }) + .collect::<Vec<_>>(); + ftl_strings.insert(*test, strings); + } + return ftl_strings; +} + +fn parser_bench(c: &mut Criterion) { + let tests = &["simple", "preferences", "menubar"]; + let ftl_strings = get_strings(tests); + + c.bench_function_over_inputs( + "parse", + move |b, &&name| { + let source = &ftl_strings[name]; + b.iter(|| { + Parser::new(source.as_str()) + .parse() + .expect("Parsing of the FTL failed.") + }) + }, + tests, + ); +} + +fn unicode_unescape_bench(c: &mut Criterion) { + let strings = &[ + "foo", + "This is an example value", + "Hello \\u00e3\\u00e9 World", + "\\u004c\\u006f\\u0072\\u0065\\u006d \\u0069\\u0070\\u0073\\u0075\\u006d \\u0064\\u006f\\u006c\\u006f\\u0072 \\u0073\\u0069\\u0074 \\u0061\\u006d\\u0065\\u0074", + "Let me introduce \\\"The\\\" Fluent", + "And here's an example of \\\\ a character to be escaped", + "But this message is completely unescape free", + "And so is this one", + "Maybe this one is as well completely escape free", + "Welcome to Mozilla Firefox", + "\\u0054\\u0068\\u0065\\u0073\\u0065 \\u0073\\u0065\\u0074\\u0074\\u0069\\u006e\\u0067\\u0073 \\u0061\\u0072\\u0065 \\u0074\\u0061\\u0069\\u006c\\u006f\\u0072\\u0065\\u0064 \\u0074\\u006f \\u0079\\u006f\\u0075\\u0072 \\u0063\\u006f\\u006d\\u0070\\u0075\\u0074\\u0065\\u0072\\u2019\\u0073 \\u0068\\u0061\\u0072\\u0064\\u0077\\u0061\\u0072\\u0065 \\u0061\\u006e\\u0064 \\u006f\\u0070\\u0065\\u0072\\u0061\\u0074\\u0069\\u006e\\u0067 \\u0073\\u0079\\u0073\\u0074\\u0065\\u006d\\u002e", + "These settings are tailored to your computer’s hardware and operating system", + "Use recommended performance settings", + "\\u0041\\u0064\\u0064\\u0069\\u0074\\u0069\\u006f\\u006e\\u0061\\u006c \\u0063\\u006f\\u006e\\u0074\\u0065\\u006e\\u0074 \\u0070\\u0072\\u006f\\u0063\\u0065\\u0073\\u0073\\u0065\\u0073 \\u0063\\u0061\\u006e \\u0069\\u006d\\u0070\\u0072\\u006f\\u0076\\u0065 \\u0070\\u0065\\u0072\\u0066\\u006f\\u0072\\u006d\\u0061\\u006e\\u0063\\u0065 \\u0077\\u0068\\u0065\\u006e \\u0075\\u0073\\u0069\\u006e\\u0067 \\u006d\\u0075\\u006c\\u0074\\u0069\\u0070\\u006c\\u0065 \\u0074\\u0061\\u0062\\u0073\\u002c \\u0062\\u0075\\u0074 \\u0077\\u0069\\u006c\\u006c \\u0061\\u006c\\u0073\\u006f \\u0075\\u0073\\u0065 \\u006d\\u006f\\u0072\\u0065 \\u006d\\u0065\\u006d\\u006f\\u0072\\u0079\\u002e", + "Additional content processes can improve performance when using multiple tabs, but will also use more memory.", + ]; + c.bench_function("unicode", move |b| { + b.iter(|| { + let mut result = String::new(); + for s in strings { + unescape_unicode(&mut result, s).unwrap(); + result.clear(); + } + }) + }); + c.bench_function("unicode_to_string", move |b| { + b.iter(|| { + for s in strings { + let _ = unescape_unicode_to_string(s); + } + }) + }); +} + +fn parser_ctx_bench(c: &mut Criterion) { + let tests = &["browser", "preferences"]; + let ftl_strings = get_ctxs(tests); + + c.bench_function_over_inputs( + "parse_ctx", + move |b, &&name| { + let sources = &ftl_strings[name]; + b.iter(|| { + for source in sources { + Parser::new(source.as_str()) + .parse() + .expect("Parsing of the FTL failed."); + } + }) + }, + tests, + ); +} + +criterion_group!( + benches, + parser_bench, + unicode_unescape_bench, + parser_ctx_bench +); +criterion_main!(benches); diff --git a/third_party/rust/fluent-syntax/src/ast/helper.rs b/third_party/rust/fluent-syntax/src/ast/helper.rs new file mode 100644 index 0000000000..923437d23b --- /dev/null +++ b/third_party/rust/fluent-syntax/src/ast/helper.rs @@ -0,0 +1,25 @@ +#[cfg(feature = "serde")] +use serde::{Deserialize, Serialize}; + +use super::Comment; +// This is a helper struct used to properly deserialize referential +// JSON comments which are single continous String, into a vec of +// content slices. +#[derive(Debug, PartialEq, Clone)] +#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] +#[cfg_attr(feature = "serde", serde(untagged))] +pub enum CommentDef<S> { + Single { content: S }, + Multi { content: Vec<S> }, +} + +impl<'s, S> From<CommentDef<S>> for Comment<S> { + fn from(input: CommentDef<S>) -> Self { + match input { + CommentDef::Single { content } => Self { + content: vec![content], + }, + CommentDef::Multi { content } => Self { content }, + } + } +} diff --git a/third_party/rust/fluent-syntax/src/ast/mod.rs b/third_party/rust/fluent-syntax/src/ast/mod.rs new file mode 100644 index 0000000000..48583441ca --- /dev/null +++ b/third_party/rust/fluent-syntax/src/ast/mod.rs @@ -0,0 +1,149 @@ +mod helper; + +#[cfg(feature = "serde")] +use serde::{Deserialize, Serialize}; + +#[derive(Debug, PartialEq, Clone)] +#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] +pub struct Resource<S> { + pub body: Vec<Entry<S>>, +} + +#[derive(Debug, PartialEq, Clone)] +#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] +#[cfg_attr(feature = "serde", serde(tag = "type"))] +pub enum Entry<S> { + Message(Message<S>), + Term(Term<S>), + Comment(Comment<S>), + GroupComment(Comment<S>), + ResourceComment(Comment<S>), + Junk { content: S }, +} + +#[derive(Debug, PartialEq, Clone)] +#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] +pub struct Message<S> { + pub id: Identifier<S>, + pub value: Option<Pattern<S>>, + pub attributes: Vec<Attribute<S>>, + pub comment: Option<Comment<S>>, +} + +#[derive(Debug, PartialEq, Clone)] +#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] +pub struct Term<S> { + pub id: Identifier<S>, + pub value: Pattern<S>, + pub attributes: Vec<Attribute<S>>, + pub comment: Option<Comment<S>>, +} + +#[derive(Debug, PartialEq, Clone)] +#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] +pub struct Pattern<S> { + pub elements: Vec<PatternElement<S>>, +} + +#[derive(Debug, PartialEq, Clone)] +#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] +#[cfg_attr(feature = "serde", serde(tag = "type"))] +pub enum PatternElement<S> { + TextElement { value: S }, + Placeable { expression: Expression<S> }, +} + +#[derive(Debug, PartialEq, Clone)] +#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] +pub struct Attribute<S> { + pub id: Identifier<S>, + pub value: Pattern<S>, +} + +#[derive(Debug, PartialEq, Clone)] +#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] +pub struct Identifier<S> { + pub name: S, +} + +#[derive(Debug, PartialEq, Clone)] +#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] +#[cfg_attr(feature = "serde", serde(tag = "type"))] +pub struct Variant<S> { + pub key: VariantKey<S>, + pub value: Pattern<S>, + pub default: bool, +} + +#[derive(Debug, PartialEq, Clone)] +#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] +#[cfg_attr(feature = "serde", serde(tag = "type"))] +pub enum VariantKey<S> { + Identifier { name: S }, + NumberLiteral { value: S }, +} + +#[derive(Debug, PartialEq, Clone)] +#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] +#[cfg_attr(feature = "serde", serde(from = "helper::CommentDef<S>"))] +pub struct Comment<S> { + pub content: Vec<S>, +} + +#[derive(Debug, PartialEq, Clone)] +#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] +#[cfg_attr(feature = "serde", serde(tag = "type"))] +pub struct CallArguments<S> { + pub positional: Vec<InlineExpression<S>>, + pub named: Vec<NamedArgument<S>>, +} + +#[derive(Debug, PartialEq, Clone)] +#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] +#[cfg_attr(feature = "serde", serde(tag = "type"))] +pub struct NamedArgument<S> { + pub name: Identifier<S>, + pub value: InlineExpression<S>, +} + +#[derive(Debug, PartialEq, Clone)] +#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] +#[cfg_attr(feature = "serde", serde(tag = "type"))] +pub enum InlineExpression<S> { + StringLiteral { + value: S, + }, + NumberLiteral { + value: S, + }, + FunctionReference { + id: Identifier<S>, + arguments: Option<CallArguments<S>>, + }, + MessageReference { + id: Identifier<S>, + attribute: Option<Identifier<S>>, + }, + TermReference { + id: Identifier<S>, + attribute: Option<Identifier<S>>, + arguments: Option<CallArguments<S>>, + }, + VariableReference { + id: Identifier<S>, + }, + Placeable { + expression: Box<Expression<S>>, + }, +} + +#[derive(Debug, PartialEq, Clone)] +#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] +#[cfg_attr(feature = "serde", serde(untagged))] +pub enum Expression<S> { + SelectExpression { + selector: InlineExpression<S>, + variants: Vec<Variant<S>>, + }, + InlineExpression(InlineExpression<S>), +} diff --git a/third_party/rust/fluent-syntax/src/bin/parser.rs b/third_party/rust/fluent-syntax/src/bin/parser.rs new file mode 100644 index 0000000000..4825b4a16d --- /dev/null +++ b/third_party/rust/fluent-syntax/src/bin/parser.rs @@ -0,0 +1,42 @@ +use fluent_syntax::parser::Parser; +use std::env; +use std::fs::File; +use std::io; +use std::io::Read; + +fn read_file(path: &str) -> Result<String, io::Error> { + let mut f = File::open(path)?; + let mut s = String::new(); + f.read_to_string(&mut s)?; + Ok(s) +} + +fn main() { + let args: Vec<String> = env::args().collect(); + let source = read_file(args.get(1).expect("Pass an argument")).expect("Failed to fetch file"); + + let (ast, errors) = match Parser::new(source.as_str()).parse() { + Ok(ast) => (ast, None), + Err((ast, err)) => (ast, Some(err)), + }; + + #[cfg(feature = "json")] + { + let target_json = serde_json::to_string_pretty(&ast).unwrap(); + println!("{}", target_json); + } + #[cfg(not(feature = "json"))] + { + use std::fmt::Write; + let mut result = String::new(); + write!(result, "{:#?}", ast).unwrap(); + println!("{}", result); + } + + if let Some(errors) = errors { + println!("\n======== Errors ========== \n"); + for err in errors { + println!("Err: {:#?}", err); + } + } +} diff --git a/third_party/rust/fluent-syntax/src/bin/update_fixtures.rs b/third_party/rust/fluent-syntax/src/bin/update_fixtures.rs new file mode 100644 index 0000000000..5ec34224b8 --- /dev/null +++ b/third_party/rust/fluent-syntax/src/bin/update_fixtures.rs @@ -0,0 +1,44 @@ +use std::fs; +use std::io; + +use fluent_syntax::parser::Parser; + +fn read_file(path: &str) -> Result<String, io::Error> { + fs::read_to_string(path) +} + +fn write_file(path: &str, source: &str) -> Result<(), io::Error> { + fs::write(path, source) +} + +fn main() { + let samples = &["menubar", "preferences", "simple"]; + let contexts = &["browser", "preferences"]; + + for sample in samples { + let path = format!("./benches/{}.ftl", sample); + let source = read_file(&path).unwrap(); + let ast = Parser::new(source).parse().unwrap(); + let target_json = serde_json::to_string_pretty(&ast).unwrap(); + let new_path = format!("./tests/fixtures/benches/{}.json", sample); + write_file(&new_path, &target_json).unwrap(); + } + + for test in contexts { + let paths = fs::read_dir(format!("./benches/contexts/{}", test)).unwrap(); + for path in paths.into_iter() { + let p = path.unwrap().path(); + let file_name = p.file_name().unwrap().to_str().unwrap(); + let path = p.to_str().unwrap(); + let source = read_file(path).unwrap(); + let ast = Parser::new(source).parse().unwrap(); + let target_json = serde_json::to_string_pretty(&ast).unwrap(); + let new_path = format!( + "./tests/fixtures/benches/contexts/{}/{}", + test, + file_name.replace(".ftl", ".json") + ); + write_file(&new_path, &target_json).unwrap(); + } + } +} diff --git a/third_party/rust/fluent-syntax/src/lib.rs b/third_party/rust/fluent-syntax/src/lib.rs new file mode 100644 index 0000000000..658fa44a4d --- /dev/null +++ b/third_party/rust/fluent-syntax/src/lib.rs @@ -0,0 +1,3 @@ +pub mod ast; +pub mod parser; +pub mod unicode; diff --git a/third_party/rust/fluent-syntax/src/parser/comment.rs b/third_party/rust/fluent-syntax/src/parser/comment.rs new file mode 100644 index 0000000000..3ab97ffb92 --- /dev/null +++ b/third_party/rust/fluent-syntax/src/parser/comment.rs @@ -0,0 +1,80 @@ +use super::{Parser, Result, Slice}; +use crate::ast; + +#[derive(Debug, PartialEq, Clone, Copy)] +pub(super) enum Level { + None = 0, + Regular = 1, + Group = 2, + Resource = 3, +} + +impl<'s, S> Parser<S> +where + S: Slice<'s>, +{ + pub(super) fn get_comment(&mut self) -> Result<(ast::Comment<S>, Level)> { + let mut level = Level::None; + let mut content = vec![]; + + while self.ptr < self.length { + let line_level = self.get_comment_level(); + if line_level == Level::None { + self.ptr -= 1; + break; + } else if level != Level::None && line_level != level { + self.ptr -= line_level as usize; + break; + } + + level = line_level; + + if self.ptr == self.length { + break; + } else if self.is_current_byte(b'\n') { + content.push(self.get_comment_line()?); + } else { + if let Err(e) = self.expect_byte(b' ') { + if content.is_empty() { + return Err(e); + } else { + self.ptr -= line_level as usize; + break; + } + } + content.push(self.get_comment_line()?); + } + self.skip_eol(); + } + + Ok((ast::Comment { content }, level)) + } + + fn get_comment_level(&mut self) -> Level { + let mut chars = 0; + + for _ in 0..3 { + if self.take_byte_if(b'#') { + chars += 1; + } + } + + match chars { + 0 => Level::None, + 1 => Level::Regular, + 2 => Level::Group, + 3 => Level::Resource, + _ => unreachable!(), + } + } + + fn get_comment_line(&mut self) -> Result<S> { + let start_pos = self.ptr; + + while self.ptr < self.length && !self.is_eol() { + self.ptr += 1; + } + + Ok(self.source.slice(start_pos..self.ptr)) + } +} diff --git a/third_party/rust/fluent-syntax/src/parser/errors.rs b/third_party/rust/fluent-syntax/src/parser/errors.rs new file mode 100644 index 0000000000..e1b26bdd73 --- /dev/null +++ b/third_party/rust/fluent-syntax/src/parser/errors.rs @@ -0,0 +1,128 @@ +use std::fmt::{self, Display, Formatter}; + +#[derive(Debug, PartialEq, Clone)] +pub struct ParserError { + pub pos: (usize, usize), + pub slice: Option<(usize, usize)>, + pub kind: ErrorKind, +} + +impl std::error::Error for ParserError {} + +impl Display for ParserError { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + Display::fmt(&self.kind, f) + } +} + +macro_rules! error { + ($kind:expr, $start:expr) => {{ + Err(ParserError { + pos: ($start, $start + 1), + slice: None, + kind: $kind, + }) + }}; + ($kind:expr, $start:expr, $end:expr) => {{ + Err(ParserError { + pos: ($start, $end), + slice: None, + kind: $kind, + }) + }}; +} + +#[derive(Debug, PartialEq, Clone)] +pub enum ErrorKind { + Generic, + ExpectedEntry, + ExpectedToken(char), + ExpectedCharRange { range: String }, + ExpectedMessageField { entry_id: String }, + ExpectedTermField { entry_id: String }, + ForbiddenWhitespace, + ForbiddenCallee, + ForbiddenKey, + MissingDefaultVariant, + MissingVariants, + MissingValue, + MissingVariantKey, + MissingLiteral, + MultipleDefaultVariants, + MessageReferenceAsSelector, + TermReferenceAsSelector, + MessageAttributeAsSelector, + TermAttributeAsPlaceable, + UnterminatedStringExpression, + PositionalArgumentFollowsNamed, + DuplicatedNamedArgument(String), + ForbiddenVariantAccessor, + UnknownEscapeSequence(String), + InvalidUnicodeEscapeSequence(String), + UnbalancedClosingBrace, + ExpectedInlineExpression, + ExpectedSimpleExpressionAsSelector, +} + +impl Display for ErrorKind { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + match self { + Self::Generic => write!(f, "An error occurred"), + Self::ExpectedEntry => write!(f, "Expected an entry"), + Self::ExpectedToken(letter) => { + write!(f, "Expected a token starting with \"{}\"", letter) + } + Self::ExpectedCharRange { range } => write!(f, "Expected one of \"{}\"", range), + Self::ExpectedMessageField { entry_id } => { + write!(f, "Expected a message field for \"{}\"", entry_id) + } + Self::ExpectedTermField { entry_id } => { + write!(f, "Expected a term field for \"{}\"", entry_id) + } + Self::ForbiddenWhitespace => write!(f, "Whitespace is not allowed here"), + Self::ForbiddenCallee => write!(f, "Callee is not allowed here"), + Self::ForbiddenKey => write!(f, "Key is not allowed here"), + Self::MissingDefaultVariant => { + write!(f, "The select expression must have a default variant") + } + Self::MissingVariants => { + write!(f, "The select expression must have one or more variants") + } + Self::MissingValue => write!(f, "Expected a value"), + Self::MissingVariantKey => write!(f, "Expected a variant key"), + Self::MissingLiteral => write!(f, "Expected a literal"), + Self::MultipleDefaultVariants => { + write!(f, "A select expression can only have one default variant",) + } + Self::MessageReferenceAsSelector => { + write!(f, "Message references can't be used as a selector") + } + Self::TermReferenceAsSelector => { + write!(f, "Term references can't be used as a selector") + } + Self::MessageAttributeAsSelector => { + write!(f, "Message attributes can't be used as a selector") + } + Self::TermAttributeAsPlaceable => { + write!(f, "Term attributes can't be used as a placeable") + } + Self::UnterminatedStringExpression => write!(f, "Unterminated string expression"), + Self::PositionalArgumentFollowsNamed => { + write!(f, "Positional arguments must come before named arguments",) + } + Self::DuplicatedNamedArgument(name) => { + write!(f, "The \"{}\" argument appears twice", name) + } + Self::ForbiddenVariantAccessor => write!(f, "Forbidden variant accessor"), + Self::UnknownEscapeSequence(seq) => write!(f, "Unknown escape sequence, \"{}\"", seq), + Self::InvalidUnicodeEscapeSequence(seq) => { + write!(f, "Invalid unicode escape sequence, \"{}\"", seq) + } + Self::UnbalancedClosingBrace => write!(f, "Unbalanced closing brace"), + Self::ExpectedInlineExpression => write!(f, "Expected an inline expression"), + Self::ExpectedSimpleExpressionAsSelector => { + write!(f, "Expected a simple expression as selector") + } + } + } +} diff --git a/third_party/rust/fluent-syntax/src/parser/expression.rs b/third_party/rust/fluent-syntax/src/parser/expression.rs new file mode 100644 index 0000000000..0a2d3c78c8 --- /dev/null +++ b/third_party/rust/fluent-syntax/src/parser/expression.rs @@ -0,0 +1,148 @@ +use super::errors::{ErrorKind, ParserError}; +use super::{Parser, Result, Slice}; +use crate::ast; + +impl<'s, S> Parser<S> +where + S: Slice<'s>, +{ + pub(super) fn get_expression(&mut self) -> Result<ast::Expression<S>> { + let exp = self.get_inline_expression()?; + + self.skip_blank(); + + if !self.is_current_byte(b'-') || !self.is_byte_at(b'>', self.ptr + 1) { + if let ast::InlineExpression::TermReference { ref attribute, .. } = exp { + if attribute.is_some() { + return error!(ErrorKind::TermAttributeAsPlaceable, self.ptr); + } + } + return Ok(ast::Expression::InlineExpression(exp)); + } + + match exp { + ast::InlineExpression::MessageReference { ref attribute, .. } => { + if attribute.is_none() { + return error!(ErrorKind::MessageReferenceAsSelector, self.ptr); + } else { + return error!(ErrorKind::MessageAttributeAsSelector, self.ptr); + } + } + ast::InlineExpression::TermReference { ref attribute, .. } => { + if attribute.is_none() { + return error!(ErrorKind::TermReferenceAsSelector, self.ptr); + } + } + ast::InlineExpression::StringLiteral { .. } + | ast::InlineExpression::NumberLiteral { .. } + | ast::InlineExpression::VariableReference { .. } + | ast::InlineExpression::FunctionReference { .. } => {} + _ => { + return error!(ErrorKind::ExpectedSimpleExpressionAsSelector, self.ptr); + } + }; + + self.ptr += 2; // -> + + self.skip_blank_inline(); + if !self.skip_eol() { + return error!( + ErrorKind::ExpectedCharRange { + range: "\n | \r\n".to_string() + }, + self.ptr + ); + } + self.skip_blank(); + + let variants = self.get_variants()?; + + Ok(ast::Expression::SelectExpression { + selector: exp, + variants, + }) + } + + pub(super) fn get_inline_expression(&mut self) -> Result<ast::InlineExpression<S>> { + match self.source.as_ref().as_bytes().get(self.ptr) { + Some(b'"') => { + self.ptr += 1; // " + let start = self.ptr; + while let Some(b) = self.source.as_ref().as_bytes().get(self.ptr) { + match b { + b'\\' => match self.source.as_ref().as_bytes().get(self.ptr + 1) { + Some(b'\\') | Some(b'{') | Some(b'"') => self.ptr += 2, + Some(b'u') => { + self.ptr += 2; + self.skip_unicode_escape_sequence(4)?; + } + Some(b'U') => { + self.ptr += 2; + self.skip_unicode_escape_sequence(6)?; + } + _ => return error!(ErrorKind::Generic, self.ptr), + }, + b'"' => { + break; + } + b'\n' => { + return error!(ErrorKind::Generic, self.ptr); + } + _ => self.ptr += 1, + } + } + + self.expect_byte(b'"')?; + let slice = self.source.slice(start..self.ptr - 1); + Ok(ast::InlineExpression::StringLiteral { value: slice }) + } + Some(b) if b.is_ascii_digit() => { + let num = self.get_number_literal()?; + Ok(ast::InlineExpression::NumberLiteral { value: num }) + } + Some(b'-') => { + self.ptr += 1; // - + if self.is_identifier_start() { + let id = self.get_identifier()?; + let attribute = self.get_attribute_accessor()?; + let arguments = self.get_call_arguments()?; + Ok(ast::InlineExpression::TermReference { + id, + attribute, + arguments, + }) + } else { + self.ptr -= 1; + let num = self.get_number_literal()?; + Ok(ast::InlineExpression::NumberLiteral { value: num }) + } + } + Some(b'$') => { + self.ptr += 1; // - + let id = self.get_identifier()?; + Ok(ast::InlineExpression::VariableReference { id }) + } + Some(b) if b.is_ascii_alphabetic() => { + let id = self.get_identifier()?; + let arguments = self.get_call_arguments()?; + if arguments.is_some() { + if !Self::is_callee(id.name.as_ref().as_bytes()) { + return error!(ErrorKind::ForbiddenCallee, self.ptr); + } + + Ok(ast::InlineExpression::FunctionReference { id, arguments }) + } else { + let attribute = self.get_attribute_accessor()?; + Ok(ast::InlineExpression::MessageReference { id, attribute }) + } + } + Some(b'{') => { + let exp = self.get_placeable()?; + Ok(ast::InlineExpression::Placeable { + expression: Box::new(exp), + }) + } + _ => error!(ErrorKind::ExpectedInlineExpression, self.ptr), + } + } +} diff --git a/third_party/rust/fluent-syntax/src/parser/helper.rs b/third_party/rust/fluent-syntax/src/parser/helper.rs new file mode 100644 index 0000000000..363bba2864 --- /dev/null +++ b/third_party/rust/fluent-syntax/src/parser/helper.rs @@ -0,0 +1,171 @@ +use super::errors::{ErrorKind, ParserError}; +use super::{Parser, Result, Slice}; + +impl<'s, S> Parser<S> +where + S: Slice<'s>, +{ + pub(super) fn is_current_byte(&self, b: u8) -> bool { + self.source.as_ref().as_bytes().get(self.ptr) == Some(&b) + } + + pub(super) fn is_byte_at(&self, b: u8, pos: usize) -> bool { + self.source.as_ref().as_bytes().get(pos) == Some(&b) + } + + pub(super) fn skip_to_next_entry_start(&mut self) { + while let Some(b) = self.source.as_ref().as_bytes().get(self.ptr) { + let new_line = + self.ptr == 0 || self.source.as_ref().as_bytes().get(self.ptr - 1) == Some(&b'\n'); + + if new_line && (b.is_ascii_alphabetic() || [b'-', b'#'].contains(b)) { + break; + } + + self.ptr += 1; + } + } + + pub(super) fn skip_eol(&mut self) -> bool { + match self.source.as_ref().as_bytes().get(self.ptr) { + Some(b'\n') => { + self.ptr += 1; + true + } + Some(b'\r') if self.is_byte_at(b'\n', self.ptr + 1) => { + self.ptr += 2; + true + } + _ => false, + } + } + + pub(super) fn skip_unicode_escape_sequence(&mut self, length: usize) -> Result<()> { + let start = self.ptr; + for _ in 0..length { + match self.source.as_ref().as_bytes().get(self.ptr) { + Some(b) if b.is_ascii_hexdigit() => self.ptr += 1, + _ => break, + } + } + if self.ptr - start != length { + let end = if self.ptr >= self.length { + self.ptr + } else { + self.ptr + 1 + }; + let seq = self.source.slice(start..end).as_ref().to_owned(); + return error!(ErrorKind::InvalidUnicodeEscapeSequence(seq), self.ptr); + } + Ok(()) + } + + pub(super) fn is_identifier_start(&self) -> bool { + matches!(self.source.as_ref().as_bytes().get(self.ptr), Some(b) if b.is_ascii_alphabetic()) + } + + pub(super) fn take_byte_if(&mut self, b: u8) -> bool { + if self.is_current_byte(b) { + self.ptr += 1; + true + } else { + false + } + } + + pub(super) fn skip_blank_block(&mut self) -> usize { + let mut count = 0; + loop { + let start = self.ptr; + self.skip_blank_inline(); + if !self.skip_eol() { + self.ptr = start; + break; + } + count += 1; + } + count + } + + pub(super) fn skip_blank(&mut self) { + loop { + match self.source.as_ref().as_bytes().get(self.ptr) { + Some(b' ') | Some(b'\n') => self.ptr += 1, + Some(b'\r') + if self.source.as_ref().as_bytes().get(self.ptr + 1) == Some(&b'\n') => + { + self.ptr += 2 + } + _ => break, + } + } + } + + pub(super) fn skip_blank_inline(&mut self) -> usize { + let start = self.ptr; + while let Some(b' ') = self.source.as_ref().as_bytes().get(self.ptr) { + self.ptr += 1; + } + self.ptr - start + } + + pub(super) fn is_byte_pattern_continuation(b: u8) -> bool { + ![b'}', b'.', b'[', b'*'].contains(&b) + } + + pub(super) fn is_callee(name: &[u8]) -> bool { + name.iter() + .all(|c| c.is_ascii_uppercase() || c.is_ascii_digit() || *c == b'_' || *c == b'-') + } + + pub(super) fn expect_byte(&mut self, b: u8) -> Result<()> { + if !self.is_current_byte(b) { + return error!(ErrorKind::ExpectedToken(b as char), self.ptr); + } + self.ptr += 1; + Ok(()) + } + + pub(super) fn is_number_start(&self) -> bool { + matches!(self.source.as_ref().as_bytes().get(self.ptr), Some(b) if (b == &b'-') || b.is_ascii_digit()) + } + + pub(super) fn is_eol(&self) -> bool { + match self.source.as_ref().as_bytes().get(self.ptr) { + Some(b'\n') => true, + Some(b'\r') if self.is_byte_at(b'\n', self.ptr + 1) => true, + _ => false, + } + } + + pub(super) fn skip_digits(&mut self) -> Result<()> { + let start = self.ptr; + loop { + match self.source.as_ref().as_bytes().get(self.ptr) { + Some(b) if b.is_ascii_digit() => self.ptr += 1, + _ => break, + } + } + if start == self.ptr { + error!( + ErrorKind::ExpectedCharRange { + range: "0-9".to_string() + }, + self.ptr + ) + } else { + Ok(()) + } + } + + pub(super) fn get_number_literal(&mut self) -> Result<S> { + let start = self.ptr; + self.take_byte_if(b'-'); + self.skip_digits()?; + if self.take_byte_if(b'.') { + self.skip_digits()?; + } + + Ok(self.source.slice(start..self.ptr)) + } +} diff --git a/third_party/rust/fluent-syntax/src/parser/mod.rs b/third_party/rust/fluent-syntax/src/parser/mod.rs new file mode 100644 index 0000000000..9fc08847dc --- /dev/null +++ b/third_party/rust/fluent-syntax/src/parser/mod.rs @@ -0,0 +1,387 @@ +#[macro_use] +mod errors; +mod comment; +mod expression; +mod helper; +mod pattern; +mod slice; + +use crate::ast; +use slice::Slice; +use std::result; + +pub use errors::{ErrorKind, ParserError}; + +pub type Result<T> = result::Result<T, ParserError>; + +pub struct Parser<S> { + source: S, + ptr: usize, + length: usize, +} + +impl<'s, S> Parser<S> +where + S: Slice<'s>, +{ + pub fn new(source: S) -> Self { + let length = source.as_ref().len(); + Self { + source, + ptr: 0, + length, + } + } + + pub fn parse( + &mut self, + ) -> std::result::Result<ast::Resource<S>, (ast::Resource<S>, Vec<ParserError>)> { + let mut errors = vec![]; + + let mut body = vec![]; + + self.skip_blank_block(); + let mut last_comment = None; + let mut last_blank_count = 0; + + while self.ptr < self.length { + let entry_start = self.ptr; + let mut entry = self.get_entry(entry_start); + + if let Some(comment) = last_comment.take() { + match entry { + Ok(ast::Entry::Message(ref mut msg)) if last_blank_count < 2 => { + msg.comment = Some(comment); + } + Ok(ast::Entry::Term(ref mut term)) if last_blank_count < 2 => { + term.comment = Some(comment); + } + _ => { + body.push(ast::Entry::Comment(comment)); + } + } + } + + match entry { + Ok(ast::Entry::Comment(comment)) => { + last_comment = Some(comment); + } + Ok(entry) => { + body.push(entry); + } + Err(mut err) => { + self.skip_to_next_entry_start(); + err.slice = Some((entry_start, self.ptr)); + errors.push(err); + let content = self.source.slice(entry_start..self.ptr); + body.push(ast::Entry::Junk { content }); + } + } + last_blank_count = self.skip_blank_block(); + } + + if let Some(last_comment) = last_comment.take() { + body.push(ast::Entry::Comment(last_comment)); + } + if errors.is_empty() { + Ok(ast::Resource { body }) + } else { + Err((ast::Resource { body }, errors)) + } + } + + fn get_entry(&mut self, entry_start: usize) -> Result<ast::Entry<S>> { + let entry = match self.source.as_ref().as_bytes().get(self.ptr) { + Some(b'#') => { + let (comment, level) = self.get_comment()?; + match level { + comment::Level::Regular => ast::Entry::Comment(comment), + comment::Level::Group => ast::Entry::GroupComment(comment), + comment::Level::Resource => ast::Entry::ResourceComment(comment), + comment::Level::None => unreachable!(), + } + } + Some(b'-') => ast::Entry::Term(self.get_term(entry_start)?), + _ => ast::Entry::Message(self.get_message(entry_start)?), + }; + Ok(entry) + } + + fn get_message(&mut self, entry_start: usize) -> Result<ast::Message<S>> { + let id = self.get_identifier()?; + self.skip_blank_inline(); + self.expect_byte(b'=')?; + let pattern = self.get_pattern()?; + + self.skip_blank_block(); + + let attributes = self.get_attributes(); + + if pattern.is_none() && attributes.is_empty() { + let entry_id = id.name.as_ref().to_owned(); + return error!( + ErrorKind::ExpectedMessageField { entry_id }, + entry_start, self.ptr + ); + } + + Ok(ast::Message { + id, + value: pattern, + attributes, + comment: None, + }) + } + + fn get_term(&mut self, entry_start: usize) -> Result<ast::Term<S>> { + self.expect_byte(b'-')?; + let id = self.get_identifier()?; + self.skip_blank_inline(); + self.expect_byte(b'=')?; + self.skip_blank_inline(); + + let value = self.get_pattern()?; + + self.skip_blank_block(); + + let attributes = self.get_attributes(); + + if let Some(value) = value { + Ok(ast::Term { + id, + value, + attributes, + comment: None, + }) + } else { + error!( + ErrorKind::ExpectedTermField { + entry_id: id.name.as_ref().to_owned() + }, + entry_start, self.ptr + ) + } + } + + fn get_attributes(&mut self) -> Vec<ast::Attribute<S>> { + let mut attributes = vec![]; + + loop { + let line_start = self.ptr; + self.skip_blank_inline(); + if !self.is_current_byte(b'.') { + self.ptr = line_start; + break; + } + + if let Ok(attr) = self.get_attribute() { + attributes.push(attr); + } else { + self.ptr = line_start; + break; + } + } + attributes + } + + fn get_attribute(&mut self) -> Result<ast::Attribute<S>> { + self.expect_byte(b'.')?; + let id = self.get_identifier()?; + self.skip_blank_inline(); + self.expect_byte(b'=')?; + let pattern = self.get_pattern()?; + + match pattern { + Some(pattern) => Ok(ast::Attribute { id, value: pattern }), + None => error!(ErrorKind::MissingValue, self.ptr), + } + } + + fn get_identifier(&mut self) -> Result<ast::Identifier<S>> { + let mut ptr = self.ptr; + + match self.source.as_ref().as_bytes().get(ptr) { + Some(b) if b.is_ascii_alphabetic() => { + ptr += 1; + } + _ => { + return error!( + ErrorKind::ExpectedCharRange { + range: "a-zA-Z".to_string() + }, + ptr + ); + } + } + + while let Some(b) = self.source.as_ref().as_bytes().get(ptr) { + if b.is_ascii_alphabetic() || b.is_ascii_digit() || [b'_', b'-'].contains(b) { + ptr += 1; + } else { + break; + } + } + + let name = self.source.slice(self.ptr..ptr); + self.ptr = ptr; + + Ok(ast::Identifier { name }) + } + + fn get_attribute_accessor(&mut self) -> Result<Option<ast::Identifier<S>>> { + if self.take_byte_if(b'.') { + let ident = self.get_identifier()?; + Ok(Some(ident)) + } else { + Ok(None) + } + } + + fn get_variant_key(&mut self) -> Result<ast::VariantKey<S>> { + if !self.take_byte_if(b'[') { + return error!(ErrorKind::ExpectedToken('['), self.ptr); + } + self.skip_blank(); + + let key = if self.is_number_start() { + ast::VariantKey::NumberLiteral { + value: self.get_number_literal()?, + } + } else { + ast::VariantKey::Identifier { + name: self.get_identifier()?.name, + } + }; + + self.skip_blank(); + + self.expect_byte(b']')?; + + Ok(key) + } + + fn get_variants(&mut self) -> Result<Vec<ast::Variant<S>>> { + let mut variants = vec![]; + let mut has_default = false; + + while self.is_current_byte(b'*') || self.is_current_byte(b'[') { + let default = self.take_byte_if(b'*'); + + if default { + if has_default { + return error!(ErrorKind::MultipleDefaultVariants, self.ptr); + } else { + has_default = true; + } + } + + let key = self.get_variant_key()?; + + let value = self.get_pattern()?; + + if let Some(value) = value { + variants.push(ast::Variant { + key, + value, + default, + }); + self.skip_blank(); + } else { + return error!(ErrorKind::MissingValue, self.ptr); + } + } + + if has_default { + Ok(variants) + } else { + error!(ErrorKind::MissingDefaultVariant, self.ptr) + } + } + + fn get_placeable(&mut self) -> Result<ast::Expression<S>> { + self.expect_byte(b'{')?; + self.skip_blank(); + let exp = self.get_expression()?; + self.skip_blank_inline(); + self.expect_byte(b'}')?; + + let invalid_expression_found = match &exp { + ast::Expression::InlineExpression(ast::InlineExpression::TermReference { + ref attribute, + .. + }) => attribute.is_some(), + _ => false, + }; + if invalid_expression_found { + return error!(ErrorKind::TermAttributeAsPlaceable, self.ptr); + } + + Ok(exp) + } + + fn get_call_arguments(&mut self) -> Result<Option<ast::CallArguments<S>>> { + self.skip_blank(); + if !self.take_byte_if(b'(') { + return Ok(None); + } + + let mut positional = vec![]; + let mut named = vec![]; + let mut argument_names = vec![]; + + self.skip_blank(); + + while self.ptr < self.length { + if self.is_current_byte(b')') { + break; + } + + let expr = self.get_inline_expression()?; + + if let ast::InlineExpression::MessageReference { + ref id, + attribute: None, + } = expr + { + self.skip_blank(); + if self.is_current_byte(b':') { + if argument_names.contains(&id.name) { + return error!( + ErrorKind::DuplicatedNamedArgument(id.name.as_ref().to_owned()), + self.ptr + ); + } + self.ptr += 1; + self.skip_blank(); + let val = self.get_inline_expression()?; + + argument_names.push(id.name.clone()); + named.push(ast::NamedArgument { + name: ast::Identifier { + name: id.name.clone(), + }, + value: val, + }); + } else { + if !argument_names.is_empty() { + return error!(ErrorKind::PositionalArgumentFollowsNamed, self.ptr); + } + positional.push(expr); + } + } else { + if !argument_names.is_empty() { + return error!(ErrorKind::PositionalArgumentFollowsNamed, self.ptr); + } + positional.push(expr); + } + + self.skip_blank(); + self.take_byte_if(b','); + self.skip_blank(); + } + + self.expect_byte(b')')?; + + Ok(Some(ast::CallArguments { positional, named })) + } +} diff --git a/third_party/rust/fluent-syntax/src/parser/pattern.rs b/third_party/rust/fluent-syntax/src/parser/pattern.rs new file mode 100644 index 0000000000..84804b0db2 --- /dev/null +++ b/third_party/rust/fluent-syntax/src/parser/pattern.rs @@ -0,0 +1,209 @@ +use super::errors::{ErrorKind, ParserError}; +use super::{Parser, Result, Slice}; +use crate::ast; + +#[derive(Debug, PartialEq)] +enum TextElementTermination { + LineFeed, + CRLF, + PlaceableStart, + EOF, +} + +// This enum tracks the placement of the text element in the pattern, which is needed for +// dedentation logic. +#[derive(Debug, PartialEq)] +enum TextElementPosition { + InitialLineStart, + LineStart, + Continuation, +} + +// This enum allows us to mark pointers in the source which will later become text elements +// but without slicing them out of the source string. This makes the indentation adjustments +// cheaper since they'll happen on the pointers, rather than extracted slices. +#[derive(Debug)] +enum PatternElementPlaceholders<S> { + Placeable(ast::Expression<S>), + // (start, end, indent, position) + TextElement(usize, usize, usize, TextElementPosition), +} + +// This enum tracks whether the text element is blank or not. +// This is important to identify text elements which should not be taken into account +// when calculating common indent. +#[derive(Debug, PartialEq)] +enum TextElementType { + Blank, + NonBlank, +} + +impl<'s, S> Parser<S> +where + S: Slice<'s>, +{ + pub(super) fn get_pattern(&mut self) -> Result<Option<ast::Pattern<S>>> { + let mut elements = vec![]; + let mut last_non_blank = None; + let mut common_indent = None; + + self.skip_blank_inline(); + + let mut text_element_role = if self.skip_eol() { + self.skip_blank_block(); + TextElementPosition::LineStart + } else { + TextElementPosition::InitialLineStart + }; + + while self.ptr < self.length { + if self.is_current_byte(b'{') { + if text_element_role == TextElementPosition::LineStart { + common_indent = Some(0); + } + let exp = self.get_placeable()?; + last_non_blank = Some(elements.len()); + elements.push(PatternElementPlaceholders::Placeable(exp)); + text_element_role = TextElementPosition::Continuation; + } else { + let slice_start = self.ptr; + let mut indent = 0; + if text_element_role == TextElementPosition::LineStart { + indent = self.skip_blank_inline(); + if self.ptr >= self.length { + break; + } + let b = self.source.as_ref().as_bytes().get(self.ptr); + if indent == 0 { + if b != Some(&b'\n') { + break; + } + } else if !Self::is_byte_pattern_continuation(*b.unwrap()) { + self.ptr = slice_start; + break; + } + } + let (start, end, text_element_type, termination_reason) = self.get_text_slice()?; + if start != end { + if text_element_role == TextElementPosition::LineStart + && text_element_type == TextElementType::NonBlank + { + if let Some(common) = common_indent { + if indent < common { + common_indent = Some(indent); + } + } else { + common_indent = Some(indent); + } + } + if text_element_role != TextElementPosition::LineStart + || text_element_type == TextElementType::NonBlank + || termination_reason == TextElementTermination::LineFeed + { + if text_element_type == TextElementType::NonBlank { + last_non_blank = Some(elements.len()); + } + elements.push(PatternElementPlaceholders::TextElement( + slice_start, + end, + indent, + text_element_role, + )); + } + } + + text_element_role = match termination_reason { + TextElementTermination::LineFeed => TextElementPosition::LineStart, + TextElementTermination::CRLF => TextElementPosition::Continuation, + TextElementTermination::PlaceableStart => TextElementPosition::Continuation, + TextElementTermination::EOF => TextElementPosition::Continuation, + }; + } + } + + if let Some(last_non_blank) = last_non_blank { + let elements = elements + .into_iter() + .take(last_non_blank + 1) + .enumerate() + .map(|(i, elem)| match elem { + PatternElementPlaceholders::Placeable(expression) => { + ast::PatternElement::Placeable { expression } + } + PatternElementPlaceholders::TextElement(start, end, indent, role) => { + let start = if role == TextElementPosition::LineStart { + common_indent.map_or_else( + || start + indent, + |common_indent| start + std::cmp::min(indent, common_indent), + ) + } else { + start + }; + let mut value = self.source.slice(start..end); + if last_non_blank == i { + value.trim(); + ast::PatternElement::TextElement { value } + } else { + ast::PatternElement::TextElement { value } + } + } + }) + .collect(); + return Ok(Some(ast::Pattern { elements })); + } + + Ok(None) + } + + fn get_text_slice( + &mut self, + ) -> Result<(usize, usize, TextElementType, TextElementTermination)> { + let start_pos = self.ptr; + let mut text_element_type = TextElementType::Blank; + + while let Some(b) = self.source.as_ref().as_bytes().get(self.ptr) { + match b { + b' ' => self.ptr += 1, + b'\n' => { + self.ptr += 1; + return Ok(( + start_pos, + self.ptr, + text_element_type, + TextElementTermination::LineFeed, + )); + } + b'\r' if self.is_byte_at(b'\n', self.ptr + 1) => { + self.ptr += 1; + return Ok(( + start_pos, + self.ptr - 1, + text_element_type, + TextElementTermination::CRLF, + )); + } + b'{' => { + return Ok(( + start_pos, + self.ptr, + text_element_type, + TextElementTermination::PlaceableStart, + )); + } + b'}' => { + return error!(ErrorKind::UnbalancedClosingBrace, self.ptr); + } + _ => { + text_element_type = TextElementType::NonBlank; + self.ptr += 1 + } + } + } + Ok(( + start_pos, + self.ptr, + text_element_type, + TextElementTermination::EOF, + )) + } +} diff --git a/third_party/rust/fluent-syntax/src/parser/slice.rs b/third_party/rust/fluent-syntax/src/parser/slice.rs new file mode 100644 index 0000000000..d44f8251fe --- /dev/null +++ b/third_party/rust/fluent-syntax/src/parser/slice.rs @@ -0,0 +1,25 @@ +use std::ops::Range; +pub trait Slice<'s>: AsRef<str> + Clone + PartialEq { + fn slice(&self, range: Range<usize>) -> Self; + fn trim(&mut self); +} + +impl<'s> Slice<'s> for String { + fn slice(&self, range: Range<usize>) -> Self { + self[range].to_string() + } + + fn trim(&mut self) { + *self = self.trim_end().to_string(); + } +} + +impl<'s> Slice<'s> for &'s str { + fn slice(&self, range: Range<usize>) -> Self { + &self[range] + } + + fn trim(&mut self) { + *self = self.trim_end(); + } +} diff --git a/third_party/rust/fluent-syntax/src/unicode.rs b/third_party/rust/fluent-syntax/src/unicode.rs new file mode 100644 index 0000000000..49301734bd --- /dev/null +++ b/third_party/rust/fluent-syntax/src/unicode.rs @@ -0,0 +1,91 @@ +use std::borrow::Cow; +use std::char; +use std::fmt; + +const UNKNOWN_CHAR: char = '�'; + +fn encode_unicode(s: Option<&str>) -> char { + s.and_then(|s| u32::from_str_radix(s, 16).ok().and_then(char::from_u32)) + .unwrap_or(UNKNOWN_CHAR) +} + +pub fn unescape_unicode<W>(w: &mut W, input: &str) -> fmt::Result +where + W: fmt::Write, +{ + let bytes = input.as_bytes(); + + let mut start = 0; + let mut ptr = 0; + + while let Some(b) = bytes.get(ptr) { + if b != &b'\\' { + ptr += 1; + continue; + } + if start != ptr { + w.write_str(&input[start..ptr])?; + } + + ptr += 1; + + let new_char = match bytes.get(ptr) { + Some(b'\\') => '\\', + Some(b'"') => '"', + Some(u @ b'u') | Some(u @ b'U') => { + let seq_start = ptr + 1; + let len = if u == &b'u' { 4 } else { 6 }; + ptr += len; + encode_unicode(input.get(seq_start..seq_start + len)) + } + _ => UNKNOWN_CHAR, + }; + ptr += 1; + w.write_char(new_char)?; + start = ptr; + } + if start != ptr { + w.write_str(&input[start..ptr])?; + } + Ok(()) +} + +pub fn unescape_unicode_to_string(input: &str) -> Cow<str> { + let bytes = input.as_bytes(); + let mut result = Cow::from(input); + + let mut ptr = 0; + + while let Some(b) = bytes.get(ptr) { + if b != &b'\\' { + if let Cow::Owned(ref mut s) = result { + s.push(*b as char); + } + ptr += 1; + continue; + } + + if let Cow::Borrowed(_) = result { + result = Cow::from(&input[0..ptr]); + } + + ptr += 1; + + let new_char = match bytes.get(ptr) { + Some(b'\\') => '\\', + Some(b'"') => '"', + Some(u @ b'u') | Some(u @ b'U') => { + let start = ptr + 1; + let len = if u == &b'u' { 4 } else { 6 }; + ptr += len; + input + .get(start..(start + len)) + .map_or(UNKNOWN_CHAR, |slice| encode_unicode(Some(slice))) + } + _ => UNKNOWN_CHAR, + }; + result.to_mut().push(new_char); + ptr += 1; + } + result +} |