/* * Copyright (C) 2012-2014 by various contributors (see AUTHORS) * Copyright (C) 2017 Apple Inc. All rights reserved. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. */ "use strict"; // util/identifier.js const utilIdentifierJS = { }; { /* eslint max-len: 0 */ // This is a trick taken from Esprima. It turns out that, on // non-Chrome browsers, to check whether a string is in a set, a // predicate containing a big ugly `switch` statement is faster than // a regular expression, and on Chrome the two are about on par. // This function uses `eval` (non-lexical) to produce such a // predicate from a space-separated string of words. // // It starts by sorting the words by length. function makePredicate(words) { words = words.split(" "); return function (str) { return words.indexOf(str) >= 0; }; } // Reserved word lists for various dialects of the language const reservedWords = { 6: makePredicate("enum await"), strict: makePredicate("implements interface let package private protected public static yield"), strictBind: makePredicate("eval arguments") }; utilIdentifierJS.reservedWords = reservedWords; // And the keywords const isKeyword = makePredicate("break case catch continue debugger default do else finally for function if return switch throw try var while with null true false instanceof typeof void delete new in this let const class extends export import yield super"); utilIdentifierJS.isKeyword = isKeyword; // ## Character categories // Big ugly regular expressions that match characters in the // whitespace, identifier, and identifier-start categories. These // are only applied when a character is found to actually have a // code point above 128. // Generated by `bin/generate-identifier-regex.js`. let nonASCIIidentifierStartChars = "\xaa\xb5\xba\xc0-\xd6\xd8-\xf6\xf8-\u02c1\u02c6-\u02d1\u02e0-\u02e4\u02ec\u02ee\u0370-\u0374\u0376\u0377\u037a-\u037d\u037f\u0386\u0388-\u038a\u038c\u038e-\u03a1\u03a3-\u03f5\u03f7-\u0481\u048a-\u052f\u0531-\u0556\u0559\u0561-\u0587\u05d0-\u05ea\u05f0-\u05f2\u0620-\u064a\u066e\u066f\u0671-\u06d3\u06d5\u06e5\u06e6\u06ee\u06ef\u06fa-\u06fc\u06ff\u0710\u0712-\u072f\u074d-\u07a5\u07b1\u07ca-\u07ea\u07f4\u07f5\u07fa\u0800-\u0815\u081a\u0824\u0828\u0840-\u0858\u08a0-\u08b4\u08b6-\u08bd\u0904-\u0939\u093d\u0950\u0958-\u0961\u0971-\u0980\u0985-\u098c\u098f\u0990\u0993-\u09a8\u09aa-\u09b0\u09b2\u09b6-\u09b9\u09bd\u09ce\u09dc\u09dd\u09df-\u09e1\u09f0\u09f1\u0a05-\u0a0a\u0a0f\u0a10\u0a13-\u0a28\u0a2a-\u0a30\u0a32\u0a33\u0a35\u0a36\u0a38\u0a39\u0a59-\u0a5c\u0a5e\u0a72-\u0a74\u0a85-\u0a8d\u0a8f-\u0a91\u0a93-\u0aa8\u0aaa-\u0ab0\u0ab2\u0ab3\u0ab5-\u0ab9\u0abd\u0ad0\u0ae0\u0ae1\u0af9\u0b05-\u0b0c\u0b0f\u0b10\u0b13-\u0b28\u0b2a-\u0b30\u0b32\u0b33\u0b35-\u0b39\u0b3d\u0b5c\u0b5d\u0b5f-\u0b61\u0b71\u0b83\u0b85-\u0b8a\u0b8e-\u0b90\u0b92-\u0b95\u0b99\u0b9a\u0b9c\u0b9e\u0b9f\u0ba3\u0ba4\u0ba8-\u0baa\u0bae-\u0bb9\u0bd0\u0c05-\u0c0c\u0c0e-\u0c10\u0c12-\u0c28\u0c2a-\u0c39\u0c3d\u0c58-\u0c5a\u0c60\u0c61\u0c80\u0c85-\u0c8c\u0c8e-\u0c90\u0c92-\u0ca8\u0caa-\u0cb3\u0cb5-\u0cb9\u0cbd\u0cde\u0ce0\u0ce1\u0cf1\u0cf2\u0d05-\u0d0c\u0d0e-\u0d10\u0d12-\u0d3a\u0d3d\u0d4e\u0d54-\u0d56\u0d5f-\u0d61\u0d7a-\u0d7f\u0d85-\u0d96\u0d9a-\u0db1\u0db3-\u0dbb\u0dbd\u0dc0-\u0dc6\u0e01-\u0e30\u0e32\u0e33\u0e40-\u0e46\u0e81\u0e82\u0e84\u0e87\u0e88\u0e8a\u0e8d\u0e94-\u0e97\u0e99-\u0e9f\u0ea1-\u0ea3\u0ea5\u0ea7\u0eaa\u0eab\u0ead-\u0eb0\u0eb2\u0eb3\u0ebd\u0ec0-\u0ec4\u0ec6\u0edc-\u0edf\u0f00\u0f40-\u0f47\u0f49-\u0f6c\u0f88-\u0f8c\u1000-\u102a\u103f\u1050-\u1055\u105a-\u105d\u1061\u1065\u1066\u106e-\u1070\u1075-\u1081\u108e\u10a0-\u10c5\u10c7\u10cd\u10d0-\u10fa\u10fc-\u1248\u124a-\u124d\u1250-\u1256\u1258\u125a-\u125d\u1260-\u1288\u128a-\u128d\u1290-\u12b0\u12b2-\u12b5\u12b8-\u12be\u12c0\u12c2-\u12c5\u12c8-\u12d6\u12d8-\u1310\u1312-\u1315\u1318-\u135a\u1380-\u138f\u13a0-\u13f5\u13f8-\u13fd\u1401-\u166c\u166f-\u167f\u1681-\u169a\u16a0-\u16ea\u16ee-\u16f8\u1700-\u170c\u170e-\u1711\u1720-\u1731\u1740-\u1751\u1760-\u176c\u176e-\u1770\u1780-\u17b3\u17d7\u17dc\u1820-\u1877\u1880-\u18a8\u18aa\u18b0-\u18f5\u1900-\u191e\u1950-\u196d\u1970-\u1974\u1980-\u19ab\u19b0-\u19c9\u1a00-\u1a16\u1a20-\u1a54\u1aa7\u1b05-\u1b33\u1b45-\u1b4b\u1b83-\u1ba0\u1bae\u1baf\u1bba-\u1be5\u1c00-\u1c23\u1c4d-\u1c4f\u1c5a-\u1c7d\u1c80-\u1c88\u1ce9-\u1cec\u1cee-\u1cf1\u1cf5\u1cf6\u1d00-\u1dbf\u1e00-\u1f15\u1f18-\u1f1d\u1f20-\u1f45\u1f48-\u1f4d\u1f50-\u1f57\u1f59\u1f5b\u1f5d\u1f5f-\u1f7d\u1f80-\u1fb4\u1fb6-\u1fbc\u1fbe\u1fc2-\u1fc4\u1fc6-\u1fcc\u1fd0-\u1fd3\u1fd6-\u1fdb\u1fe0-\u1fec\u1ff2-\u1ff4\u1ff6-\u1ffc\u2071\u207f\u2090-\u209c\u2102\u2107\u210a-\u2113\u2115\u2118-\u211d\u2124\u2126\u2128\u212a-\u2139\u213c-\u213f\u2145-\u2149\u214e\u2160-\u2188\u2c00-\u2c2e\u2c30-\u2c5e\u2c60-\u2ce4\u2ceb-\u2cee\u2cf2\u2cf3\u2d00-\u2d25\u2d27\u2d2d\u2d30-\u2d67\u2d6f\u2d80-\u2d96\u2da0-\u2da6\u2da8-\u2dae\u2db0-\u2db6\u2db8-\u2dbe\u2dc0-\u2dc6\u2dc8-\u2dce\u2dd0-\u2dd6\u2dd8-\u2dde\u3005-\u3007\u3021-\u3029\u3031-\u3035\u3038-\u303c\u3041-\u3096\u309b-\u309f\u30a1-\u30fa\u30fc-\u30ff\u3105-\u312d\u3131-\u318e\u31a0-\u31ba\u31f0-\u31ff\u3400-\u4db5\u4e00-\u9fd5\ua000-\ua48c\ua4d0-\ua4fd\ua500-\ua60c\ua610-\ua61f\ua62a\ua62b\ua640-\ua66e\ua67f-\ua69d\ua6a0-\ua6ef\ua717-\ua71f\ua722-\ua788\ua78b-\ua7ae\ua7b0-\ua7b7\ua7f7-\ua801\ua803-\ua805\ua807-\ua80a\ua80c-\ua822\ua840-\ua873\ua882-\ua8b3\ua8f2-\ua8f7\ua8fb\ua8fd\ua90a-\ua925\ua930-\ua946\ua960-\ua97c\ua984-\ua9b2\ua9cf\ua9e0-\ua9e4\ua9e6-\ua9ef\ua9fa-\ua9fe\uaa00-\uaa28\uaa40-\uaa42\uaa44-\uaa4b\uaa60-\uaa76\uaa7a\uaa7e-\uaaaf\uaab1\uaab5\uaab6\uaab9-\uaabd\uaac0\uaac2\uaadb-\uaadd\uaae0-\uaaea\uaaf2-\uaaf4\uab01-\uab06\uab09-\uab0e\uab11-\uab16\uab20-\uab26\uab28-\uab2e\uab30-\uab5a\uab5c-\uab65\uab70-\uabe2\uac00-\ud7a3\ud7b0-\ud7c6\ud7cb-\ud7fb\uf900-\ufa6d\ufa70-\ufad9\ufb00-\ufb06\ufb13-\ufb17\ufb1d\ufb1f-\ufb28\ufb2a-\ufb36\ufb38-\ufb3c\ufb3e\ufb40\ufb41\ufb43\ufb44\ufb46-\ufbb1\ufbd3-\ufd3d\ufd50-\ufd8f\ufd92-\ufdc7\ufdf0-\ufdfb\ufe70-\ufe74\ufe76-\ufefc\uff21-\uff3a\uff41-\uff5a\uff66-\uffbe\uffc2-\uffc7\uffca-\uffcf\uffd2-\uffd7\uffda-\uffdc"; let nonASCIIidentifierChars = "\u200c\u200d\xb7\u0300-\u036f\u0387\u0483-\u0487\u0591-\u05bd\u05bf\u05c1\u05c2\u05c4\u05c5\u05c7\u0610-\u061a\u064b-\u0669\u0670\u06d6-\u06dc\u06df-\u06e4\u06e7\u06e8\u06ea-\u06ed\u06f0-\u06f9\u0711\u0730-\u074a\u07a6-\u07b0\u07c0-\u07c9\u07eb-\u07f3\u0816-\u0819\u081b-\u0823\u0825-\u0827\u0829-\u082d\u0859-\u085b\u08d4-\u08e1\u08e3-\u0903\u093a-\u093c\u093e-\u094f\u0951-\u0957\u0962\u0963\u0966-\u096f\u0981-\u0983\u09bc\u09be-\u09c4\u09c7\u09c8\u09cb-\u09cd\u09d7\u09e2\u09e3\u09e6-\u09ef\u0a01-\u0a03\u0a3c\u0a3e-\u0a42\u0a47\u0a48\u0a4b-\u0a4d\u0a51\u0a66-\u0a71\u0a75\u0a81-\u0a83\u0abc\u0abe-\u0ac5\u0ac7-\u0ac9\u0acb-\u0acd\u0ae2\u0ae3\u0ae6-\u0aef\u0b01-\u0b03\u0b3c\u0b3e-\u0b44\u0b47\u0b48\u0b4b-\u0b4d\u0b56\u0b57\u0b62\u0b63\u0b66-\u0b6f\u0b82\u0bbe-\u0bc2\u0bc6-\u0bc8\u0bca-\u0bcd\u0bd7\u0be6-\u0bef\u0c00-\u0c03\u0c3e-\u0c44\u0c46-\u0c48\u0c4a-\u0c4d\u0c55\u0c56\u0c62\u0c63\u0c66-\u0c6f\u0c81-\u0c83\u0cbc\u0cbe-\u0cc4\u0cc6-\u0cc8\u0cca-\u0ccd\u0cd5\u0cd6\u0ce2\u0ce3\u0ce6-\u0cef\u0d01-\u0d03\u0d3e-\u0d44\u0d46-\u0d48\u0d4a-\u0d4d\u0d57\u0d62\u0d63\u0d66-\u0d6f\u0d82\u0d83\u0dca\u0dcf-\u0dd4\u0dd6\u0dd8-\u0ddf\u0de6-\u0def\u0df2\u0df3\u0e31\u0e34-\u0e3a\u0e47-\u0e4e\u0e50-\u0e59\u0eb1\u0eb4-\u0eb9\u0ebb\u0ebc\u0ec8-\u0ecd\u0ed0-\u0ed9\u0f18\u0f19\u0f20-\u0f29\u0f35\u0f37\u0f39\u0f3e\u0f3f\u0f71-\u0f84\u0f86\u0f87\u0f8d-\u0f97\u0f99-\u0fbc\u0fc6\u102b-\u103e\u1040-\u1049\u1056-\u1059\u105e-\u1060\u1062-\u1064\u1067-\u106d\u1071-\u1074\u1082-\u108d\u108f-\u109d\u135d-\u135f\u1369-\u1371\u1712-\u1714\u1732-\u1734\u1752\u1753\u1772\u1773\u17b4-\u17d3\u17dd\u17e0-\u17e9\u180b-\u180d\u1810-\u1819\u18a9\u1920-\u192b\u1930-\u193b\u1946-\u194f\u19d0-\u19da\u1a17-\u1a1b\u1a55-\u1a5e\u1a60-\u1a7c\u1a7f-\u1a89\u1a90-\u1a99\u1ab0-\u1abd\u1b00-\u1b04\u1b34-\u1b44\u1b50-\u1b59\u1b6b-\u1b73\u1b80-\u1b82\u1ba1-\u1bad\u1bb0-\u1bb9\u1be6-\u1bf3\u1c24-\u1c37\u1c40-\u1c49\u1c50-\u1c59\u1cd0-\u1cd2\u1cd4-\u1ce8\u1ced\u1cf2-\u1cf4\u1cf8\u1cf9\u1dc0-\u1df5\u1dfb-\u1dff\u203f\u2040\u2054\u20d0-\u20dc\u20e1\u20e5-\u20f0\u2cef-\u2cf1\u2d7f\u2de0-\u2dff\u302a-\u302f\u3099\u309a\ua620-\ua629\ua66f\ua674-\ua67d\ua69e\ua69f\ua6f0\ua6f1\ua802\ua806\ua80b\ua823-\ua827\ua880\ua881\ua8b4-\ua8c5\ua8d0-\ua8d9\ua8e0-\ua8f1\ua900-\ua909\ua926-\ua92d\ua947-\ua953\ua980-\ua983\ua9b3-\ua9c0\ua9d0-\ua9d9\ua9e5\ua9f0-\ua9f9\uaa29-\uaa36\uaa43\uaa4c\uaa4d\uaa50-\uaa59\uaa7b-\uaa7d\uaab0\uaab2-\uaab4\uaab7\uaab8\uaabe\uaabf\uaac1\uaaeb-\uaaef\uaaf5\uaaf6\uabe3-\uabea\uabec\uabed\uabf0-\uabf9\ufb1e\ufe00-\ufe0f\ufe20-\ufe2f\ufe33\ufe34\ufe4d-\ufe4f\uff10-\uff19\uff3f"; const nonASCIIidentifierStart = new RegExp("[" + nonASCIIidentifierStartChars + "]"); const nonASCIIidentifier = new RegExp("[" + nonASCIIidentifierStartChars + nonASCIIidentifierChars + "]"); nonASCIIidentifierStartChars = nonASCIIidentifierChars = null; // These are a run-length and offset encoded representation of the // >0xffff code points that are a valid part of identifiers. The // offset starts at 0x10000, and each pair of numbers represents an // offset to the next range, and then a size of the range. They were // generated by `bin/generate-identifier-regex.js`. // eslint-disable-next-line comma-spacing const astralIdentifierStartCodes = [0,11,2,25,2,18,2,1,2,14,3,13,35,122,70,52,268,28,4,48,48,31,17,26,6,37,11,29,3,35,5,7,2,4,43,157,19,35,5,35,5,39,9,51,157,310,10,21,11,7,153,5,3,0,2,43,2,1,4,0,3,22,11,22,10,30,66,18,2,1,11,21,11,25,71,55,7,1,65,0,16,3,2,2,2,26,45,28,4,28,36,7,2,27,28,53,11,21,11,18,14,17,111,72,56,50,14,50,785,52,76,44,33,24,27,35,42,34,4,0,13,47,15,3,22,0,2,0,36,17,2,24,85,6,2,0,2,3,2,14,2,9,8,46,39,7,3,1,3,21,2,6,2,1,2,4,4,0,19,0,13,4,159,52,19,3,54,47,21,1,2,0,185,46,42,3,37,47,21,0,60,42,86,25,391,63,32,0,449,56,264,8,2,36,18,0,50,29,881,921,103,110,18,195,2749,1070,4050,582,8634,568,8,30,114,29,19,47,17,3,32,20,6,18,881,68,12,0,67,12,65,0,32,6124,20,754,9486,1,3071,106,6,12,4,8,8,9,5991,84,2,70,2,1,3,0,3,1,3,3,2,11,2,0,2,6,2,64,2,3,3,7,2,6,2,27,2,3,2,4,2,0,4,6,2,339,3,24,2,24,2,30,2,24,2,30,2,24,2,30,2,24,2,30,2,24,2,7,4149,196,60,67,1213,3,2,26,2,1,2,0,3,0,2,9,2,3,2,0,2,0,7,0,5,0,2,0,2,0,2,2,2,1,2,0,3,0,2,0,2,0,2,0,2,0,2,1,2,0,3,3,2,6,2,3,2,3,2,0,2,9,2,16,6,2,2,4,2,16,4421,42710,42,4148,12,221,3,5761,10591,541]; // eslint-disable-next-line comma-spacing const astralIdentifierCodes = [509,0,227,0,150,4,294,9,1368,2,2,1,6,3,41,2,5,0,166,1,1306,2,54,14,32,9,16,3,46,10,54,9,7,2,37,13,2,9,52,0,13,2,49,13,10,2,4,9,83,11,7,0,161,11,6,9,7,3,57,0,2,6,3,1,3,2,10,0,11,1,3,6,4,4,193,17,10,9,87,19,13,9,214,6,3,8,28,1,83,16,16,9,82,12,9,9,84,14,5,9,423,9,838,7,2,7,17,9,57,21,2,13,19882,9,135,4,60,6,26,9,1016,45,17,3,19723,1,5319,4,4,5,9,7,3,6,31,3,149,2,1418,49,513,54,5,49,9,0,15,0,23,4,2,14,1361,6,2,16,3,6,2,1,2,4,2214,6,110,6,6,9,792487,239]; // This has a complexity linear to the value of the code. The // assumption is that looking up astral identifier characters is // rare. function isInAstralSet(code, set) { let pos = 0x10000; for (let i = 0; i < set.length; i += 2) { pos += set[i]; if (pos > code) return false; pos += set[i + 1]; if (pos >= code) return true; } } // Test whether a given character code starts an identifier. function isIdentifierStart(code) { if (code < 65) return code === 36; if (code < 91) return true; if (code < 97) return code === 95; if (code < 123) return true; if (code <= 0xffff) return code >= 0xaa && nonASCIIidentifierStart.test(String.fromCharCode(code)); return isInAstralSet(code, astralIdentifierStartCodes); } utilIdentifierJS.isIdentifierStart = isIdentifierStart; // Test whether a given character is part of an identifier. function isIdentifierChar(code) { if (code < 48) return code === 36; if (code < 58) return true; if (code < 65) return false; if (code < 91) return true; if (code < 97) return code === 95; if (code < 123) return true; if (code <= 0xffff) return code >= 0xaa && nonASCIIidentifier.test(String.fromCharCode(code)); return isInAstralSet(code, astralIdentifierStartCodes) || isInAstralSet(code, astralIdentifierCodes); } utilIdentifierJS.isIdentifierChar = isIdentifierChar; } // util/whitespace.js const utilWhitespaceJS = {}; { const lineBreak = /\r\n?|\n|\u2028|\u2029/; utilWhitespaceJS.lineBreak = lineBreak; const lineBreakG = new RegExp(lineBreak.source, "g"); utilWhitespaceJS.lineBreakG = lineBreakG; function isNewLine(code){ return code === 10 || code === 13 || code === 0x2028 || code === 0x2029; } utilWhitespaceJS.isNewLine = isNewLine; const nonASCIIwhitespace = /[\u1680\u180e\u2000-\u200a\u202f\u205f\u3000\ufeff]/; utilWhitespaceJS.nonASCIIwhitespace = nonASCIIwhitespace; } // util/location.js const utilLocationJS = {}; { const lineBreakG = utilWhitespaceJS.lineBreakG; // These are used when `options.locations` is on, for the // `startLoc` and `endLoc` properties. class Position { constructor(line, col) { this.line = line; this.column = col; } } utilLocationJS.Position = Position; class SourceLocation { constructor(start, end) { this.start = start; this.end = end; } } utilLocationJS.SourceLocation = SourceLocation; // The `getLineInfo` function is mostly useful when the // `locations` option is off (for performance reasons) and you // want to find the line/column position for a given character // offset. `input` should be the code string that the offset refers // into. function getLineInfo(input, offset) { for (let line = 1, cur = 0; ;) { lineBreakG.lastIndex = cur; const match = lineBreakG.exec(input); if (match && match.index < offset) { ++line; cur = match.index + match[0].length; } else { return new Position(line, offset - cur); } } } utilLocationJS.getLineInfo = getLineInfo; } // tokenizer/types.js const tokenizerTypesJS = {}; { // ## Token types // The assignment of fine-grained, information-carrying type objects // allows the tokenizer to store the information it has about a // token in a way that is very cheap for the parser to look up. // All token type variables start with an underscore, to make them // easy to recognize. // The `beforeExpr` property is used to disambiguate between regular // expressions and divisions. It is set on all token types that can // be followed by an expression (thus, a slash after them would be a // regular expression). // // `isLoop` marks a keyword as starting a loop, which is important // to know when parsing a label, in order to allow or disallow // continue jumps to that label. const beforeExpr = true; const startsExpr = true; const isLoop = true; const isAssign = true; const prefix = true; const postfix = true; class TokenType { constructor(label, conf = {}) { this.label = label; this.keyword = conf.keyword; this.beforeExpr = !!conf.beforeExpr; this.startsExpr = !!conf.startsExpr; this.rightAssociative = !!conf.rightAssociative; this.isLoop = !!conf.isLoop; this.isAssign = !!conf.isAssign; this.prefix = !!conf.prefix; this.postfix = !!conf.postfix; this.binop = conf.binop || null; this.updateContext = null; } } tokenizerTypesJS.TokenType = TokenType; class KeywordTokenType extends TokenType { constructor(name, options = {}) { options.keyword = name; super(name, options); } } class BinopTokenType extends TokenType { constructor(name, prec) { super(name, { beforeExpr, binop: prec }); } } tokenizerTypesJS.BinopTokenType = BinopTokenType; const types = { num: new TokenType("num", { startsExpr }), regexp: new TokenType("regexp", { startsExpr }), string: new TokenType("string", { startsExpr }), name: new TokenType("name", { startsExpr }), eof: new TokenType("eof"), // Punctuation token types. bracketL: new TokenType("[", { beforeExpr, startsExpr }), bracketR: new TokenType("]"), braceL: new TokenType("{", { beforeExpr, startsExpr }), braceBarL: new TokenType("{|", { beforeExpr, startsExpr }), braceR: new TokenType("}"), braceBarR: new TokenType("|}"), parenL: new TokenType("(", { beforeExpr, startsExpr }), parenR: new TokenType(")"), comma: new TokenType(",", { beforeExpr }), semi: new TokenType(";", { beforeExpr }), colon: new TokenType(":", { beforeExpr }), doubleColon: new TokenType("::", { beforeExpr }), dot: new TokenType("."), question: new TokenType("?", { beforeExpr }), arrow: new TokenType("=>", { beforeExpr }), template: new TokenType("template"), ellipsis: new TokenType("...", { beforeExpr }), backQuote: new TokenType("`", { startsExpr }), dollarBraceL: new TokenType("${", { beforeExpr, startsExpr }), at: new TokenType("@"), // Operators. These carry several kinds of properties to help the // parser use them properly (the presence of these properties is // what categorizes them as operators). // // `binop`, when present, specifies that this operator is a binary // operator, and will refer to its precedence. // // `prefix` and `postfix` mark the operator as a prefix or postfix // unary operator. // // `isAssign` marks all of `=`, `+=`, `-=` etcetera, which act as // binary operators with a very low precedence, that should result // in AssignmentExpression nodes. eq: new TokenType("=", { beforeExpr, isAssign }), assign: new TokenType("_=", { beforeExpr, isAssign }), incDec: new TokenType("++/--", { prefix, postfix, startsExpr }), prefix: new TokenType("prefix", { beforeExpr, prefix, startsExpr }), logicalOR: new BinopTokenType("||", 1), logicalAND: new BinopTokenType("&&", 2), bitwiseOR: new BinopTokenType("|", 3), bitwiseXOR: new BinopTokenType("^", 4), bitwiseAND: new BinopTokenType("&", 5), equality: new BinopTokenType("==/!=", 6), relational: new BinopTokenType("", 7), bitShift: new BinopTokenType("<>", 8), plusMin: new TokenType("+/-", { beforeExpr, binop: 9, prefix, startsExpr }), modulo: new BinopTokenType("%", 10), star: new BinopTokenType("*", 10), slash: new BinopTokenType("/", 10), exponent: new TokenType("**", { beforeExpr, binop: 11, rightAssociative: true }) }; tokenizerTypesJS.types = types; const keywords = { "break": new KeywordTokenType("break"), "case": new KeywordTokenType("case", { beforeExpr }), "catch": new KeywordTokenType("catch"), "continue": new KeywordTokenType("continue"), "debugger": new KeywordTokenType("debugger"), "default": new KeywordTokenType("default", { beforeExpr }), "do": new KeywordTokenType("do", { isLoop, beforeExpr }), "else": new KeywordTokenType("else", { beforeExpr }), "finally": new KeywordTokenType("finally"), "for": new KeywordTokenType("for", { isLoop }), "function": new KeywordTokenType("function", { startsExpr }), "if": new KeywordTokenType("if"), "return": new KeywordTokenType("return", { beforeExpr }), "switch": new KeywordTokenType("switch"), "throw": new KeywordTokenType("throw", { beforeExpr }), "try": new KeywordTokenType("try"), "var": new KeywordTokenType("var"), "let": new KeywordTokenType("let"), "const": new KeywordTokenType("const"), "while": new KeywordTokenType("while", { isLoop }), "with": new KeywordTokenType("with"), "new": new KeywordTokenType("new", { beforeExpr, startsExpr }), "this": new KeywordTokenType("this", { startsExpr }), "super": new KeywordTokenType("super", { startsExpr }), "class": new KeywordTokenType("class"), "extends": new KeywordTokenType("extends", { beforeExpr }), "export": new KeywordTokenType("export"), "import": new KeywordTokenType("import", { startsExpr }), "yield": new KeywordTokenType("yield", { beforeExpr, startsExpr }), "null": new KeywordTokenType("null", { startsExpr }), "true": new KeywordTokenType("true", { startsExpr }), "false": new KeywordTokenType("false", { startsExpr }), "in": new KeywordTokenType("in", { beforeExpr, binop: 7 }), "instanceof": new KeywordTokenType("instanceof", { beforeExpr, binop: 7 }), "typeof": new KeywordTokenType("typeof", { beforeExpr, prefix, startsExpr }), "void": new KeywordTokenType("void", { beforeExpr, prefix, startsExpr }), "delete": new KeywordTokenType("delete", { beforeExpr, prefix, startsExpr }) }; tokenizerTypesJS.keywords = keywords; // Map keyword names to token types. Object.keys(keywords).forEach((name) => { types["_" + name] = keywords[name]; }); } // tokenizer/context.js const tokenizerContextJS = {}; { // The algorithm used to determine whether a regexp can appear at a // given point in the program is loosely based on sweet.js' approach. // See https://github.com/mozilla/sweet.js/wiki/design const tt = tokenizerTypesJS.types; const lineBreak = utilWhitespaceJS.lineBreak; class TokContext { constructor( token, isExpr, preserveSpace, override, ) { this.token = token; this.isExpr = !!isExpr; this.preserveSpace = !!preserveSpace; this.override = override; } //token: string; //isExpr: boolean; //preserveSpace: boolean; //override: ?Function; } tokenizerContextJS.TokContext = TokContext; const types = { braceStatement: new TokContext("{", false), braceExpression: new TokContext("{", true), templateQuasi: new TokContext("${", true), parenStatement: new TokContext("(", false), parenExpression: new TokContext("(", true), template: new TokContext("`", true, true, (p) => p.readTmplToken()), functionExpression: new TokContext("function", true) }; tokenizerContextJS.types = types; // Token-specific context update code tt.parenR.updateContext = tt.braceR.updateContext = function () { if (this.state.context.length === 1) { this.state.exprAllowed = true; return; } const out = this.state.context.pop(); if (out === types.braceStatement && this.curContext() === types.functionExpression) { this.state.context.pop(); this.state.exprAllowed = false; } else if (out === types.templateQuasi) { this.state.exprAllowed = true; } else { this.state.exprAllowed = !out.isExpr; } }; tt.name.updateContext = function (prevType) { this.state.exprAllowed = false; if (prevType === tt._let || prevType === tt._const || prevType === tt._var) { if (lineBreak.test(this.input.slice(this.state.end))) { this.state.exprAllowed = true; } } }; tt.braceL.updateContext = function (prevType) { this.state.context.push(this.braceIsBlock(prevType) ? types.braceStatement : types.braceExpression); this.state.exprAllowed = true; }; tt.dollarBraceL.updateContext = function () { this.state.context.push(types.templateQuasi); this.state.exprAllowed = true; }; tt.parenL.updateContext = function (prevType) { const statementParens = prevType === tt._if || prevType === tt._for || prevType === tt._with || prevType === tt._while; this.state.context.push(statementParens ? types.parenStatement : types.parenExpression); this.state.exprAllowed = true; }; tt.incDec.updateContext = function () { // tokExprAllowed stays unchanged }; tt._function.updateContext = function () { if (this.curContext() !== types.braceStatement) { this.state.context.push(types.functionExpression); } this.state.exprAllowed = false; }; tt.backQuote.updateContext = function () { if (this.curContext() === types.template) { this.state.context.pop(); } else { this.state.context.push(types.template); } this.state.exprAllowed = false; }; } // tokenizer/state.js let tokenizerStateJS = {}; { const TokContext = tokenizerContextJS.TokContext; const TokenType = tokenizerTypesJS.TokenType; const Position = utilLocationJS.Position; const ct = tokenizerContextJS.types; const tt = tokenizerTypesJS.types; //export default class State { class State { init(options, input) { this.strict = options.strictMode === false ? false : options.sourceType === "module"; this.input = input; this.potentialArrowAt = -1; this.inMethod = this.inFunction = this.inGenerator = this.inAsync = this.inPropertyName = this.inType = this.noAnonFunctionType = false; this.labels = []; this.decorators = []; this.tokens = []; this.comments = []; this.trailingComments = []; this.leadingComments = []; this.commentStack = []; this.pos = this.lineStart = 0; this.curLine = options.startLine; this.type = tt.eof; this.value = null; this.start = this.end = this.pos; this.startLoc = this.endLoc = this.curPosition(); this.lastTokEndLoc = this.lastTokStartLoc = null; this.lastTokStart = this.lastTokEnd = this.pos; this.context = [ct.braceStatement]; this.exprAllowed = true; this.containsEsc = this.containsOctal = false; this.octalPosition = null; this.invalidTemplateEscapePosition = null; this.exportedIdentifiers = []; return this; } //// TODO //strict: boolean; //// TODO //input: string; //// Used to signify the start of a potential arrow function //potentialArrowAt: number; //// Flags to track whether we are in a function, a generator. //inFunction: boolean; //inGenerator: boolean; //inMethod: boolean; //inAsync: boolean; //inType: boolean; //inPropertyName: boolean; //// Labels in scope. //labels: Array; //// Leading decorators. //decorators: Array; //// Token store. //tokens: Array; //// Comment store. //comments: Array; //// Comment attachment store //trailingComments: Array; //leadingComments: Array; //commentStack: Array; //// The current position of the tokenizer in the input. //pos: number; //lineStart: number; //curLine: number; //// Properties of the current token: //// Its type //type: TokenType; //// For tokens that include more information than their type, the value //value: any; //// Its start and end offset //start: number; //end: number; //// And, if locations are used, the {line, column} object //// corresponding to those offsets //startLoc: Position; //endLoc: Position; //// Position information for the previous token //lastTokEndLoc: ?Position; //lastTokStartLoc: ?Position; //lastTokStart: number; //lastTokEnd: number; //// The context stack is used to superficially track syntactic //// context to predict whether a regular expression is allowed in a //// given position. //context: Array; //exprAllowed: boolean; //// Used to signal to callers of `readWord1` whether the word //// contained any escape sequences. This is needed because words with //// escape sequences must not be interpreted as keywords. //containsEsc: boolean; //// TODO //containsOctal: boolean; //octalPosition: ?number; //// Names of exports store. `default` is stored as a name for both //// `export default foo;` and `export { foo as default };`. //exportedIdentifiers: Array; curPosition() { return new Position(this.curLine, this.pos - this.lineStart); } clone(skipArrays) { const state = new State; for (const key in this) { let val = this[key]; if ((!skipArrays || key === "context") && Array.isArray(val)) { val = val.slice(); } state[key] = val; } return state; } } tokenizerStateJS = State; } // tokenizer/index.js let tokenizerIndexJS = {}; { /* eslint max-len: 0 */ const TokenType = tokenizerTypesJS.TokenType; const {isIdentifierStart, isIdentifierChar, isKeyword} = utilIdentifierJS; const { types: tt , keywords: keywordTypes } = tokenizerTypesJS; const { types: ct } = tokenizerContextJS; const { SourceLocation } = utilLocationJS; const { lineBreak, lineBreakG, isNewLine, nonASCIIwhitespace } = utilWhitespaceJS; const State = tokenizerStateJS; // Object type used to represent tokens. Note that normally, tokens // simply exist as properties on the parser object. This is only // used for the onToken callback and the external tokenizer. //export class Token { class Token { constructor(state) { this.type = state.type; this.value = state.value; this.start = state.start; this.end = state.end; this.loc = new SourceLocation(state.startLoc, state.endLoc); } } // ## Tokenizer function codePointToString(code) { // UTF-16 Decoding if (code <= 0xFFFF) { return String.fromCharCode(code); } else { return String.fromCharCode(((code - 0x10000) >> 10) + 0xD800, ((code - 0x10000) & 1023) + 0xDC00); } } class Tokenizer { constructor(options, input) { this.state = new State; this.state.init(options, input); } // Move to the next token next() { if (!this.isLookahead) { this.state.tokens.push(new Token(this.state)); } this.state.lastTokEnd = this.state.end; this.state.lastTokStart = this.state.start; this.state.lastTokEndLoc = this.state.endLoc; this.state.lastTokStartLoc = this.state.startLoc; this.nextToken(); } // TODO eat(type) { if (this.match(type)) { this.next(); return true; } else { return false; } } // TODO match(type) { return this.state.type === type; } // TODO isKeyword(word) { return isKeyword(word); } // TODO lookahead() { const old = this.state; this.state = old.clone(true); this.isLookahead = true; this.next(); this.isLookahead = false; const curr = this.state.clone(true); this.state = old; return curr; } // Toggle strict mode. Re-reads the next number or string to please // pedantic tests (`"use strict"; 010;` should fail). setStrict(strict) { this.state.strict = strict; if (!this.match(tt.num) && !this.match(tt.string)) return; this.state.pos = this.state.start; while (this.state.pos < this.state.lineStart) { this.state.lineStart = this.input.lastIndexOf("\n", this.state.lineStart - 2) + 1; --this.state.curLine; } this.nextToken(); } curContext() { return this.state.context[this.state.context.length - 1]; } // Read a single token, updating the parser object's token-related // properties. nextToken() { const curContext = this.curContext(); if (!curContext || !curContext.preserveSpace) this.skipSpace(); this.state.containsOctal = false; this.state.octalPosition = null; this.state.start = this.state.pos; this.state.startLoc = this.state.curPosition(); if (this.state.pos >= this.input.length) return this.finishToken(tt.eof); if (curContext.override) { return curContext.override(this); } else { return this.readToken(this.fullCharCodeAtPos()); } } readToken(code) { // Identifier or keyword. '\uXXXX' sequences are allowed in // identifiers, so '\' also dispatches to that. if (isIdentifierStart(code) || code === 92 /* '\' */) { return this.readWord(); } else { return this.getTokenFromCode(code); } } fullCharCodeAtPos() { const code = this.input.charCodeAt(this.state.pos); if (code <= 0xd7ff || code >= 0xe000) return code; const next = this.input.charCodeAt(this.state.pos + 1); return (code << 10) + next - 0x35fdc00; } pushComment(block, text, start, end, startLoc, endLoc) { const comment = { type: block ? "CommentBlock" : "CommentLine", value: text, start: start, end: end, loc: new SourceLocation(startLoc, endLoc) }; if (!this.isLookahead) { this.state.tokens.push(comment); this.state.comments.push(comment); this.addComment(comment); } } skipBlockComment() { const startLoc = this.state.curPosition(); const start = this.state.pos; const end = this.input.indexOf("*/", this.state.pos += 2); if (end === -1) this.raise(this.state.pos - 2, "Unterminated comment"); this.state.pos = end + 2; lineBreakG.lastIndex = start; let match; while ((match = lineBreakG.exec(this.input)) && match.index < this.state.pos) { ++this.state.curLine; this.state.lineStart = match.index + match[0].length; } this.pushComment(true, this.input.slice(start + 2, end), start, this.state.pos, startLoc, this.state.curPosition()); } skipLineComment(startSkip) { const start = this.state.pos; const startLoc = this.state.curPosition(); let ch = this.input.charCodeAt(this.state.pos += startSkip); while (this.state.pos < this.input.length && ch !== 10 && ch !== 13 && ch !== 8232 && ch !== 8233) { ++this.state.pos; ch = this.input.charCodeAt(this.state.pos); } this.pushComment(false, this.input.slice(start + startSkip, this.state.pos), start, this.state.pos, startLoc, this.state.curPosition()); } // Called at the start of the parse and after every token. Skips // whitespace and comments, and. skipSpace() { loop: while (this.state.pos < this.input.length) { const ch = this.input.charCodeAt(this.state.pos); switch (ch) { case 32: case 160: // ' ' ++this.state.pos; break; case 13: if (this.input.charCodeAt(this.state.pos + 1) === 10) { ++this.state.pos; } case 10: case 8232: case 8233: ++this.state.pos; ++this.state.curLine; this.state.lineStart = this.state.pos; break; case 47: // '/' switch (this.input.charCodeAt(this.state.pos + 1)) { case 42: // '*' this.skipBlockComment(); break; case 47: this.skipLineComment(2); break; default: break loop; } break; default: if (ch > 8 && ch < 14 || ch >= 5760 && nonASCIIwhitespace.test(String.fromCharCode(ch))) { ++this.state.pos; } else { break loop; } } } } // Called at the end of every token. Sets `end`, `val`, and // maintains `context` and `exprAllowed`, and skips the space after // the token, so that the next one's `start` will point at the // right position. finishToken(type, val) { this.state.end = this.state.pos; this.state.endLoc = this.state.curPosition(); const prevType = this.state.type; this.state.type = type; this.state.value = val; this.updateContext(prevType); } // ### Token reading // This is the function that is called to fetch the next token. It // is somewhat obscure, because it works in character codes rather // than characters, and because operator parsing has been inlined // into it. // // All in the name of speed. // readToken_dot() { const next = this.input.charCodeAt(this.state.pos + 1); if (next >= 48 && next <= 57) { return this.readNumber(true); } const next2 = this.input.charCodeAt(this.state.pos + 2); if (next === 46 && next2 === 46) { // 46 = dot '.' this.state.pos += 3; return this.finishToken(tt.ellipsis); } else { ++this.state.pos; return this.finishToken(tt.dot); } } readToken_slash() { // '/' if (this.state.exprAllowed) { ++this.state.pos; return this.readRegexp(); } const next = this.input.charCodeAt(this.state.pos + 1); if (next === 61) { return this.finishOp(tt.assign, 2); } else { return this.finishOp(tt.slash, 1); } } readToken_mult_modulo(code) { // '%*' let type = code === 42 ? tt.star : tt.modulo; let width = 1; let next = this.input.charCodeAt(this.state.pos + 1); if (next === 42) { // '*' width++; next = this.input.charCodeAt(this.state.pos + 2); type = tt.exponent; } if (next === 61) { width++; type = tt.assign; } return this.finishOp(type, width); } readToken_pipe_amp(code) { // '|&' const next = this.input.charCodeAt(this.state.pos + 1); if (next === code) return this.finishOp(code === 124 ? tt.logicalOR : tt.logicalAND, 2); if (next === 61) return this.finishOp(tt.assign, 2); if (code === 124 && next === 125 && this.hasPlugin("flow")) return this.finishOp(tt.braceBarR, 2); return this.finishOp(code === 124 ? tt.bitwiseOR : tt.bitwiseAND, 1); } readToken_caret() { // '^' const next = this.input.charCodeAt(this.state.pos + 1); if (next === 61) { return this.finishOp(tt.assign, 2); } else { return this.finishOp(tt.bitwiseXOR, 1); } } readToken_plus_min(code) { // '+-' const next = this.input.charCodeAt(this.state.pos + 1); if (next === code) { if (next === 45 && this.input.charCodeAt(this.state.pos + 2) === 62 && lineBreak.test(this.input.slice(this.state.lastTokEnd, this.state.pos))) { // A `-->` line comment this.skipLineComment(3); this.skipSpace(); return this.nextToken(); } return this.finishOp(tt.incDec, 2); } if (next === 61) { return this.finishOp(tt.assign, 2); } else { return this.finishOp(tt.plusMin, 1); } } readToken_lt_gt(code) { // '<>' const next = this.input.charCodeAt(this.state.pos + 1); let size = 1; if (next === code) { size = code === 62 && this.input.charCodeAt(this.state.pos + 2) === 62 ? 3 : 2; if (this.input.charCodeAt(this.state.pos + size) === 61) return this.finishOp(tt.assign, size + 1); return this.finishOp(tt.bitShift, size); } if (next === 33 && code === 60 && this.input.charCodeAt(this.state.pos + 2) === 45 && this.input.charCodeAt(this.state.pos + 3) === 45) { if (this.inModule) this.unexpected(); // `