/* This Source Code Form is subject to the terms of the Mozilla Public * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ "use strict"; const { InspectorCSSParserWrapper, } = require("resource://devtools/shared/css/lexer.js"); add_task(function test_lexer() { const LEX_TESTS = [ ["simple", [{ tokenType: "Ident", text: "simple", value: "simple" }]], [ "simple: { hi; }", [ { tokenType: "Ident", text: "simple", value: "simple" }, { tokenType: "Colon", text: ":" }, { tokenType: "WhiteSpace", text: " " }, { tokenType: "CurlyBracketBlock", text: "{" }, { tokenType: "WhiteSpace", text: " " }, { tokenType: "Ident", text: "hi", value: "hi" }, { tokenType: "Semicolon", text: ";" }, { tokenType: "WhiteSpace", text: " " }, { tokenType: "CloseCurlyBracket", text: "}" }, ], ], [ "/* whatever */", [{ tokenType: "Comment", text: "/* whatever */", value: " whatever " }], ], [ "'string'", [{ tokenType: "QuotedString", text: "'string'", value: "string" }], ], [ '"string"', [{ tokenType: "QuotedString", text: `"string"`, value: "string" }], ], [ "rgb(1,2,3)", [ { tokenType: "Function", text: "rgb(", value: "rgb" }, { tokenType: "Number", text: "1", number: 1 }, { tokenType: "Comma", text: "," }, { tokenType: "Number", text: "2", number: 2 }, { tokenType: "Comma", text: "," }, { tokenType: "Number", text: "3", number: 3 }, { tokenType: "CloseParenthesis", text: ")" }, ], ], ["@media", [{ tokenType: "AtKeyword", text: "@media", value: "media" }]], ["#hibob", [{ tokenType: "IDHash", text: "#hibob", value: "hibob" }]], ["#123", [{ tokenType: "Hash", text: "#123", value: "123" }]], [ "23px", [{ tokenType: "Dimension", text: "23px", number: 23, unit: "px" }], ], ["23%", [{ tokenType: "Percentage", text: "23%", number: 0.23 }]], [ "url(http://example.com)", [ { tokenType: "UnquotedUrl", text: "url(http://example.com)", value: "http://example.com", }, ], ], [ "url('http://example.com')", [ { tokenType: "Function", text: "url(", value: "url" }, { tokenType: "QuotedString", text: "'http://example.com'", value: "http://example.com", }, { tokenType: "CloseParenthesis", text: ")" }, ], ], [ "url( 'http://example.com' )", [ { tokenType: "Function", text: "url(", value: "url" }, { tokenType: "WhiteSpace", text: " " }, { tokenType: "QuotedString", text: "'http://example.com'", value: "http://example.com", }, { tokenType: "WhiteSpace", text: " " }, { tokenType: "CloseParenthesis", text: ")" }, ], ], // In CSS Level 3, this is an ordinary URL, not a BAD_URL. [ "url(http://example.com", [ { tokenType: "UnquotedUrl", text: "url(http://example.com", value: "http://example.com", }, ], ], [ "url(http://example.com @", [ { tokenType: "BadUrl", text: "url(http://example.com @", value: "http://example.com @", }, ], ], [ "quo\\ting", [{ tokenType: "Ident", text: "quo\\ting", value: "quoting" }], ], [ "'bad string\n", [ { tokenType: "BadString", text: "'bad string", value: "bad string" }, { tokenType: "WhiteSpace", text: "\n" }, ], ], ["~=", [{ tokenType: "IncludeMatch", text: "~=" }]], ["|=", [{ tokenType: "DashMatch", text: "|=" }]], ["^=", [{ tokenType: "PrefixMatch", text: "^=" }]], ["$=", [{ tokenType: "SuffixMatch", text: "$=" }]], ["*=", [{ tokenType: "SubstringMatch", text: "*=" }]], [ "", [ { tokenType: "CDO", text: "" }, ], ], // earlier versions of CSS had "bad comment" tokens, but in level 3, // unterminated comments are just comments. [ "/* bad comment", [{ tokenType: "Comment", text: "/* bad comment", value: " bad comment" }], ], ]; const test = (cssText, tokenTypes) => { const lexer = new InspectorCSSParserWrapper(cssText); let reconstructed = ""; let lastTokenEnd = 0; let i = 0; let token; while ((token = lexer.nextToken())) { const expectedToken = tokenTypes[i]; Assert.deepEqual( { tokenType: token.tokenType, text: token.text, value: token.value, number: token.number, unit: token.unit, }, { tokenType: expectedToken.tokenType, text: expectedToken.text, value: expectedToken.value ?? null, number: expectedToken.number ?? null, unit: expectedToken.unit ?? null, }, `Got expected token #${i} for "${cssText}"` ); Assert.greater(token.endOffset, token.startOffset); equal(token.startOffset, lastTokenEnd); lastTokenEnd = token.endOffset; reconstructed += cssText.substring(token.startOffset, token.endOffset); ++i; } // Ensure that we saw the correct number of tokens. equal(i, tokenTypes.length); // Ensure that the reported offsets cover all the text. equal(reconstructed, cssText); }; for (const [cssText, rustTokenTypes] of LEX_TESTS) { info(`Test "${cssText}"`); test(cssText, rustTokenTypes); } }); add_task(function test_lexer_linecol() { const LINECOL_TESTS = [ ["simple", ["Ident:0:0", ":0:6"]], ["\n stuff", ["WhiteSpace:0:0", "Ident:1:4", ":1:9"]], [ '"string with \\\nnewline" \r\n', ["QuotedString:0:0", "WhiteSpace:1:8", ":2:0"], ], ]; const test = (cssText, locations) => { const lexer = new InspectorCSSParserWrapper(cssText); let i = 0; let token; const testLocation = () => { const startLine = lexer.parser.lineNumber; const startColumn = lexer.parser.columnNumber; // We do this in a bit of a funny way so that we can also test the // location of the EOF. let combined = ":" + startLine + ":" + startColumn; if (token) { combined = token.tokenType + combined; } equal(combined, locations[i]); ++i; }; while ((token = lexer.nextToken())) { testLocation(); } // Collect location after we consumed all the tokens testLocation(); // Ensure that we saw the correct number of tokens. equal(i, locations.length); }; for (const [cssText, rustLocations] of LINECOL_TESTS) { info(`Test "${cssText}"`); test(cssText, rustLocations); } }); add_task(function test_lexer_eofchar() { const EOFCHAR_TESTS = [ ["hello", "hello"], ["hello \\", "hello \\\\"], ["'hello", "'hello'"], ['"hello', '"hello"'], ["'hello\\", "'hello\\\\'"], ['"hello\\', '"hello\\\\"'], ["/*hello", "/*hello*/"], ["/*hello*", "/*hello*/"], ["/*hello\\", "/*hello\\*/"], ["url(hello", "url(hello)"], ["url('hello", "url('hello')"], ['url("hello', 'url("hello")'], ["url(hello\\", "url(hello\\\\)"], ["url('hello\\", "url('hello\\\\')"], ['url("hello\\', 'url("hello\\\\")'], // Ensure that passing a different inputString to performEOFFixup // doesn't cause an assertion trying to strip a backslash from the // end of an empty string. ["'\\", "\\'", ""], ]; const test = (cssText, expectedAppend, argText) => { const lexer = new InspectorCSSParserWrapper(cssText, { trackEOFChars: true, }); while (lexer.nextToken()) { // We don't need to do anything with the tokens. We only want to consume the iterator // so we can safely call performEOFFixup. } info("EOF char test, input = " + cssText); equal(lexer.performEOFFixup(argText), expectedAppend); }; for (const [cssText, expectedAppend, argText = cssText] of EOFCHAR_TESTS) { info(`Test "${cssText}"`); test(cssText, expectedAppend, argText); } });