diff options
Diffstat (limited to 'comm/mailnews/base/test/unit/test_mailstoreConverter.js')
| -rw-r--r-- | comm/mailnews/base/test/unit/test_mailstoreConverter.js | 376 |
1 files changed, 376 insertions, 0 deletions
diff --git a/comm/mailnews/base/test/unit/test_mailstoreConverter.js b/comm/mailnews/base/test/unit/test_mailstoreConverter.js new file mode 100644 index 0000000000..f440b9c8cd --- /dev/null +++ b/comm/mailnews/base/test/unit/test_mailstoreConverter.js @@ -0,0 +1,376 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +var { convertMailStoreTo } = ChromeUtils.import( + "resource:///modules/mailstoreConverter.jsm" +); + +Services.prefs.setCharPref( + "mail.serverDefaultStoreContractID", + "@mozilla.org/msgstore/berkeleystore;1" +); + +// Test data for round-trip test. +let testEmails = [ + // Base64 encoded bodies. + "../../../data/01-plaintext.eml", + "../../../data/02-plaintext+attachment.eml", + "../../../data/03-HTML.eml", + "../../../data/04-HTML+attachment.eml", + "../../../data/05-HTML+embedded-image.eml", + "../../../data/06-plaintext+HMTL.eml", + "../../../data/07-plaintext+(HTML+embedded-image).eml", + "../../../data/08-plaintext+HTML+attachment.eml", + "../../../data/09-(HTML+embedded-image)+attachment.eml", + "../../../data/10-plaintext+(HTML+embedded-image)+attachment.eml", + + // Bodies with non-ASCII characters in UTF-8 and other charsets. + "../../../data/11-plaintext.eml", + "../../../data/12-plaintext+attachment.eml", // using ISO-8859-7 (Greek) + "../../../data/13-HTML.eml", + "../../../data/14-HTML+attachment.eml", + "../../../data/15-HTML+embedded-image.eml", + "../../../data/16-plaintext+HMTL.eml", // text part is base64 encoded + "../../../data/17-plaintext+(HTML+embedded-image).eml", // HTML part is base64 encoded + "../../../data/18-plaintext+HTML+attachment.eml", + "../../../data/19-(HTML+embedded-image)+attachment.eml", + "../../../data/20-plaintext+(HTML+embedded-image)+attachment.eml", // using windows-1252 + + // Bodies with non-ASCII characters in UTF-8 and other charsets, all encoded with quoted printable. + "../../../data/21-plaintext.eml", + "../../../data/22-plaintext+attachment.eml", // using ISO-8859-7 (Greek) + "../../../data/23-HTML.eml", + "../../../data/24-HTML+attachment.eml", + "../../../data/25-HTML+embedded-image.eml", + "../../../data/26-plaintext+HMTL.eml", // text part is base64 encoded + "../../../data/27-plaintext+(HTML+embedded-image).eml", // HTML part is base64 encoded + "../../../data/28-plaintext+HTML+attachment.eml", + "../../../data/29-(HTML+embedded-image)+attachment.eml", + "../../../data/30-plaintext+(HTML+embedded-image)+attachment.eml", // using windows-1252 +]; + +function run_test() { + localAccountUtils.loadLocalMailAccount(); + + add_task(async function () { + await doMboxTest("test1", "../../../data/mbox_modern", 2); + await doMboxTest("test2", "../../../data/mbox_mboxrd", 2); + await doMboxTest("test3", "../../../data/mbox_unquoted", 2); + await roundTripTest(); + // Ideas for more tests: + // - check a really big mbox + // - check with really huge message (larger than one chunk) + // - check mbox with "From " line on chunk boundary + // - add tests for maildir->mbox conversion + // - check that conversions preserve message body (ie that the + // "From " line escaping scheme is reversible) + }); + + run_next_test(); +} + +/** + * Helper to create a server, account and inbox, and install an + * mbox file. + * + * @param {string} srvName - A unique server name to use for the test. + * @param {string} mboxFilename - mbox file to install and convert. + * @returns {nsIMsgIncomingServer} a server. + */ +function setupServer(srvName, mboxFilename) { + // {nsIMsgIncomingServer} pop server for the test. + let server = MailServices.accounts.createIncomingServer( + srvName, + "localhost", + "pop3" + ); + let account = MailServices.accounts.createAccount(); + account.incomingServer = server; + server.QueryInterface(Ci.nsIPop3IncomingServer); + server.valid = true; + + let inbox = account.incomingServer.rootFolder.getFolderWithFlags( + Ci.nsMsgFolderFlags.Inbox + ); + + // install the mbox file + let mboxFile = do_get_file(mboxFilename); + mboxFile.copyTo(inbox.filePath.parent, inbox.filePath.leafName); + + // TODO: is there some way to make folder rescan the mbox? + // We don't need it for this, but would be nice to do things properly. + return server; +} + +/** + * Perform an mbox->maildir conversion test. + * + * @param {string} srvName - A unique server name to use for the test. + * @param {string} mboxFilename - mbox file to install and convert. + * @param {number} expectCnt - Number of messages expected. + * @returns {nsIMsgIncomingServer} a server. + */ +async function doMboxTest(srvName, mboxFilename, expectCnt) { + // set up an account+server+inbox and copy in the test mbox file + let server = setupServer(srvName, mboxFilename); + + let mailstoreContractId = Services.prefs.getCharPref( + "mail.server." + server.key + ".storeContractID" + ); + + await convertMailStoreTo(mailstoreContractId, server, new EventTarget()); + + // Converted. Now find resulting Inbox/cur directory so + // we can count the messages there. + + let inbox = server.rootFolder.getFolderWithFlags(Ci.nsMsgFolderFlags.Inbox); + // NOTE: the conversion updates the path of the root folder, + // but _not_ the path of the inbox... + // Ideally, we'd just use inbox.filePath here, but + // instead we'll have compose the path manually. + + let curDir = server.rootFolder.filePath; + curDir.append(inbox.filePath.leafName); + curDir.append("cur"); + + // Sanity check. + Assert.ok(curDir.isDirectory(), "'cur' directory created"); + + // Check number of messages in Inbox/cur is what we expect. + let cnt = [...curDir.directoryEntries].length; + + Assert.equal( + cnt, + expectCnt, + "expected number of messages (" + mboxFilename + ")" + ); +} + +/** + * Create a temporary directory. The caller is responsible for deleting it. + * + * @param {string} prefix - Generated dir name will be of the form: + * "<prefix><random_sequence>". + * @returns {string} full path of new directory. + */ +async function tempDir(prefix) { + if (!prefix) { + prefix = ""; + } + let tmpDir = Services.dirsvc.get("TmpD", Ci.nsIFile).path; + while (true) { + let name = prefix + Math.floor(Math.random() * 0xffffffff).toString(16); + let fullPath = PathUtils.join(tmpDir, name); + try { + await IOUtils.makeDirectory(fullPath, { ignoreExisting: false }); + return fullPath; + } catch (e) { + // If directory already exists, try another name. Else bail out. + if ( + !(DOMException.isInstance(e) && e.name === "NoModificationAllowedError") + ) { + throw e; + } + } + } +} + +/** + * Test that messages survive unscathed in a roundtrip conversion, + * maildir -> mbox -> maildir. + * The final mailbox should have an identical set of files to the initial one, + * albeit with different filenames. + * Purely filesystem based. + * + * Would be nice to do a mbox->maildir->mbox roundtrip too, but that'd involve + * parsing the mbox files to compare them (can't just compare mbox files because + * message order and "From " lines can change). + */ +async function roundTripTest() { + // Set up initial maildir structure + let initialRoot = await tempDir("initial"); + + let inbox = PathUtils.join(initialRoot, "INBOX"); + await IOUtils.makeDirectory(inbox); + // Create a couple of subdirs under INBOX + let subdir = PathUtils.join(initialRoot, "INBOX.sbd"); + await IOUtils.makeDirectory(subdir); + let foodir = PathUtils.join(subdir, "foo"); + await IOUtils.makeDirectory(foodir); + let bardir = PathUtils.join(subdir, "bar"); + await IOUtils.makeDirectory(bardir); + + // Populate all the folders with some test emails. + const absolutePaths = testEmails.map(path => do_get_file(path).path); + await populateMaildir(inbox, absolutePaths); + await populateMaildir(foodir, absolutePaths); + await populateMaildir(bardir, absolutePaths); + + // Add a pick of "special" files, which should survive the trip verbatim. + for (let special of ["filterlog.html", "feeds.json", "rules.dat"]) { + let f = PathUtils.join(initialRoot, special); + await IOUtils.writeUTF8(f, f); // Use the filename for content. + } + + // Create root dirs for intermediate and final result. + let mboxRoot = await tempDir("mbox"); + let finalRoot = await tempDir("final"); + + // Convert: maildir -> mbox -> maildir + await doConvert("maildir", initialRoot, "mbox", mboxRoot); + await doConvert("mbox", mboxRoot, "maildir", finalRoot); + + // compare results - use checksums, because filenames will differ. + await recursiveMaildirCompare(initialRoot, finalRoot); +} + +/** + * Helper to adapt the callbacks from converterWorker into a promise. + * + * @param {string} srcType - type of source ("maildir", "mbox") + * @param {string} srcRoot - root directory containing the src folders. + * @param {string} destType - type of destination ("maildir", "mbox") + * @param {string} destRoot - root directory to place converted store. + * @returns {Promise} resolved when when conversion is complete. + */ +function doConvert(srcType, srcRoot, destType, destRoot) { + return new Promise(function (resolve, reject) { + let worker = new ChromeWorker("resource:///modules/converterWorker.js"); + worker.addEventListener("message", function (ev) { + if (ev.data.msg == "success") { + resolve(); + } + }); + worker.addEventListener("error", function (ev) { + reject(ev.message); + }); + // Go. + worker.postMessage({ + srcType, + destType, + srcRoot, + destRoot, + }); + }); +} + +/** + * Copy a list of email files (.eml) files into a maildir, creating "cur" + * and "tmp" subdirs if required. + * + * @param {string} maildir - Path to the maildir directory. + * @param {Array<string>} emailFiles - paths of source .eml files to copy. + */ +async function populateMaildir(maildir, emailFiles) { + let cur = PathUtils.join(maildir, "cur"); + await IOUtils.makeDirectory(cur); + await IOUtils.makeDirectory(PathUtils.join(maildir, "tmp")); + + // Normally maildir files would have a name derived from their msg-id field, + // but here we'll just use a timestamp-based one to save parsing them. + let ident = Date.now(); + for (let src of emailFiles) { + let dest = PathUtils.join(cur, ident.toString() + ".eml"); + ident += 1; + await IOUtils.copy(src, dest); + } +} + +/* + * List files in a directory (excludes subdirectories). + * + * @param {String} dirPath - Full path of directory. + * @returns {Array<String} full paths of the files. + */ +async function listFiles(dirPath) { + let files = []; + // Note: IOUtils has no dir iterator at time of writing. + for (const path of await IOUtils.getChildren(dirPath)) { + let fileInfo = await IOUtils.stat(path); + if (fileInfo.type !== "directory") { + files.push(path); + } + } + return files; +} + +/* + * Calculate md5 checksum for a file. + * + * @param {String} fileName - Full path to file. + * @returns {String} checksum of the file contents. + */ +async function md5Sum(fileName) { + let md5 = Cc["@mozilla.org/security/hash;1"].createInstance(Ci.nsICryptoHash); + md5.init(Ci.nsICryptoHash.MD5); + let raw = await IOUtils.read(fileName); + md5.update(raw, raw.byteLength); + return md5.finish(true); +} + +/** + * Compare all maildir directories in two directory trees. + * The comparison is per-maildir, by looking at the checksums of their emails. + * Asserts a test fail if any differences are found. + * + * @param {string} rootA - path to root of maildir store A. + * @param {string} rootB - path to root of maildir store B. + */ +async function recursiveMaildirCompare(rootA, rootB) { + let subdirs = []; + let maildirs = []; + let otherFiles = []; + for (let path of await IOUtils.getChildren(rootA)) { + let stat = await IOUtils.stat(path); + let name = PathUtils.filename(path); + if (stat.type === "directory") { + if (name.endsWith(".sbd")) { + subdirs.push(name); + } else { + // Assume all other dirs are maildirs. + maildirs.push(name); + } + } else { + otherFiles.push(name); + } + } + + // Compare the maildirs we found here. + let md5DirContents = async function (dirPath) { + let checksums = []; + for (let f of await listFiles(dirPath)) { + checksums.push(await md5Sum(f)); + } + return checksums; + }; + + for (let name of maildirs) { + let checksumsA = await md5DirContents(PathUtils.join(rootA, name, "cur")); + let checksumsB = await md5DirContents(PathUtils.join(rootB, name, "cur")); + + checksumsA.sort(); + checksumsB.sort(); + let match = checksumsA.length == checksumsB.length; + for (let i = 0; match && i < checksumsA.length; i++) { + match = checksumsA[i] == checksumsB[i]; + } + Assert.ok(match, "roundtrip preserves messages in maildir " + name); + } + + // Make sure any "special" files survived the trip intact. + for (let name of otherFiles) { + let checksumA = await md5Sum(PathUtils.join(rootA, name)); + let pathB = PathUtils.join(rootB, name); + let checksumB = (await IOUtils.exists(pathB)) ? await md5Sum(pathB) : null; + Assert.equal(checksumA, checksumB, "roundtrip preserves " + name); + } + + // Recurse down into .sbd dirs. + for (let name of subdirs) { + await recursiveMaildirCompare( + PathUtils.join(rootA, name), + PathUtils.join(rootB, name) + ); + } +} |