From 6bf0a5cb5034a7e684dcc3500e841785237ce2dd Mon Sep 17 00:00:00 2001
From: Daniel Baumann <daniel.baumann@progress-linux.org>
Date: Sun, 7 Apr 2024 19:32:43 +0200
Subject: Adding upstream version 1:115.7.0.

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
---
 .../base/test/unit/test_mailstoreConverter.js      | 376 +++++++++++++++++++++
 1 file changed, 376 insertions(+)
 create mode 100644 comm/mailnews/base/test/unit/test_mailstoreConverter.js

(limited to 'comm/mailnews/base/test/unit/test_mailstoreConverter.js')

diff --git a/comm/mailnews/base/test/unit/test_mailstoreConverter.js b/comm/mailnews/base/test/unit/test_mailstoreConverter.js
new file mode 100644
index 0000000000..f440b9c8cd
--- /dev/null
+++ b/comm/mailnews/base/test/unit/test_mailstoreConverter.js
@@ -0,0 +1,376 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+var { convertMailStoreTo } = ChromeUtils.import(
+  "resource:///modules/mailstoreConverter.jsm"
+);
+
+Services.prefs.setCharPref(
+  "mail.serverDefaultStoreContractID",
+  "@mozilla.org/msgstore/berkeleystore;1"
+);
+
+// Test data for round-trip test.
+let testEmails = [
+  // Base64 encoded bodies.
+  "../../../data/01-plaintext.eml",
+  "../../../data/02-plaintext+attachment.eml",
+  "../../../data/03-HTML.eml",
+  "../../../data/04-HTML+attachment.eml",
+  "../../../data/05-HTML+embedded-image.eml",
+  "../../../data/06-plaintext+HMTL.eml",
+  "../../../data/07-plaintext+(HTML+embedded-image).eml",
+  "../../../data/08-plaintext+HTML+attachment.eml",
+  "../../../data/09-(HTML+embedded-image)+attachment.eml",
+  "../../../data/10-plaintext+(HTML+embedded-image)+attachment.eml",
+
+  // Bodies with non-ASCII characters in UTF-8 and other charsets.
+  "../../../data/11-plaintext.eml",
+  "../../../data/12-plaintext+attachment.eml", // using ISO-8859-7 (Greek)
+  "../../../data/13-HTML.eml",
+  "../../../data/14-HTML+attachment.eml",
+  "../../../data/15-HTML+embedded-image.eml",
+  "../../../data/16-plaintext+HMTL.eml", // text part is base64 encoded
+  "../../../data/17-plaintext+(HTML+embedded-image).eml", // HTML part is base64 encoded
+  "../../../data/18-plaintext+HTML+attachment.eml",
+  "../../../data/19-(HTML+embedded-image)+attachment.eml",
+  "../../../data/20-plaintext+(HTML+embedded-image)+attachment.eml", // using windows-1252
+
+  // Bodies with non-ASCII characters in UTF-8 and other charsets, all encoded with quoted printable.
+  "../../../data/21-plaintext.eml",
+  "../../../data/22-plaintext+attachment.eml", // using ISO-8859-7 (Greek)
+  "../../../data/23-HTML.eml",
+  "../../../data/24-HTML+attachment.eml",
+  "../../../data/25-HTML+embedded-image.eml",
+  "../../../data/26-plaintext+HMTL.eml", // text part is base64 encoded
+  "../../../data/27-plaintext+(HTML+embedded-image).eml", // HTML part is base64 encoded
+  "../../../data/28-plaintext+HTML+attachment.eml",
+  "../../../data/29-(HTML+embedded-image)+attachment.eml",
+  "../../../data/30-plaintext+(HTML+embedded-image)+attachment.eml", // using windows-1252
+];
+
+function run_test() {
+  localAccountUtils.loadLocalMailAccount();
+
+  add_task(async function () {
+    await doMboxTest("test1", "../../../data/mbox_modern", 2);
+    await doMboxTest("test2", "../../../data/mbox_mboxrd", 2);
+    await doMboxTest("test3", "../../../data/mbox_unquoted", 2);
+    await roundTripTest();
+    // Ideas for more tests:
+    // - check a really big mbox
+    // - check with really huge message (larger than one chunk)
+    // - check mbox with "From " line on chunk boundary
+    // - add tests for maildir->mbox conversion
+    // - check that conversions preserve message body (ie that the
+    //   "From " line escaping scheme is reversible)
+  });
+
+  run_next_test();
+}
+
+/**
+ * Helper to create a server, account and inbox, and install an
+ * mbox file.
+ *
+ * @param {string} srvName - A unique server name to use for the test.
+ * @param {string} mboxFilename - mbox file to install and convert.
+ * @returns {nsIMsgIncomingServer} a server.
+ */
+function setupServer(srvName, mboxFilename) {
+  // {nsIMsgIncomingServer} pop server for the test.
+  let server = MailServices.accounts.createIncomingServer(
+    srvName,
+    "localhost",
+    "pop3"
+  );
+  let account = MailServices.accounts.createAccount();
+  account.incomingServer = server;
+  server.QueryInterface(Ci.nsIPop3IncomingServer);
+  server.valid = true;
+
+  let inbox = account.incomingServer.rootFolder.getFolderWithFlags(
+    Ci.nsMsgFolderFlags.Inbox
+  );
+
+  // install the mbox file
+  let mboxFile = do_get_file(mboxFilename);
+  mboxFile.copyTo(inbox.filePath.parent, inbox.filePath.leafName);
+
+  // TODO: is there some way to make folder rescan the mbox?
+  // We don't need it for this, but would be nice to do things properly.
+  return server;
+}
+
+/**
+ * Perform an mbox->maildir conversion test.
+ *
+ * @param {string} srvName - A unique server name to use for the test.
+ * @param {string} mboxFilename - mbox file to install and convert.
+ * @param {number} expectCnt - Number of messages expected.
+ * @returns {nsIMsgIncomingServer} a server.
+ */
+async function doMboxTest(srvName, mboxFilename, expectCnt) {
+  // set up an account+server+inbox and copy in the test mbox file
+  let server = setupServer(srvName, mboxFilename);
+
+  let mailstoreContractId = Services.prefs.getCharPref(
+    "mail.server." + server.key + ".storeContractID"
+  );
+
+  await convertMailStoreTo(mailstoreContractId, server, new EventTarget());
+
+  // Converted. Now find resulting Inbox/cur directory so
+  // we can count the messages there.
+
+  let inbox = server.rootFolder.getFolderWithFlags(Ci.nsMsgFolderFlags.Inbox);
+  // NOTE: the conversion updates the path of the root folder,
+  // but _not_ the path of the inbox...
+  // Ideally, we'd just use inbox.filePath here, but
+  // instead we'll have compose the path manually.
+
+  let curDir = server.rootFolder.filePath;
+  curDir.append(inbox.filePath.leafName);
+  curDir.append("cur");
+
+  // Sanity check.
+  Assert.ok(curDir.isDirectory(), "'cur' directory created");
+
+  // Check number of messages in Inbox/cur is what we expect.
+  let cnt = [...curDir.directoryEntries].length;
+
+  Assert.equal(
+    cnt,
+    expectCnt,
+    "expected number of messages (" + mboxFilename + ")"
+  );
+}
+
+/**
+ * Create a temporary directory. The caller is responsible for deleting it.
+ *
+ * @param {string} prefix - Generated dir name will be of the form:
+ *                          "<prefix><random_sequence>".
+ * @returns {string} full path of new directory.
+ */
+async function tempDir(prefix) {
+  if (!prefix) {
+    prefix = "";
+  }
+  let tmpDir = Services.dirsvc.get("TmpD", Ci.nsIFile).path;
+  while (true) {
+    let name = prefix + Math.floor(Math.random() * 0xffffffff).toString(16);
+    let fullPath = PathUtils.join(tmpDir, name);
+    try {
+      await IOUtils.makeDirectory(fullPath, { ignoreExisting: false });
+      return fullPath;
+    } catch (e) {
+      // If directory already exists, try another name. Else bail out.
+      if (
+        !(DOMException.isInstance(e) && e.name === "NoModificationAllowedError")
+      ) {
+        throw e;
+      }
+    }
+  }
+}
+
+/**
+ * Test that messages survive unscathed in a roundtrip conversion,
+ * maildir -> mbox -> maildir.
+ * The final mailbox should have an identical set of files to the initial one,
+ * albeit with different filenames.
+ * Purely filesystem based.
+ *
+ * Would be nice to do a mbox->maildir->mbox roundtrip too, but that'd involve
+ * parsing the mbox files to compare them (can't just compare mbox files because
+ * message order and "From " lines can change).
+ */
+async function roundTripTest() {
+  // Set up initial maildir structure
+  let initialRoot = await tempDir("initial");
+
+  let inbox = PathUtils.join(initialRoot, "INBOX");
+  await IOUtils.makeDirectory(inbox);
+  // Create a couple of subdirs under INBOX
+  let subdir = PathUtils.join(initialRoot, "INBOX.sbd");
+  await IOUtils.makeDirectory(subdir);
+  let foodir = PathUtils.join(subdir, "foo");
+  await IOUtils.makeDirectory(foodir);
+  let bardir = PathUtils.join(subdir, "bar");
+  await IOUtils.makeDirectory(bardir);
+
+  // Populate all the folders with some test emails.
+  const absolutePaths = testEmails.map(path => do_get_file(path).path);
+  await populateMaildir(inbox, absolutePaths);
+  await populateMaildir(foodir, absolutePaths);
+  await populateMaildir(bardir, absolutePaths);
+
+  // Add a pick of "special" files, which should survive the trip verbatim.
+  for (let special of ["filterlog.html", "feeds.json", "rules.dat"]) {
+    let f = PathUtils.join(initialRoot, special);
+    await IOUtils.writeUTF8(f, f); // Use the filename for content.
+  }
+
+  // Create root dirs for intermediate and final result.
+  let mboxRoot = await tempDir("mbox");
+  let finalRoot = await tempDir("final");
+
+  // Convert: maildir -> mbox -> maildir
+  await doConvert("maildir", initialRoot, "mbox", mboxRoot);
+  await doConvert("mbox", mboxRoot, "maildir", finalRoot);
+
+  // compare results - use checksums, because filenames will differ.
+  await recursiveMaildirCompare(initialRoot, finalRoot);
+}
+
+/**
+ * Helper to adapt the callbacks from converterWorker into a promise.
+ *
+ * @param {string} srcType - type of source ("maildir", "mbox")
+ * @param {string} srcRoot - root directory containing the src folders.
+ * @param {string} destType - type of destination ("maildir", "mbox")
+ * @param {string} destRoot - root directory to place converted store.
+ * @returns {Promise} resolved when when conversion is complete.
+ */
+function doConvert(srcType, srcRoot, destType, destRoot) {
+  return new Promise(function (resolve, reject) {
+    let worker = new ChromeWorker("resource:///modules/converterWorker.js");
+    worker.addEventListener("message", function (ev) {
+      if (ev.data.msg == "success") {
+        resolve();
+      }
+    });
+    worker.addEventListener("error", function (ev) {
+      reject(ev.message);
+    });
+    // Go.
+    worker.postMessage({
+      srcType,
+      destType,
+      srcRoot,
+      destRoot,
+    });
+  });
+}
+
+/**
+ * Copy a list of email files (.eml) files into a maildir, creating "cur"
+ * and "tmp" subdirs if required.
+ *
+ * @param {string} maildir - Path to the maildir directory.
+ * @param {Array<string>} emailFiles - paths of source .eml files to copy.
+ */
+async function populateMaildir(maildir, emailFiles) {
+  let cur = PathUtils.join(maildir, "cur");
+  await IOUtils.makeDirectory(cur);
+  await IOUtils.makeDirectory(PathUtils.join(maildir, "tmp"));
+
+  // Normally maildir files would have a name derived from their msg-id field,
+  // but here we'll just use a timestamp-based one to save parsing them.
+  let ident = Date.now();
+  for (let src of emailFiles) {
+    let dest = PathUtils.join(cur, ident.toString() + ".eml");
+    ident += 1;
+    await IOUtils.copy(src, dest);
+  }
+}
+
+/*
+ * List files in a directory (excludes subdirectories).
+ *
+ * @param {String} dirPath - Full path of directory.
+ * @returns {Array<String} full paths of the files.
+ */
+async function listFiles(dirPath) {
+  let files = [];
+  // Note: IOUtils has no dir iterator at time of writing.
+  for (const path of await IOUtils.getChildren(dirPath)) {
+    let fileInfo = await IOUtils.stat(path);
+    if (fileInfo.type !== "directory") {
+      files.push(path);
+    }
+  }
+  return files;
+}
+
+/*
+ * Calculate md5 checksum for a file.
+ *
+ * @param {String} fileName - Full path to file.
+ * @returns {String} checksum of the file contents.
+ */
+async function md5Sum(fileName) {
+  let md5 = Cc["@mozilla.org/security/hash;1"].createInstance(Ci.nsICryptoHash);
+  md5.init(Ci.nsICryptoHash.MD5);
+  let raw = await IOUtils.read(fileName);
+  md5.update(raw, raw.byteLength);
+  return md5.finish(true);
+}
+
+/**
+ * Compare all maildir directories in two directory trees.
+ * The comparison is per-maildir, by looking at the checksums of their emails.
+ * Asserts a test fail if any differences are found.
+ *
+ * @param {string} rootA - path to root of maildir store A.
+ * @param {string} rootB - path to root of maildir store B.
+ */
+async function recursiveMaildirCompare(rootA, rootB) {
+  let subdirs = [];
+  let maildirs = [];
+  let otherFiles = [];
+  for (let path of await IOUtils.getChildren(rootA)) {
+    let stat = await IOUtils.stat(path);
+    let name = PathUtils.filename(path);
+    if (stat.type === "directory") {
+      if (name.endsWith(".sbd")) {
+        subdirs.push(name);
+      } else {
+        // Assume all other dirs are maildirs.
+        maildirs.push(name);
+      }
+    } else {
+      otherFiles.push(name);
+    }
+  }
+
+  // Compare the maildirs we found here.
+  let md5DirContents = async function (dirPath) {
+    let checksums = [];
+    for (let f of await listFiles(dirPath)) {
+      checksums.push(await md5Sum(f));
+    }
+    return checksums;
+  };
+
+  for (let name of maildirs) {
+    let checksumsA = await md5DirContents(PathUtils.join(rootA, name, "cur"));
+    let checksumsB = await md5DirContents(PathUtils.join(rootB, name, "cur"));
+
+    checksumsA.sort();
+    checksumsB.sort();
+    let match = checksumsA.length == checksumsB.length;
+    for (let i = 0; match && i < checksumsA.length; i++) {
+      match = checksumsA[i] == checksumsB[i];
+    }
+    Assert.ok(match, "roundtrip preserves messages in maildir " + name);
+  }
+
+  // Make sure any "special" files survived the trip intact.
+  for (let name of otherFiles) {
+    let checksumA = await md5Sum(PathUtils.join(rootA, name));
+    let pathB = PathUtils.join(rootB, name);
+    let checksumB = (await IOUtils.exists(pathB)) ? await md5Sum(pathB) : null;
+    Assert.equal(checksumA, checksumB, "roundtrip preserves " + name);
+  }
+
+  // Recurse down into .sbd dirs.
+  for (let name of subdirs) {
+    await recursiveMaildirCompare(
+      PathUtils.join(rootA, name),
+      PathUtils.join(rootB, name)
+    );
+  }
+}
-- 
cgit v1.2.3