Adding upstream version 7.0.14-dfsg.upstream/7.0.14-dfsg

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

1 files changed, 1280 insertions, 0 deletions

diff --git a/src/libs/xpcom18a4/xpcom/io/nsNativeCharsetUtils.cpp b/src/libs/xpcom18a4/xpcom/io/nsNativeCharsetUtils.cpp
new file mode 100644
index 00000000..55b5c63c
--- /dev/null
+++ b/src/libs/xpcom18a4/xpcom/io/nsNativeCharsetUtils.cpp
@@ -0,0 +1,1280 @@
+/* ***** BEGIN LICENSE BLOCK *****
+ * Version: MPL 1.1/GPL 2.0/LGPL 2.1
+ *
+ * The contents of this file are subject to the Mozilla Public License Version
+ * 1.1 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * http://www.mozilla.org/MPL/
+ *
+ * Software distributed under the License is distributed on an "AS IS" basis,
+ * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
+ * for the specific language governing rights and limitations under the
+ * License.
+ *
+ * The Original Code is Mozilla.
+ *
+ * The Initial Developer of the Original Code is
+ * Netscape Communications Corporation.
+ * Portions created by the Initial Developer are Copyright (C) 2002
+ * the Initial Developer. All Rights Reserved.
+ *
+ * Contributor(s):
+ *   Darin Fisher <darin@netscape.com>
+ *   Brian Stell <bstell@ix.netcom.com>
+ *   Frank Tang <ftang@netscape.com>
+ *   Brendan Eich <brendan@mozilla.org>
+ *   Sergei Dolgov <sergei_d@fi.fi.tartu.ee>
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * either the GNU General Public License Version 2 or later (the "GPL"), or
+ * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
+ * in which case the provisions of the GPL or the LGPL are applicable instead
+ * of those above. If you wish to allow use of your version of this file only
+ * under the terms of either the GPL or the LGPL, and not to allow others to
+ * use your version of this file under the terms of the MPL, indicate your
+ * decision by deleting the provisions above and replace them with the notice
+ * and other provisions required by the GPL or the LGPL. If you do not delete
+ * the provisions above, a recipient may use your version of this file under
+ * the terms of any one of the MPL, the GPL or the LGPL.
+ *
+ * ***** END LICENSE BLOCK ***** */
+
+#include "xpcom-private.h"
+
+//-----------------------------------------------------------------------------
+// XP_UNIX
+//-----------------------------------------------------------------------------
+#if defined(XP_UNIX)
+
+#include <stdlib.h>   // mbtowc, wctomb
+#include <locale.h>   // setlocale
+#include "nscore.h"
+#include "prlock.h"
+#include "nsAString.h"
+#include "nsReadableUtils.h"
+
+//
+// choose a conversion library.  we used to use mbrtowc/wcrtomb under Linux,
+// but that doesn't work for non-BMP characters whether we use '-fshort-wchar'
+// or not (see bug 206811 and 
+// news://news.mozilla.org:119/bajml3$fvr1@ripley.netscape.com). we now use
+// iconv for all platforms where nltypes.h and nllanginfo.h are present 
+// along with iconv.
+//
+#if defined(HAVE_ICONV) && defined(HAVE_NL_TYPES_H) && defined(HAVE_LANGINFO_CODESET)
+#define USE_ICONV 1
+#else
+#define USE_STDCONV 1
+#endif
+
+static void
+isolatin1_to_utf16(const char **input, PRUint32 *inputLeft, PRUnichar **output, PRUint32 *outputLeft)
+{
+    while (*inputLeft && *outputLeft) {
+        **output = (unsigned char) **input;
+        (*input)++;
+        (*inputLeft)--;
+        (*output)++;
+        (*outputLeft)--;
+    }
+}
+
+static void
+utf16_to_isolatin1(const PRUnichar **input, PRUint32 *inputLeft, char **output, PRUint32 *outputLeft)
+{
+    while (*inputLeft && *outputLeft) {
+        **output = (unsigned char) **input;
+        (*input)++;
+        (*inputLeft)--;
+        (*output)++;
+        (*outputLeft)--;
+    }
+}
+
+//-----------------------------------------------------------------------------
+// conversion using iconv
+//-----------------------------------------------------------------------------
+#if defined(USE_ICONV)
+#include <nl_types.h> // CODESET
+#include <langinfo.h> // nl_langinfo
+#include <iconv.h>    // iconv_open, iconv, iconv_close
+#include <errno.h>
+
+#if defined(HAVE_ICONV_WITH_CONST_INPUT)
+#define ICONV_INPUT(x) (x)
+#else
+#define ICONV_INPUT(x) ((char **)x)
+#endif
+
+// solaris definitely needs this, but we'll enable it by default
+// just in case... but we know for sure that iconv(3) in glibc
+// doesn't need this.
+#if !defined(__GLIBC__)
+#define ENABLE_UTF8_FALLBACK_SUPPORT
+#endif
+
+#define INVALID_ICONV_T ((iconv_t) -1)
+
+static inline size_t
+xp_iconv(iconv_t converter,
+         const char **input,
+         size_t      *inputLeft,
+         char       **output,
+         size_t      *outputLeft)
+{
+    size_t res, outputAvail = outputLeft ? *outputLeft : 0;
+    res = iconv(converter, ICONV_INPUT(input), inputLeft, output, outputLeft);
+    if (res == (size_t) -1) {
+        // on some platforms (e.g., linux) iconv will fail with
+        // E2BIG if it cannot convert _all_ of its input.  it'll
+        // still adjust all of the in/out params correctly, so we
+        // can ignore this error.  the assumption is that we will
+        // be called again to complete the conversion.
+        if ((errno == E2BIG) && (*outputLeft < outputAvail))
+            res = 0;
+    }
+    return res;
+}
+
+static inline void
+xp_iconv_reset(iconv_t converter)
+{
+    // NOTE: the man pages on Solaris claim that you can pass NULL
+    // for all parameter to reset the converter, but beware the 
+    // evil Solaris crash if you go down this route >:-)
+    
+    const char *zero_char_in_ptr  = NULL;
+    char       *zero_char_out_ptr = NULL;
+    size_t      zero_size_in      = 0,
+                zero_size_out     = 0;
+
+    xp_iconv(converter, &zero_char_in_ptr,
+                        &zero_size_in,
+                        &zero_char_out_ptr,
+                        &zero_size_out);
+}
+
+static inline iconv_t
+xp_iconv_open(const char **to_list, const char **from_list)
+{
+    iconv_t res;
+    const char **from_name;
+    const char **to_name;
+
+    // try all possible combinations to locate a converter.
+    to_name = to_list;
+    while (*to_name) {
+        if (**to_name) {
+            from_name = from_list;
+            while (*from_name) {
+                if (**from_name) {
+                    res = iconv_open(*to_name, *from_name);
+                    if (res != INVALID_ICONV_T)
+                        return res;
+                }
+                from_name++;
+            }
+        }
+        to_name++;
+    }
+
+    return INVALID_ICONV_T;
+}
+
+/* 
+ * PRUnichar[] is NOT a UCS-2 array BUT a UTF-16 string. Therefore, we
+ * have to use UTF-16 with iconv(3) on platforms where it's supported.
+ * However, the way UTF-16 and UCS-2 are interpreted varies across platforms 
+ * and implementations of iconv(3). On Tru64, it also depends on the environment
+ * variable. To avoid the trouble arising from byte-swapping 
+ * (bug 208809), we have to try UTF-16LE/BE and UCS-2LE/BE before falling 
+ * back to UTF-16 and UCS-2 and variants. We assume that UTF-16 and UCS-2 
+ * on systems without UTF-16LE/BE and UCS-2LE/BE have the native endianness,
+ * which isn't the case of glibc 2.1.x, for which we use 'UNICODELITTLE'
+ * and 'UNICODEBIG'. It's also not true of Tru64 V4 when the environment
+ * variable ICONV_BYTEORDER is set to 'big-endian', about which not much 
+ * can be done other than adding a note in the release notes. (bug 206811)
+ */
+static const char *UTF_16_NAMES[] = {
+#if defined(IS_LITTLE_ENDIAN)
+    "UTF-16LE",
+#if defined(__GLIBC__)
+    "UNICODELITTLE",
+#endif
+    "UCS-2LE",
+#else
+    "UTF-16BE",
+#if defined(__GLIBC__)
+    "UNICODEBIG",
+#endif
+    "UCS-2BE",
+#endif
+    "UTF-16",
+    "UCS-2",
+    "UCS2",
+    "UCS_2",
+    "ucs-2",
+    "ucs2",
+    "ucs_2",
+    NULL
+};
+
+#if defined(ENABLE_UTF8_FALLBACK_SUPPORT)
+static const char *UTF_8_NAMES[] = {
+    "UTF-8",
+    "UTF8",
+    "UTF_8",
+    "utf-8",
+    "utf8",
+    "utf_8",
+    NULL
+};
+#endif
+
+static const char *ISO_8859_1_NAMES[] = {
+    "ISO-8859-1",
+#if !defined(__GLIBC__)
+    "ISO8859-1",
+    "ISO88591",
+    "ISO_8859_1",
+    "ISO8859_1",
+    "iso-8859-1",
+    "iso8859-1",
+    "iso88591",
+    "iso_8859_1",
+    "iso8859_1",
+#endif
+    NULL
+};
+
+class nsNativeCharsetConverter
+{
+public:
+    nsNativeCharsetConverter();
+   ~nsNativeCharsetConverter();
+
+    nsresult NativeToUnicode(const char      **input , PRUint32 *inputLeft,
+                             PRUnichar       **output, PRUint32 *outputLeft);
+    nsresult UnicodeToNative(const PRUnichar **input , PRUint32 *inputLeft,
+                             char            **output, PRUint32 *outputLeft);
+
+    static void GlobalInit();
+    static void GlobalShutdown();
+
+private:
+    static iconv_t gNativeToUnicode;
+    static iconv_t gUnicodeToNative;
+#if defined(ENABLE_UTF8_FALLBACK_SUPPORT)
+    static iconv_t gNativeToUTF8;
+    static iconv_t gUTF8ToNative;
+    static iconv_t gUnicodeToUTF8;
+    static iconv_t gUTF8ToUnicode;
+#endif
+    static PRLock *gLock;
+    static PRBool  gInitialized;
+
+    static void LazyInit();
+
+    static void Lock()   { if (gLock) PR_Lock(gLock);   }
+    static void Unlock() { if (gLock) PR_Unlock(gLock); }
+};
+
+iconv_t nsNativeCharsetConverter::gNativeToUnicode = INVALID_ICONV_T;
+iconv_t nsNativeCharsetConverter::gUnicodeToNative = INVALID_ICONV_T;
+#if defined(ENABLE_UTF8_FALLBACK_SUPPORT)
+iconv_t nsNativeCharsetConverter::gNativeToUTF8    = INVALID_ICONV_T;
+iconv_t nsNativeCharsetConverter::gUTF8ToNative    = INVALID_ICONV_T;
+iconv_t nsNativeCharsetConverter::gUnicodeToUTF8   = INVALID_ICONV_T;
+iconv_t nsNativeCharsetConverter::gUTF8ToUnicode   = INVALID_ICONV_T;
+#endif
+PRLock *nsNativeCharsetConverter::gLock            = nsnull;
+PRBool  nsNativeCharsetConverter::gInitialized     = PR_FALSE;
+
+void
+nsNativeCharsetConverter::LazyInit()
+{
+    const char  *blank_list[] = { "", NULL };
+    const char **native_charset_list = blank_list;
+    const char  *native_charset = nl_langinfo(CODESET);
+    if (native_charset == nsnull) {
+        NS_ERROR("native charset is unknown");
+        // fallback to ISO-8859-1
+        native_charset_list = ISO_8859_1_NAMES;
+    }
+    else
+        native_charset_list[0] = native_charset;
+
+    gNativeToUnicode = xp_iconv_open(UTF_16_NAMES, native_charset_list);
+    gUnicodeToNative = xp_iconv_open(native_charset_list, UTF_16_NAMES);
+
+#if defined(ENABLE_UTF8_FALLBACK_SUPPORT)
+    if (gNativeToUnicode == INVALID_ICONV_T) {
+        gNativeToUTF8 = xp_iconv_open(UTF_8_NAMES, native_charset_list);
+        gUTF8ToUnicode = xp_iconv_open(UTF_16_NAMES, UTF_8_NAMES);
+        NS_ASSERTION(gNativeToUTF8 != INVALID_ICONV_T, "no native to utf-8 converter");
+        NS_ASSERTION(gUTF8ToUnicode != INVALID_ICONV_T, "no utf-8 to utf-16 converter");
+    }
+    if (gUnicodeToNative == INVALID_ICONV_T) {
+        gUnicodeToUTF8 = xp_iconv_open(UTF_8_NAMES, UTF_16_NAMES);
+        gUTF8ToNative = xp_iconv_open(native_charset_list, UTF_8_NAMES);
+        NS_ASSERTION(gUnicodeToUTF8 != INVALID_ICONV_T, "no utf-16 to utf-8 converter");
+        NS_ASSERTION(gUTF8ToNative != INVALID_ICONV_T, "no utf-8 to native converter");
+    }
+#else
+    NS_ASSERTION(gNativeToUnicode != INVALID_ICONV_T, "no native to utf-16 converter");
+    NS_ASSERTION(gUnicodeToNative != INVALID_ICONV_T, "no utf-16 to native converter");
+#endif
+
+    /*
+     * On Solaris 8 (and newer?), the iconv modules converting to UCS-2
+     * prepend a byte order mark unicode character (BOM, u+FEFF) during
+     * the first use of the iconv converter. The same is the case of 
+     * glibc 2.2.9x and Tru64 V5 (see bug 208809) when 'UTF-16' is used. 
+     * However, we use 'UTF-16LE/BE' in both cases, instead so that we 
+     * should be safe. But just in case...
+     *
+     * This dummy conversion gets rid of the BOMs and fixes bug 153562.
+     */
+    char dummy_input[1] = { ' ' };
+    char dummy_output[4];
+
+    if (gNativeToUnicode != INVALID_ICONV_T) {
+	const char *input = dummy_input;
+	size_t input_left = sizeof(dummy_input);
+	char *output = dummy_output;
+	size_t output_left = sizeof(dummy_output);
+
+	xp_iconv(gNativeToUnicode, &input, &input_left, &output, &output_left);
+    }
+#if defined(ENABLE_UTF8_FALLBACK_SUPPORT)
+    if (gUTF8ToUnicode != INVALID_ICONV_T) {
+	const char *input = dummy_input;
+	size_t input_left = sizeof(dummy_input);
+	char *output = dummy_output;
+	size_t output_left = sizeof(dummy_output);
+
+	xp_iconv(gUTF8ToUnicode, &input, &input_left, &output, &output_left);
+    }
+#endif
+
+    gInitialized = PR_TRUE;
+}
+
+void
+nsNativeCharsetConverter::GlobalInit()
+{
+    gLock = PR_NewLock();
+    NS_ASSERTION(gLock, "lock creation failed");
+}
+
+void
+nsNativeCharsetConverter::GlobalShutdown()
+{
+    if (gLock) {
+        PR_DestroyLock(gLock);
+        gLock = nsnull;
+    }
+
+    if (gNativeToUnicode != INVALID_ICONV_T) {
+        iconv_close(gNativeToUnicode);
+        gNativeToUnicode = INVALID_ICONV_T;
+    }
+
+    if (gUnicodeToNative != INVALID_ICONV_T) {
+        iconv_close(gUnicodeToNative);
+        gUnicodeToNative = INVALID_ICONV_T;
+    }
+
+#if defined(ENABLE_UTF8_FALLBACK_SUPPORT)
+    if (gNativeToUTF8 != INVALID_ICONV_T) {
+        iconv_close(gNativeToUTF8);
+        gNativeToUTF8 = INVALID_ICONV_T;
+    }
+    if (gUTF8ToNative != INVALID_ICONV_T) {
+        iconv_close(gUTF8ToNative);
+        gUTF8ToNative = INVALID_ICONV_T;
+    }
+    if (gUnicodeToUTF8 != INVALID_ICONV_T) {
+        iconv_close(gUnicodeToUTF8);
+        gUnicodeToUTF8 = INVALID_ICONV_T;
+    }
+    if (gUTF8ToUnicode != INVALID_ICONV_T) {
+        iconv_close(gUTF8ToUnicode);
+        gUTF8ToUnicode = INVALID_ICONV_T;
+    }
+#endif
+
+    gInitialized = PR_FALSE;
+}
+
+nsNativeCharsetConverter::nsNativeCharsetConverter()
+{
+    Lock();
+    if (!gInitialized)
+        LazyInit();
+}
+
+nsNativeCharsetConverter::~nsNativeCharsetConverter()
+{
+    // reset converters for next time
+    if (gNativeToUnicode != INVALID_ICONV_T)
+        xp_iconv_reset(gNativeToUnicode);
+    if (gUnicodeToNative != INVALID_ICONV_T)
+        xp_iconv_reset(gUnicodeToNative);
+#if defined(ENABLE_UTF8_FALLBACK_SUPPORT)
+    if (gNativeToUTF8 != INVALID_ICONV_T)
+        xp_iconv_reset(gNativeToUTF8);
+    if (gUTF8ToNative != INVALID_ICONV_T)
+        xp_iconv_reset(gUTF8ToNative);
+    if (gUnicodeToUTF8 != INVALID_ICONV_T)
+        xp_iconv_reset(gUnicodeToUTF8);
+    if (gUTF8ToUnicode != INVALID_ICONV_T)
+        xp_iconv_reset(gUTF8ToUnicode);
+#endif
+    Unlock();
+}
+
+nsresult
+nsNativeCharsetConverter::NativeToUnicode(const char **input,
+                                          PRUint32    *inputLeft,
+                                          PRUnichar  **output,
+                                          PRUint32    *outputLeft)
+{
+    size_t res = 0;
+    size_t inLeft = (size_t) *inputLeft;
+    size_t outLeft = (size_t) *outputLeft * 2;
+
+    if (gNativeToUnicode != INVALID_ICONV_T) {
+
+        res = xp_iconv(gNativeToUnicode, input, &inLeft, (char **) output, &outLeft);
+
+        *inputLeft = inLeft;
+        *outputLeft = outLeft / 2;
+        if (res != (size_t) -1) 
+            return NS_OK;
+
+        NS_WARNING("conversion from native to utf-16 failed");
+
+        // reset converter
+        xp_iconv_reset(gNativeToUnicode);
+    }
+#if defined(ENABLE_UTF8_FALLBACK_SUPPORT)
+    else if ((gNativeToUTF8 != INVALID_ICONV_T) &&
+             (gUTF8ToUnicode != INVALID_ICONV_T)) {
+        // convert first to UTF8, then from UTF8 to UCS2
+        const char *in = *input;
+
+        char ubuf[1024];
+
+        // we assume we're always called with enough space in |output|,
+        // so convert many chars at a time...
+        while (inLeft) {
+            char *p = ubuf;
+            size_t n = sizeof(ubuf);
+            res = xp_iconv(gNativeToUTF8, &in, &inLeft, &p, &n);
+            if (res == (size_t) -1) {
+                NS_ERROR("conversion from native to utf-8 failed");
+                break;
+            }
+            NS_ASSERTION(outLeft > 0, "bad assumption");
+            p = ubuf;
+            n = sizeof(ubuf) - n;
+            res = xp_iconv(gUTF8ToUnicode, (const char **) &p, &n, (char **) output, &outLeft);
+            if (res == (size_t) -1) {
+                NS_ERROR("conversion from utf-8 to utf-16 failed");
+                break;
+            }
+        }
+
+        (*input) += (*inputLeft - inLeft);
+        *inputLeft = inLeft;
+        *outputLeft = outLeft / 2;
+
+        if (res != (size_t) -1) 
+            return NS_OK;
+
+        // reset converters
+        xp_iconv_reset(gNativeToUTF8);
+        xp_iconv_reset(gUTF8ToUnicode);
+    }
+#endif
+
+    // fallback: zero-pad and hope for the best
+    // XXX This is lame and we have to do better.
+    isolatin1_to_utf16(input, inputLeft, output, outputLeft);
+
+    return NS_OK;
+}
+
+nsresult
+nsNativeCharsetConverter::UnicodeToNative(const PRUnichar **input,
+                                          PRUint32         *inputLeft,
+                                          char            **output,
+                                          PRUint32         *outputLeft)
+{
+    size_t res = 0;
+    size_t inLeft = (size_t) *inputLeft * 2;
+    size_t outLeft = (size_t) *outputLeft;
+
+    if (gUnicodeToNative != INVALID_ICONV_T) {
+        res = xp_iconv(gUnicodeToNative, (const char **) input, &inLeft, output, &outLeft);
+
+        if (res != (size_t) -1) {
+            *inputLeft = inLeft / 2;
+            *outputLeft = outLeft;
+            return NS_OK;
+        }
+
+        NS_ERROR("iconv failed");
+
+        // reset converter
+        xp_iconv_reset(gUnicodeToNative);
+    }
+#if defined(ENABLE_UTF8_FALLBACK_SUPPORT)
+    else if ((gUnicodeToUTF8 != INVALID_ICONV_T) &&
+             (gUTF8ToNative != INVALID_ICONV_T)) {
+        const char *in = (const char *) *input;
+
+        char ubuf[6]; // max utf-8 char length (really only needs to be 4 bytes)
+
+        // convert one uchar at a time...
+        while (inLeft && outLeft) {
+            char *p = ubuf;
+            size_t n = sizeof(ubuf), one_uchar = sizeof(PRUnichar);
+            res = xp_iconv(gUnicodeToUTF8, &in, &one_uchar, &p, &n);
+            if (res == (size_t) -1) {
+                NS_ERROR("conversion from utf-16 to utf-8 failed");
+                break;
+            }
+            p = ubuf;
+            n = sizeof(ubuf) - n;
+            res = xp_iconv(gUTF8ToNative, (const char **) &p, &n, output, &outLeft);
+            if (res == (size_t) -1) {
+                if (errno == E2BIG) {
+                    // not enough room for last uchar... back up and return.
+                    in -= sizeof(PRUnichar);
+                    res = 0;
+                }
+                else
+                    NS_ERROR("conversion from utf-8 to native failed");
+                break;
+            }
+            inLeft -= sizeof(PRUnichar);
+        }
+
+        if (res != (size_t) -1) {
+            (*input) += (*inputLeft - inLeft/2);
+            *inputLeft = inLeft/2;
+            *outputLeft = outLeft;
+            return NS_OK;
+        }
+
+        // reset converters
+        xp_iconv_reset(gUnicodeToUTF8);
+        xp_iconv_reset(gUTF8ToNative);
+    }
+#endif
+
+    // fallback: truncate and hope for the best
+    utf16_to_isolatin1(input, inputLeft, output, outputLeft);
+
+    return NS_OK;
+}
+
+#endif // USE_ICONV
+
+//-----------------------------------------------------------------------------
+// conversion using mb[r]towc/wc[r]tomb
+//-----------------------------------------------------------------------------
+#if defined(USE_STDCONV)
+#if defined(HAVE_WCRTOMB) || defined(HAVE_MBRTOWC)
+#include <wchar.h>    // mbrtowc, wcrtomb
+#endif
+
+class nsNativeCharsetConverter
+{
+public:
+    nsNativeCharsetConverter();
+
+    nsresult NativeToUnicode(const char      **input , PRUint32 *inputLeft,
+                             PRUnichar       **output, PRUint32 *outputLeft);
+    nsresult UnicodeToNative(const PRUnichar **input , PRUint32 *inputLeft,
+                             char            **output, PRUint32 *outputLeft);
+
+    static void GlobalInit();
+    static void GlobalShutdown() { }
+
+private:
+    static PRBool gWCharIsUnicode;
+
+#if defined(HAVE_WCRTOMB) || defined(HAVE_MBRTOWC)
+    mbstate_t ps;
+#endif
+};
+
+PRBool nsNativeCharsetConverter::gWCharIsUnicode = PR_FALSE;
+
+nsNativeCharsetConverter::nsNativeCharsetConverter()
+{
+#if defined(HAVE_WCRTOMB) || defined(HAVE_MBRTOWC)
+    memset(&ps, 0, sizeof(ps));
+#endif
+}
+
+void
+nsNativeCharsetConverter::GlobalInit()
+{
+    // verify that wchar_t for the current locale is actually unicode.
+    // if it is not, then we should avoid calling mbtowc/wctomb and
+    // just fallback on zero-pad/truncation conversion.
+    //
+    // this test cannot be done at build time because the encoding of
+    // wchar_t may depend on the runtime locale.  sad, but true!!
+    //
+    // so, if wchar_t is unicode then converting an ASCII character
+    // to wchar_t should not change its numeric value.  we'll just
+    // check what happens with the ASCII 'a' character.
+    //
+    // this test is not perfect... obviously, it could yield false
+    // positives, but then at least ASCII text would be converted
+    // properly (or maybe just the 'a' character) -- oh well :(
+
+    char a = 'a';
+    unsigned int w = 0;
+
+    int res = mbtowc((wchar_t *) &w, &a, 1);
+
+    gWCharIsUnicode = (res != -1 && w == 'a');
+
+#ifdef DEBUG
+    if (!gWCharIsUnicode)
+        NS_WARNING("wchar_t is not unicode (unicode conversion will be lossy)");
+#endif
+}
+
+nsresult
+nsNativeCharsetConverter::NativeToUnicode(const char **input,
+                                          PRUint32    *inputLeft,
+                                          PRUnichar  **output,
+                                          PRUint32    *outputLeft)
+{
+    if (gWCharIsUnicode) {
+        int incr;
+
+        // cannot use wchar_t here since it may have been redefined (e.g.,
+        // via -fshort-wchar).  hopefully, sizeof(tmp) is sufficient XP.
+        unsigned int tmp = 0;
+        while (*inputLeft && *outputLeft) {
+#ifdef HAVE_MBRTOWC
+            incr = (int) mbrtowc((wchar_t *) &tmp, *input, *inputLeft, &ps);
+#else
+            // XXX is this thread-safe?
+            incr = (int) mbtowc((wchar_t *) &tmp, *input, *inputLeft);
+#endif
+            if (incr < 0) {
+                NS_WARNING("mbtowc failed: possible charset mismatch");
+                // zero-pad and hope for the best
+                tmp = (unsigned char) **input;
+                incr = 1;
+            }
+            **output = (PRUnichar) tmp;
+            (*input) += incr;
+            (*inputLeft) -= incr;
+            (*output)++;
+            (*outputLeft)--;
+        }
+    }
+    else {
+        // wchar_t isn't unicode, so the best we can do is treat the
+        // input as if it is isolatin1 :(
+        isolatin1_to_utf16(input, inputLeft, output, outputLeft);
+    }
+
+    return NS_OK;
+}
+
+nsresult
+nsNativeCharsetConverter::UnicodeToNative(const PRUnichar **input,
+                                          PRUint32         *inputLeft,
+                                          char            **output,
+                                          PRUint32         *outputLeft)
+{
+    if (gWCharIsUnicode) {
+        int incr;
+
+        while (*inputLeft && *outputLeft >= MB_CUR_MAX) {
+#ifdef HAVE_WCRTOMB
+            incr = (int) wcrtomb(*output, (wchar_t) **input, &ps);
+#else
+            // XXX is this thread-safe?
+            incr = (int) wctomb(*output, (wchar_t) **input);
+#endif
+            if (incr < 0) {
+                NS_WARNING("mbtowc failed: possible charset mismatch");
+                **output = (unsigned char) **input; // truncate
+                incr = 1;
+            }
+            // most likely we're dead anyways if this assertion should fire
+            NS_ASSERTION(PRUint32(incr) <= *outputLeft, "wrote beyond end of string");
+            (*output) += incr;
+            (*outputLeft) -= incr;
+            (*input)++;
+            (*inputLeft)--;
+        }
+    }
+    else {
+        // wchar_t isn't unicode, so the best we can do is treat the
+        // input as if it is isolatin1 :(
+        utf16_to_isolatin1(input, inputLeft, output, outputLeft);
+    }
+
+    return NS_OK;
+}
+
+#endif // USE_STDCONV
+
+//-----------------------------------------------------------------------------
+// API implementation
+//-----------------------------------------------------------------------------
+
+NS_COM nsresult
+NS_CopyNativeToUnicode(const nsACString &input, nsAString &output)
+{
+    output.Truncate();
+
+    PRUint32 inputLen = input.Length();
+
+    nsACString::const_iterator iter;
+    input.BeginReading(iter);
+
+    //
+    // OPTIMIZATION: preallocate space for largest possible result; convert
+    // directly into the result buffer to avoid intermediate buffer copy.
+    //
+    // this will generally result in a larger allocation, but that seems
+    // better than an extra buffer copy.
+    //
+    output.SetLength(inputLen);
+    nsAString::iterator out_iter;
+    output.BeginWriting(out_iter);
+
+    PRUnichar *result = out_iter.get();
+    PRUint32 resultLeft = inputLen;
+
+    const char *buf = iter.get();
+    PRUint32 bufLeft = inputLen;
+
+    nsNativeCharsetConverter conv;
+    nsresult rv = conv.NativeToUnicode(&buf, &bufLeft, &result, &resultLeft);
+    if (NS_SUCCEEDED(rv)) {
+        NS_ASSERTION(bufLeft == 0, "did not consume entire input buffer");
+        output.SetLength(inputLen - resultLeft);
+    }
+    return rv;
+}
+
+NS_COM nsresult
+NS_CopyUnicodeToNative(const nsAString &input, nsACString &output)
+{
+    output.Truncate();
+
+    nsAString::const_iterator iter, end;
+    input.BeginReading(iter);
+    input.EndReading(end);
+
+    // cannot easily avoid intermediate buffer copy.
+    char temp[4096];
+
+    nsNativeCharsetConverter conv;
+
+    const PRUnichar *buf = iter.get();
+    PRUint32 bufLeft = Distance(iter, end);
+    while (bufLeft) {
+        char *p = temp;
+        PRUint32 tempLeft = sizeof(temp);
+
+        nsresult rv = conv.UnicodeToNative(&buf, &bufLeft, &p, &tempLeft);
+        if (NS_FAILED(rv)) return rv;
+
+        if (tempLeft < sizeof(temp))
+            output.Append(temp, sizeof(temp) - tempLeft);
+    }
+    return NS_OK;
+}
+
+void
+NS_StartupNativeCharsetUtils()
+{
+    //
+    // need to initialize the locale or else charset conversion will fail.
+    // better not delay this in case some other component alters the locale
+    // settings.
+    //
+    // XXX we assume that we are called early enough that we should
+    // always be the first to care about the locale's charset.
+    //
+    setlocale(LC_CTYPE, "");
+
+    nsNativeCharsetConverter::GlobalInit();
+}
+
+void
+NS_ShutdownNativeCharsetUtils()
+{
+    nsNativeCharsetConverter::GlobalShutdown();
+}
+
+//-----------------------------------------------------------------------------
+// XP_BEOS
+//-----------------------------------------------------------------------------
+#elif defined(XP_BEOS)
+
+#include "nsAString.h"
+#include "nsReadableUtils.h"
+#include "nsString.h"
+
+NS_COM nsresult
+NS_CopyNativeToUnicode(const nsACString &input, nsAString  &output)
+{
+    CopyUTF8toUTF16(input, output);
+    return NS_OK;
+}
+
+NS_COM nsresult
+NS_CopyUnicodeToNative(const nsAString  &input, nsACString &output)
+{
+    CopyUTF16toUTF8(input, output);
+    return NS_OK;
+}
+
+void
+NS_StartupNativeCharsetUtils()
+{
+}
+
+void
+NS_ShutdownNativeCharsetUtils()
+{
+}
+
+//-----------------------------------------------------------------------------
+// XP_WIN
+//-----------------------------------------------------------------------------
+#elif defined(XP_WIN)
+
+#include <windows.h>
+#include "nsAString.h"
+
+NS_COM nsresult
+NS_CopyNativeToUnicode(const nsACString &input, nsAString &output)
+{
+    PRUint32 inputLen = input.Length();
+
+    nsACString::const_iterator iter;
+    input.BeginReading(iter);
+
+    const char *buf = iter.get();
+
+    // determine length of result
+    PRUint32 resultLen = 0;
+    int n = ::MultiByteToWideChar(CP_ACP, 0, buf, inputLen, NULL, 0);
+    if (n > 0)
+        resultLen += n;
+
+    // allocate sufficient space
+    output.SetLength(resultLen);
+    if (resultLen > 0) {
+        nsAString::iterator out_iter;
+        output.BeginWriting(out_iter);
+
+        PRUnichar *result = out_iter.get();
+
+        ::MultiByteToWideChar(CP_ACP, 0, buf, inputLen, result, resultLen);
+    }
+    return NS_OK;
+}
+
+NS_COM nsresult
+NS_CopyUnicodeToNative(const nsAString  &input, nsACString &output)
+{
+    PRUint32 inputLen = input.Length();
+
+    nsAString::const_iterator iter;
+    input.BeginReading(iter);
+
+    const PRUnichar *buf = iter.get();
+
+    // determine length of result
+    PRUint32 resultLen = 0;
+
+    int n = ::WideCharToMultiByte(CP_ACP, 0, buf, inputLen, NULL, 0, NULL, NULL);
+    if (n > 0)
+        resultLen += n;
+
+    // allocate sufficient space
+    output.SetLength(resultLen);
+    if (resultLen > 0) {
+        nsACString::iterator out_iter;
+        output.BeginWriting(out_iter);
+
+        // default "defaultChar" is '?', which is an illegal character on windows
+        // file system.  That will cause file uncreatable. Change it to '_'
+        const char defaultChar = '_';
+
+        char *result = out_iter.get();
+
+        ::WideCharToMultiByte(CP_ACP, 0, buf, inputLen, result, resultLen,
+                              &defaultChar, NULL);
+    }
+    return NS_OK;
+}
+
+void
+NS_StartupNativeCharsetUtils()
+{
+}
+
+void
+NS_ShutdownNativeCharsetUtils()
+{
+}
+
+//-----------------------------------------------------------------------------
+// XP_OS2
+//-----------------------------------------------------------------------------
+#elif defined(XP_OS2)
+
+#define INCL_DOS
+#include <os2.h>
+#include <uconv.h>
+#include "nsAString.h"
+#include <ulserrno.h>
+#include "nsNativeCharsetUtils.h"
+
+static UconvObject UnicodeConverter = NULL;
+
+NS_COM nsresult
+NS_CopyNativeToUnicode(const nsACString &input, nsAString  &output)
+{
+    PRUint32 inputLen = input.Length();
+
+    nsACString::const_iterator iter;
+    input.BeginReading(iter);
+    const char *inputStr = iter.get();
+
+    // determine length of result
+    PRUint32 resultLen = inputLen;
+    output.SetLength(resultLen);
+
+    nsAString::iterator out_iter;
+    output.BeginWriting(out_iter);
+    UniChar *result = (UniChar*)out_iter.get();
+
+    size_t cSubs = 0;
+    size_t resultLeft = resultLen;
+
+    if (!UnicodeConverter)
+      NS_StartupNativeCharsetUtils();
+
+    int unirc = ::UniUconvToUcs(UnicodeConverter, (void**)&inputStr, &inputLen,
+                                &result, &resultLeft, &cSubs);
+
+    NS_ASSERTION(unirc != UCONV_E2BIG, "Path too big");
+
+    if (unirc != ULS_SUCCESS) {
+        output.Truncate();
+        return NS_ERROR_FAILURE;
+    }
+
+    // Need to update string length to reflect how many bytes were actually
+    // written.
+    output.Truncate(resultLen - resultLeft);
+    return NS_OK;
+}
+
+NS_COM nsresult
+NS_CopyUnicodeToNative(const nsAString &input, nsACString &output)
+{
+    size_t inputLen = input.Length();
+
+    nsAString::const_iterator iter;
+    input.BeginReading(iter);
+    UniChar* inputStr = (UniChar*) NS_CONST_CAST(PRUnichar*, iter.get());
+
+    // maximum length of unicode string of length x converted to native
+    // codepage is x*2
+    size_t resultLen = inputLen * 2;
+    output.SetLength(resultLen);
+
+    nsACString::iterator out_iter;
+    output.BeginWriting(out_iter);
+    char *result = out_iter.get();
+
+    size_t cSubs = 0;
+    size_t resultLeft = resultLen;
+
+    if (!UnicodeConverter)
+      NS_StartupNativeCharsetUtils();
+  
+    int unirc = ::UniUconvFromUcs(UnicodeConverter, &inputStr, &inputLen,
+                                  (void**)&result, &resultLeft, &cSubs);
+
+    NS_ASSERTION(unirc != UCONV_E2BIG, "Path too big");
+  
+    if (unirc != ULS_SUCCESS) {
+        output.Truncate();
+        return NS_ERROR_FAILURE;
+    }
+
+    // Need to update string length to reflect how many bytes were actually
+    // written.
+    output.Truncate(resultLen - resultLeft);
+    return NS_OK;
+}
+
+void
+NS_StartupNativeCharsetUtils()
+{
+    ULONG ulLength;
+    ULONG ulCodePage;
+    DosQueryCp(sizeof(ULONG), &ulCodePage, &ulLength);
+
+    UniChar codepage[20];
+    int unirc = ::UniMapCpToUcsCp(ulCodePage, codepage, 20);
+    if (unirc == ULS_SUCCESS) {
+        unirc = ::UniCreateUconvObject(codepage, &UnicodeConverter);
+        if (unirc == ULS_SUCCESS) {
+            uconv_attribute_t attr;
+            ::UniQueryUconvObject(UnicodeConverter, &attr, sizeof(uconv_attribute_t), 
+                                  NULL, NULL, NULL);
+            attr.options = UCONV_OPTION_SUBSTITUTE_BOTH;
+            attr.subchar_len=1;
+            attr.subchar[0]='_';
+            ::UniSetUconvObject(UnicodeConverter, &attr);
+        }
+    }
+}
+
+void
+NS_ShutdownNativeCharsetUtils()
+{
+    ::UniFreeUconvObject(UnicodeConverter);
+}
+
+//-----------------------------------------------------------------------------
+// XP_MAC
+//-----------------------------------------------------------------------------
+#elif defined(XP_MAC)
+
+#include <UnicodeConverter.h>
+#include <TextCommon.h>
+#include <Script.h>
+#include <MacErrors.h>
+#include "nsAString.h"
+
+class nsFSStringConversionMac {
+public:
+     static nsresult UCSToFS(const nsAString& aIn, nsACString& aOut);  
+     static nsresult FSToUCS(const nsACString& ain, nsAString& aOut);  
+
+     static void CleanUp();
+
+private:
+     static TextEncoding GetSystemEncoding();
+     static nsresult PrepareEncoder();
+     static nsresult PrepareDecoder();
+     
+     static UnicodeToTextInfo sEncoderInfo;
+     static TextToUnicodeInfo sDecoderInfo;
+};
+
+UnicodeToTextInfo nsFSStringConversionMac::sEncoderInfo = nsnull;
+TextToUnicodeInfo nsFSStringConversionMac::sDecoderInfo = nsnull;
+
+nsresult nsFSStringConversionMac::UCSToFS(const nsAString& aIn, nsACString& aOut)
+{
+    nsresult rv = PrepareEncoder();
+    if (NS_FAILED(rv)) return rv;
+    
+    OSStatus err = noErr;
+    char stackBuffer[512];
+
+    aOut.Truncate();
+
+    // for each chunk of |aIn|...
+    nsReadingIterator<PRUnichar> iter;
+    aIn.BeginReading(iter);
+
+    PRUint32 fragmentLength = PRUint32(iter.size_forward());        
+    UInt32 bytesLeft = fragmentLength * sizeof(UniChar);
+        
+    do {
+        UInt32 bytesRead = 0, bytesWritten = 0;
+        err = ::ConvertFromUnicodeToText(sEncoderInfo,
+                                         bytesLeft,
+                                         (const UniChar*)iter.get(),
+                                         kUnicodeUseFallbacksMask | kUnicodeLooseMappingsMask,
+                                         0, nsnull, nsnull, nsnull,
+                                         sizeof(stackBuffer),
+                                         &bytesRead,
+                                         &bytesWritten,
+                                         stackBuffer);
+        if (err == kTECUsedFallbacksStatus)
+            err = noErr;
+        else if (err == kTECOutputBufferFullStatus) {
+            bytesLeft -= bytesRead;
+            iter.advance(bytesRead / sizeof(UniChar));
+        }
+        aOut.Append(stackBuffer, bytesWritten);
+    }
+    while (err == kTECOutputBufferFullStatus);
+
+    return (err == noErr) ? NS_OK : NS_ERROR_FAILURE;
+}
+
+nsresult nsFSStringConversionMac::FSToUCS(const nsACString& aIn, nsAString& aOut)
+{
+    nsresult rv = PrepareDecoder();
+    if (NS_FAILED(rv)) return rv;
+    
+    OSStatus err = noErr;
+    UniChar stackBuffer[512];
+
+    aOut.Truncate(0);
+
+    // for each chunk of |aIn|...
+    nsReadingIterator<char> iter;
+    aIn.BeginReading(iter);
+
+    PRUint32 fragmentLength = PRUint32(iter.size_forward());        
+    UInt32 bytesLeft = fragmentLength;
+    
+    do {
+        UInt32 bytesRead = 0, bytesWritten = 0;
+        err = ::ConvertFromTextToUnicode(sDecoderInfo,
+                                         bytesLeft,
+                                         iter.get(),
+                                         kUnicodeUseFallbacksMask | kUnicodeLooseMappingsMask,
+                                         0, nsnull, nsnull, nsnull,
+                                         sizeof(stackBuffer),
+                                         &bytesRead,
+                                         &bytesWritten,
+                                         stackBuffer);
+        if (err == kTECUsedFallbacksStatus)
+            err = noErr;
+        else if (err == kTECOutputBufferFullStatus) {
+            bytesLeft -= bytesRead;
+            iter.advance(bytesRead);
+        }
+        aOut.Append((PRUnichar *)stackBuffer, bytesWritten / sizeof(PRUnichar));
+    }
+    while (err == kTECOutputBufferFullStatus);
+
+    return (err == noErr) ? NS_OK : NS_ERROR_FAILURE;
+}
+
+void nsFSStringConversionMac::CleanUp()
+{
+    if (sDecoderInfo) {
+        ::DisposeTextToUnicodeInfo(&sDecoderInfo);
+        sDecoderInfo = nsnull;
+    }
+    if (sEncoderInfo) {
+        ::DisposeUnicodeToTextInfo(&sEncoderInfo);
+        sEncoderInfo = nsnull;
+    }  
+}
+
+TextEncoding nsFSStringConversionMac::GetSystemEncoding()
+{
+    OSStatus err;
+    TextEncoding theEncoding;
+    
+    err = ::UpgradeScriptInfoToTextEncoding(smSystemScript, kTextLanguageDontCare,
+        kTextRegionDontCare, NULL, &theEncoding);
+    
+    if (err != noErr)
+        theEncoding = kTextEncodingMacRoman;
+    
+    return theEncoding;
+}
+
+nsresult nsFSStringConversionMac::PrepareEncoder()
+{
+    nsresult rv = NS_OK;
+    if (!sEncoderInfo) {
+        OSStatus err;
+        err = ::CreateUnicodeToTextInfoByEncoding(GetSystemEncoding(), &sEncoderInfo);
+        if (err)
+            rv = NS_ERROR_FAILURE;
+    }
+    return rv;
+}
+
+nsresult nsFSStringConversionMac::PrepareDecoder()
+{
+    nsresult rv = NS_OK;
+    if (!sDecoderInfo) {
+        OSStatus err;
+        err = ::CreateTextToUnicodeInfoByEncoding(GetSystemEncoding(), &sDecoderInfo);
+        if (err)
+            rv = NS_ERROR_FAILURE;
+    }
+    return rv;
+}
+
+NS_COM nsresult
+NS_CopyNativeToUnicode(const nsACString &input, nsAString  &output)
+{
+    return nsFSStringConversionMac::FSToUCS(input, output);
+}
+
+NS_COM nsresult
+NS_CopyUnicodeToNative(const nsAString  &input, nsACString &output)
+{
+    return nsFSStringConversionMac::UCSToFS(input, output);
+}
+
+void
+NS_StartupNativeCharsetUtils()
+{
+}
+
+void
+NS_ShutdownNativeCharsetUtils()
+{
+    nsFSStringConversionMac::CleanUp();
+}
+
+//-----------------------------------------------------------------------------
+// default : truncate/zeropad
+//-----------------------------------------------------------------------------
+#else
+
+#include "nsReadableUtils.h"
+
+NS_COM nsresult
+NS_CopyNativeToUnicode(const nsACString &input, nsAString  &output)
+{
+    CopyASCIItoUCS2(input, output);
+    return NS_OK;
+}
+
+NS_COM nsresult
+NS_CopyUnicodeToNative(const nsAString  &input, nsACString &output)
+{
+    CopyUCS2toASCII(input, output);
+    return NS_OK;
+}
+
+void
+NS_StartupNativeCharsetUtils()
+{
+}
+
+void
+NS_ShutdownNativeCharsetUtils()
+{
+}
+
+#endif