Adding upstream version 1:7.0.4.upstream/1%7.0.4upstream

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

1 files changed, 519 insertions, 0 deletions

diff --git a/sax/source/expatwrap/xml2utf.cxx b/sax/source/expatwrap/xml2utf.cxx
new file mode 100644
index 000000000..420f90b6c
--- /dev/null
+++ b/sax/source/expatwrap/xml2utf.cxx
@@ -0,0 +1,519 @@
+/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */
+/*
+ * This file is part of the LibreOffice project.
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ * This file incorporates work covered by the following license notice:
+ *
+ *   Licensed to the Apache Software Foundation (ASF) under one or more
+ *   contributor license agreements. See the NOTICE file distributed
+ *   with this work for additional information regarding copyright
+ *   ownership. The ASF licenses this file to you under the Apache
+ *   License, Version 2.0 (the "License"); you may not use this file
+ *   except in compliance with the License. You may obtain a copy of
+ *   the License at http://www.apache.org/licenses/LICENSE-2.0 .
+ */
+#include <string.h>
+
+#include <algorithm>
+
+#include <sal/types.h>
+
+#include <rtl/textenc.h>
+#include <rtl/tencinfo.h>
+#include <com/sun/star/io/NotConnectedException.hpp>
+#include <com/sun/star/io/XInputStream.hpp>
+
+using namespace ::com::sun::star::uno;
+using namespace ::com::sun::star::io;
+
+
+#include <xml2utf.hxx>
+#include <memory>
+
+namespace sax_expatwrap {
+
+sal_Int32 XMLFile2UTFConverter::readAndConvert( Sequence<sal_Int8> &seq , sal_Int32 nMaxToRead )
+{
+    if( ! m_in.is() ) {
+        throw NotConnectedException();
+    }
+    if( ! m_bStarted ) {
+        // it should be possible to find the encoding attribute
+        // within the first 512 bytes == 128 chars in UCS-4
+        nMaxToRead = ::std::max( sal_Int32(512) , nMaxToRead );
+    }
+
+    sal_Int32 nRead;
+    Sequence< sal_Int8 > seqStart;
+    while( true )
+    {
+        nRead = m_in->readSomeBytes( seq , nMaxToRead );
+
+        if( nRead + seqStart.getLength())
+        {
+            // if nRead is 0, the file is already eof.
+            if( ! m_bStarted && nRead )
+            {
+                // ensure that enough data is available to parse encoding
+                if( seqStart.hasElements() )
+                {
+                  // prefix with what we had so far.
+                  sal_Int32 nLength = seq.getLength();
+                  seq.realloc( seqStart.getLength() + nLength );
+
+                  memmove (seq.getArray() + seqStart.getLength(),
+                       seq.getConstArray(),
+                       nLength);
+                  memcpy  (seq.getArray(),
+                       seqStart.getConstArray(),
+                       seqStart.getLength());
+                }
+
+                // autodetection with the first bytes
+                if( ! isEncodingRecognizable( seq ) )
+                {
+                  // remember what we have so far.
+                  seqStart = seq;
+
+                  // read more !
+                  continue;
+                }
+                if( scanForEncoding( seq ) || !m_sEncoding.isEmpty() ) {
+                    // initialize decoding
+                    initializeDecoding();
+                }
+                seqStart = Sequence < sal_Int8 > ();
+            }
+
+            // do the encoding
+            if( m_pText2Unicode && m_pUnicode2Text &&
+                m_pText2Unicode->canContinue() ) {
+
+                Sequence<sal_Unicode> seqUnicode = m_pText2Unicode->convert( seq );
+                seq = m_pUnicode2Text->convert( seqUnicode.getConstArray(), seqUnicode.getLength() );
+            }
+
+            if( ! m_bStarted )
+            {
+                // it must now be ensured, that no encoding attribute exist anymore
+                // ( otherwise the expat-Parser will crash )
+                // This must be done after decoding !
+                // ( e.g. Files decoded in ucs-4 cannot be read properly )
+                m_bStarted = true;
+                removeEncoding( seq );
+            }
+            nRead = seq.getLength();
+        }
+
+        break;
+    }
+    return nRead;
+}
+
+void XMLFile2UTFConverter::removeEncoding( Sequence<sal_Int8> &seq )
+{
+    const sal_Int8 *pSource = seq.getArray();
+    if (!(seq.getLength() >= 5 && !strncmp(reinterpret_cast<const char *>(pSource), "<?xml", 5)))
+        return;
+
+    // scan for encoding
+    OString str( reinterpret_cast<char const *>(pSource), seq.getLength() );
+
+    // cut sequence to first line break
+    // find first line break;
+    int nMax = str.indexOf( 10 );
+    if( nMax >= 0 )
+    {
+        str = str.copy( 0 , nMax );
+    }
+
+    int nFound = str.indexOf( " encoding" );
+    if( nFound < 0 )        return;
+
+    int nStop;
+    int nStart = str.indexOf( "\"" , nFound );
+    if( nStart < 0 || str.indexOf( "'" , nFound ) < nStart )
+    {
+        nStart = str.indexOf( "'" , nFound );
+        nStop  = str.indexOf( "'" , nStart +1 );
+    }
+    else
+    {
+        nStop  = str.indexOf( "\"" , nStart +1);
+    }
+
+    if( nStart >= 0 && nStop >= 0 && nStart+1 < nStop )
+    {
+        // remove encoding tag from file
+        memmove(        &( seq.getArray()[nFound] ) ,
+                        &( seq.getArray()[nStop+1]) ,
+                        seq.getLength() - nStop -1);
+        seq.realloc( seq.getLength() - ( nStop+1 - nFound ) );
+    }
+}
+
+// Checks, if enough data has been accumulated to recognize the encoding
+bool XMLFile2UTFConverter::isEncodingRecognizable( const Sequence< sal_Int8 > &seq)
+{
+    const sal_Int8 *pSource = seq.getConstArray();
+    bool bCheckIfFirstClosingBracketExsists = false;
+
+    if( seq.getLength() < 8 ) {
+        // no recognition possible, when less than 8 bytes are available
+        return false;
+    }
+
+    if( ! strncmp( reinterpret_cast<const char *>(pSource), "<?xml", 5 ) ) {
+        // scan if the <?xml tag finishes within this buffer
+        bCheckIfFirstClosingBracketExsists = true;
+    }
+    else if( ('<' == pSource[0] || '<' == pSource[2] ) &&
+             ('?' == pSource[4] || '?' == pSource[6] ) )
+    {
+        // check for utf-16
+        bCheckIfFirstClosingBracketExsists = true;
+    }
+    else if( ( '<' == pSource[1] || '<' == pSource[3] ) &&
+             ( '?' == pSource[5] || '?' == pSource[7] ) )
+    {
+        // check for
+        bCheckIfFirstClosingBracketExsists = true;
+    }
+
+    if( bCheckIfFirstClosingBracketExsists )
+    {
+        // whole <?xml tag is valid
+        return std::find(seq.begin(), seq.end(), '>') != seq.end();
+    }
+
+    // No <? tag in front, no need for a bigger buffer
+    return true;
+}
+
+bool XMLFile2UTFConverter::scanForEncoding( Sequence< sal_Int8 > &seq )
+{
+    const sal_uInt8 *pSource = reinterpret_cast<const sal_uInt8*>( seq.getConstArray() );
+    bool bReturn = true;
+
+    if( seq.getLength() < 4 ) {
+        // no recognition possible, when less than 4 bytes are available
+        return false;
+    }
+
+    // first level : detect possible file formats
+    if (seq.getLength() >= 5 && !strncmp(reinterpret_cast<const char *>(pSource), "<?xml", 5)) {
+        // scan for encoding
+        OString str( reinterpret_cast<const char *>(pSource), seq.getLength() );
+
+        // cut sequence to first line break
+        //find first line break;
+        int nMax = str.indexOf( 10 );
+        if( nMax >= 0 )
+        {
+            str = str.copy( 0 , nMax );
+        }
+
+        int nFound = str.indexOf( " encoding" );
+        if( nFound >= 0 ) {
+            int nStop;
+            int nStart = str.indexOf( "\"" , nFound );
+            if( nStart < 0 || str.indexOf( "'" , nFound ) < nStart )
+            {
+                nStart = str.indexOf( "'" , nFound );
+                nStop  = str.indexOf( "'" , nStart +1 );
+            }
+            else
+            {
+                nStop  = str.indexOf( "\"" , nStart +1);
+            }
+            if( nStart >= 0 && nStop >= 0 && nStart+1 < nStop )
+            {
+                // encoding found finally
+                m_sEncoding = str.copy( nStart+1 , nStop - nStart - 1 );
+            }
+        }
+    }
+    else if( 0xFE == pSource[0] &&
+             0xFF == pSource[1] ) {
+        // UTF-16 big endian
+        // conversion is done so that encoding information can be easily extracted
+        m_sEncoding = "utf-16";
+    }
+    else if( 0xFF == pSource[0] &&
+             0xFE == pSource[1] ) {
+        // UTF-16 little endian
+        // conversion is done so that encoding information can be easily extracted
+        m_sEncoding = "utf-16";
+    }
+    else if( 0x00 == pSource[0] && 0x3c == pSource[1]  && 0x00 == pSource[2] && 0x3f == pSource[3] ) {
+        // UTF-16 big endian without byte order mark (this is (strictly speaking) an error.)
+        // The byte order mark is simply added
+
+        // simply add the byte order mark !
+        seq.realloc( seq.getLength() + 2 );
+        memmove( &( seq.getArray()[2] ) , seq.getArray() , seq.getLength() - 2 );
+        reinterpret_cast<sal_uInt8*>(seq.getArray())[0] = 0xFE;
+        reinterpret_cast<sal_uInt8*>(seq.getArray())[1] = 0xFF;
+
+        m_sEncoding = "utf-16";
+    }
+    else if( 0x3c == pSource[0] && 0x00 == pSource[1]  && 0x3f == pSource[2] && 0x00 == pSource[3] ) {
+        // UTF-16 little endian without byte order mark (this is (strictly speaking) an error.)
+        // The byte order mark is simply added
+
+        seq.realloc( seq.getLength() + 2 );
+        memmove( &( seq.getArray()[2] ) , seq.getArray() , seq.getLength() - 2 );
+        reinterpret_cast<sal_uInt8*>(seq.getArray())[0] = 0xFF;
+        reinterpret_cast<sal_uInt8*>(seq.getArray())[1] = 0xFE;
+
+        m_sEncoding = "utf-16";
+    }
+    else if( 0xEF == pSource[0] &&
+             0xBB == pSource[1] &&
+             0xBF == pSource[2] )
+    {
+        // UTF-8 BOM (byte order mark); signifies utf-8, and not byte order
+        // The BOM is removed.
+        memmove( seq.getArray(), &( seq.getArray()[3] ), seq.getLength()-3 );
+        seq.realloc( seq.getLength() - 3 );
+        m_sEncoding = "utf-8";
+    }
+    else if( 0x00 == pSource[0] && 0x00 == pSource[1]  && 0x00 == pSource[2] && 0x3c == pSource[3] ) {
+        // UCS-4 big endian
+        m_sEncoding = "ucs-4";
+    }
+    else if( 0x3c == pSource[0] && 0x00 == pSource[1]  && 0x00 == pSource[2] && 0x00 == pSource[3] ) {
+        // UCS-4 little endian
+        m_sEncoding = "ucs-4";
+    }
+/* TODO: no need to test for the moment since we return sal_False like default case anyway
+    else if( 0x4c == pSource[0] && 0x6f == pSource[1]  &&
+             0xa7 == static_cast<unsigned char> (pSource[2]) &&
+             0x94 == static_cast<unsigned char> (pSource[3]) ) {
+        // EBCDIC
+        bReturn = sal_False;   // must be extended
+    }
+*/
+    else {
+        // other
+        // UTF8 is directly recognized by the parser.
+        bReturn = false;
+    }
+
+    return bReturn;
+}
+
+void XMLFile2UTFConverter::initializeDecoding()
+{
+
+    if( !m_sEncoding.isEmpty() )
+    {
+        rtl_TextEncoding encoding = rtl_getTextEncodingFromMimeCharset( m_sEncoding.getStr() );
+        if( encoding != RTL_TEXTENCODING_UTF8 )
+        {
+            m_pText2Unicode = std::make_unique<Text2UnicodeConverter>( m_sEncoding );
+            m_pUnicode2Text = std::make_unique<Unicode2TextConverter>( RTL_TEXTENCODING_UTF8 );
+        }
+    }
+}
+
+
+// Text2UnicodeConverter
+
+
+Text2UnicodeConverter::Text2UnicodeConverter( const OString &sEncoding )
+    : m_convText2Unicode(nullptr)
+    , m_contextText2Unicode(nullptr)
+{
+    rtl_TextEncoding encoding = rtl_getTextEncodingFromMimeCharset( sEncoding.getStr() );
+    if( RTL_TEXTENCODING_DONTKNOW == encoding )
+    {
+        m_bCanContinue = false;
+        m_bInitialized = false;
+    }
+    else
+    {
+        init( encoding );
+    }
+}
+
+Text2UnicodeConverter::~Text2UnicodeConverter()
+{
+    if( m_bInitialized )
+    {
+        rtl_destroyTextToUnicodeContext( m_convText2Unicode , m_contextText2Unicode );
+        rtl_destroyUnicodeToTextConverter( m_convText2Unicode );
+    }
+}
+
+void Text2UnicodeConverter::init( rtl_TextEncoding encoding )
+{
+    m_bCanContinue = true;
+    m_bInitialized = true;
+
+    m_convText2Unicode  = rtl_createTextToUnicodeConverter(encoding);
+    m_contextText2Unicode = rtl_createTextToUnicodeContext( m_convText2Unicode );
+}
+
+
+Sequence<sal_Unicode> Text2UnicodeConverter::convert( const Sequence<sal_Int8> &seqText )
+{
+    sal_uInt32 uiInfo;
+    sal_Size nSrcCvtBytes   = 0;
+    sal_Size nTargetCount   = 0;
+    sal_Size nSourceCount   = 0;
+
+    // the whole source size
+    sal_Int32   nSourceSize = seqText.getLength() + m_seqSource.getLength();
+    Sequence<sal_Unicode>   seqUnicode ( nSourceSize );
+
+    const sal_Int8 *pbSource = seqText.getConstArray();
+    std::unique_ptr<sal_Int8[]> pbTempMem;
+
+    if( m_seqSource.hasElements() ) {
+        // put old rest and new byte sequence into one array
+        pbTempMem.reset(new sal_Int8[ nSourceSize ]);
+        memcpy( pbTempMem.get() , m_seqSource.getConstArray() , m_seqSource.getLength() );
+        memcpy( &(pbTempMem[ m_seqSource.getLength() ]) , seqText.getConstArray() , seqText.getLength() );
+        pbSource = pbTempMem.get();
+
+        // set to zero again
+        m_seqSource = Sequence< sal_Int8 >();
+    }
+
+    while( true ) {
+
+        /* All invalid characters are transformed to the unicode undefined char */
+        nTargetCount +=     rtl_convertTextToUnicode(
+                                    m_convText2Unicode,
+                                    m_contextText2Unicode,
+                                    reinterpret_cast<const char *>(&( pbSource[nSourceCount] )),
+                                    nSourceSize - nSourceCount ,
+                                    &( seqUnicode.getArray()[ nTargetCount ] ),
+                                    seqUnicode.getLength() - nTargetCount,
+                                    RTL_TEXTTOUNICODE_FLAGS_UNDEFINED_DEFAULT   |
+                                    RTL_TEXTTOUNICODE_FLAGS_MBUNDEFINED_DEFAULT |
+                                    RTL_TEXTTOUNICODE_FLAGS_INVALID_DEFAULT,
+                                    &uiInfo,
+                                    &nSrcCvtBytes );
+        nSourceCount += nSrcCvtBytes;
+
+        if( uiInfo & RTL_TEXTTOUNICODE_INFO_DESTBUFFERTOOSMALL ) {
+            // save necessary bytes for next conversion
+            seqUnicode.realloc( seqUnicode.getLength() * 2 );
+            continue;
+        }
+        break;
+    }
+    if( uiInfo & RTL_TEXTTOUNICODE_INFO_SRCBUFFERTOOSMALL ) {
+        m_seqSource.realloc( nSourceSize - nSourceCount );
+        memcpy( m_seqSource.getArray() , &(pbSource[nSourceCount]) , nSourceSize-nSourceCount );
+    }
+
+    // set to correct unicode size
+    seqUnicode.realloc( nTargetCount );
+
+    return seqUnicode;
+}
+
+
+// Unicode2TextConverter
+
+
+Unicode2TextConverter::Unicode2TextConverter( rtl_TextEncoding encoding )
+{
+    m_convUnicode2Text  = rtl_createUnicodeToTextConverter( encoding );
+    m_contextUnicode2Text = rtl_createUnicodeToTextContext( m_convUnicode2Text );
+}
+
+
+Unicode2TextConverter::~Unicode2TextConverter()
+{
+    rtl_destroyUnicodeToTextContext( m_convUnicode2Text , m_contextUnicode2Text );
+    rtl_destroyUnicodeToTextConverter( m_convUnicode2Text );
+}
+
+
+Sequence<sal_Int8> Unicode2TextConverter::convert(const sal_Unicode *puSource , sal_Int32 nSourceSize)
+{
+    std::unique_ptr<sal_Unicode[]> puTempMem;
+
+    if( m_seqSource.hasElements() ) {
+        // For surrogates !
+        // put old rest and new byte sequence into one array
+        // In general when surrogates are used, they should be rarely
+        // cut off between two convert()-calls. So this code is used
+        // rarely and the extra copy is acceptable.
+        puTempMem.reset(new sal_Unicode[ nSourceSize + m_seqSource.getLength()]);
+        memcpy( puTempMem.get() ,
+                m_seqSource.getConstArray() ,
+                m_seqSource.getLength() * sizeof( sal_Unicode ) );
+        memcpy(
+            &(puTempMem[ m_seqSource.getLength() ]) ,
+            puSource ,
+            nSourceSize*sizeof( sal_Unicode ) );
+        puSource = puTempMem.get();
+        nSourceSize += m_seqSource.getLength();
+
+        m_seqSource = Sequence< sal_Unicode > ();
+    }
+
+
+    sal_Size nTargetCount = 0;
+    sal_Size nSourceCount = 0;
+
+    sal_uInt32 uiInfo;
+    sal_Size nSrcCvtChars;
+
+    // take nSourceSize * 3 as preference
+    // this is an upper boundary for converting to utf8,
+    // which most often used as the target.
+    sal_Int32 nSeqSize =  nSourceSize * 3;
+
+    Sequence<sal_Int8>  seqText( nSeqSize );
+    char *pTarget = reinterpret_cast<char *>(seqText.getArray());
+    while( true ) {
+
+        nTargetCount += rtl_convertUnicodeToText(
+                                    m_convUnicode2Text,
+                                    m_contextUnicode2Text,
+                                    &( puSource[nSourceCount] ),
+                                    nSourceSize - nSourceCount ,
+                                    &( pTarget[nTargetCount] ),
+                                    nSeqSize - nTargetCount,
+                                    RTL_UNICODETOTEXT_FLAGS_UNDEFINED_DEFAULT |
+                                    RTL_UNICODETOTEXT_FLAGS_INVALID_DEFAULT ,
+                                    &uiInfo,
+                                    &nSrcCvtChars);
+        nSourceCount += nSrcCvtChars;
+
+        if( uiInfo & RTL_UNICODETOTEXT_INFO_DESTBUFFERTOSMALL ) {
+            nSeqSize = nSeqSize *2;
+            seqText.realloc( nSeqSize );  // double array size
+            pTarget = reinterpret_cast<char *>(seqText.getArray());
+            continue;
+        }
+        break;
+    }
+
+    // for surrogates
+    if( uiInfo & RTL_UNICODETOTEXT_INFO_SRCBUFFERTOSMALL ) {
+        m_seqSource.realloc( nSourceSize - nSourceCount );
+        memcpy( m_seqSource.getArray() ,
+                &(puSource[nSourceCount]),
+                (nSourceSize - nSourceCount) * sizeof( sal_Unicode ) );
+    }
+
+    // reduce the size of the buffer (fast, no copy necessary)
+    seqText.realloc( nTargetCount );
+
+    return seqText;
+}
+
+}
+
+/* vim:set shiftwidth=4 softtabstop=4 expandtab: */