Adding upstream version 16.2.11+ds.upstream/16.2.11+dsupstream

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

1 files changed, 451 insertions, 0 deletions

diff --git a/src/rapidjson/test/unittest/encodingstest.cpp b/src/rapidjson/test/unittest/encodingstest.cpp
new file mode 100644
index 000000000..67b0391ed
--- /dev/null
+++ b/src/rapidjson/test/unittest/encodingstest.cpp
@@ -0,0 +1,451 @@
+// Tencent is pleased to support the open source community by making RapidJSON available.
+// 
+// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved.
+//
+// Licensed under the MIT License (the "License"); you may not use this file except
+// in compliance with the License. You may obtain a copy of the License at
+//
+// http://opensource.org/licenses/MIT
+//
+// Unless required by applicable law or agreed to in writing, software distributed 
+// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR 
+// CONDITIONS OF ANY KIND, either express or implied. See the License for the 
+// specific language governing permissions and limitations under the License.
+
+#include "unittest.h"
+#include "rapidjson/filereadstream.h"
+#include "rapidjson/filewritestream.h"
+#include "rapidjson/encodedstream.h"
+#include "rapidjson/stringbuffer.h"
+
+using namespace rapidjson;
+
+// Verification of encoders/decoders with Hoehrmann's UTF8 decoder
+
+// http://www.unicode.org/Public/UNIDATA/Blocks.txt
+static const unsigned kCodepointRanges[] = {
+    0x0000,     0x007F,     // Basic Latin
+    0x0080,     0x00FF,     // Latin-1 Supplement
+    0x0100,     0x017F,     // Latin Extended-A
+    0x0180,     0x024F,     // Latin Extended-B
+    0x0250,     0x02AF,     // IPA Extensions
+    0x02B0,     0x02FF,     // Spacing Modifier Letters
+    0x0300,     0x036F,     // Combining Diacritical Marks
+    0x0370,     0x03FF,     // Greek and Coptic
+    0x0400,     0x04FF,     // Cyrillic
+    0x0500,     0x052F,     // Cyrillic Supplement
+    0x0530,     0x058F,     // Armenian
+    0x0590,     0x05FF,     // Hebrew
+    0x0600,     0x06FF,     // Arabic
+    0x0700,     0x074F,     // Syriac
+    0x0750,     0x077F,     // Arabic Supplement
+    0x0780,     0x07BF,     // Thaana
+    0x07C0,     0x07FF,     // NKo
+    0x0800,     0x083F,     // Samaritan
+    0x0840,     0x085F,     // Mandaic
+    0x0900,     0x097F,     // Devanagari
+    0x0980,     0x09FF,     // Bengali
+    0x0A00,     0x0A7F,     // Gurmukhi
+    0x0A80,     0x0AFF,     // Gujarati
+    0x0B00,     0x0B7F,     // Oriya
+    0x0B80,     0x0BFF,     // Tamil
+    0x0C00,     0x0C7F,     // Telugu
+    0x0C80,     0x0CFF,     // Kannada
+    0x0D00,     0x0D7F,     // Malayalam
+    0x0D80,     0x0DFF,     // Sinhala
+    0x0E00,     0x0E7F,     // Thai
+    0x0E80,     0x0EFF,     // Lao
+    0x0F00,     0x0FFF,     // Tibetan
+    0x1000,     0x109F,     // Myanmar
+    0x10A0,     0x10FF,     // Georgian
+    0x1100,     0x11FF,     // Hangul Jamo
+    0x1200,     0x137F,     // Ethiopic
+    0x1380,     0x139F,     // Ethiopic Supplement
+    0x13A0,     0x13FF,     // Cherokee
+    0x1400,     0x167F,     // Unified Canadian Aboriginal Syllabics
+    0x1680,     0x169F,     // Ogham
+    0x16A0,     0x16FF,     // Runic
+    0x1700,     0x171F,     // Tagalog
+    0x1720,     0x173F,     // Hanunoo
+    0x1740,     0x175F,     // Buhid
+    0x1760,     0x177F,     // Tagbanwa
+    0x1780,     0x17FF,     // Khmer
+    0x1800,     0x18AF,     // Mongolian
+    0x18B0,     0x18FF,     // Unified Canadian Aboriginal Syllabics Extended
+    0x1900,     0x194F,     // Limbu
+    0x1950,     0x197F,     // Tai Le
+    0x1980,     0x19DF,     // New Tai Lue
+    0x19E0,     0x19FF,     // Khmer Symbols
+    0x1A00,     0x1A1F,     // Buginese
+    0x1A20,     0x1AAF,     // Tai Tham
+    0x1B00,     0x1B7F,     // Balinese
+    0x1B80,     0x1BBF,     // Sundanese
+    0x1BC0,     0x1BFF,     // Batak
+    0x1C00,     0x1C4F,     // Lepcha
+    0x1C50,     0x1C7F,     // Ol Chiki
+    0x1CD0,     0x1CFF,     // Vedic Extensions
+    0x1D00,     0x1D7F,     // Phonetic Extensions
+    0x1D80,     0x1DBF,     // Phonetic Extensions Supplement
+    0x1DC0,     0x1DFF,     // Combining Diacritical Marks Supplement
+    0x1E00,     0x1EFF,     // Latin Extended Additional
+    0x1F00,     0x1FFF,     // Greek Extended
+    0x2000,     0x206F,     // General Punctuation
+    0x2070,     0x209F,     // Superscripts and Subscripts
+    0x20A0,     0x20CF,     // Currency Symbols
+    0x20D0,     0x20FF,     // Combining Diacritical Marks for Symbols
+    0x2100,     0x214F,     // Letterlike Symbols
+    0x2150,     0x218F,     // Number Forms
+    0x2190,     0x21FF,     // Arrows
+    0x2200,     0x22FF,     // Mathematical Operators
+    0x2300,     0x23FF,     // Miscellaneous Technical
+    0x2400,     0x243F,     // Control Pictures
+    0x2440,     0x245F,     // Optical Character Recognition
+    0x2460,     0x24FF,     // Enclosed Alphanumerics
+    0x2500,     0x257F,     // Box Drawing
+    0x2580,     0x259F,     // Block Elements
+    0x25A0,     0x25FF,     // Geometric Shapes
+    0x2600,     0x26FF,     // Miscellaneous Symbols
+    0x2700,     0x27BF,     // Dingbats
+    0x27C0,     0x27EF,     // Miscellaneous Mathematical Symbols-A
+    0x27F0,     0x27FF,     // Supplemental Arrows-A
+    0x2800,     0x28FF,     // Braille Patterns
+    0x2900,     0x297F,     // Supplemental Arrows-B
+    0x2980,     0x29FF,     // Miscellaneous Mathematical Symbols-B
+    0x2A00,     0x2AFF,     // Supplemental Mathematical Operators
+    0x2B00,     0x2BFF,     // Miscellaneous Symbols and Arrows
+    0x2C00,     0x2C5F,     // Glagolitic
+    0x2C60,     0x2C7F,     // Latin Extended-C
+    0x2C80,     0x2CFF,     // Coptic
+    0x2D00,     0x2D2F,     // Georgian Supplement
+    0x2D30,     0x2D7F,     // Tifinagh
+    0x2D80,     0x2DDF,     // Ethiopic Extended
+    0x2DE0,     0x2DFF,     // Cyrillic Extended-A
+    0x2E00,     0x2E7F,     // Supplemental Punctuation
+    0x2E80,     0x2EFF,     // CJK Radicals Supplement
+    0x2F00,     0x2FDF,     // Kangxi Radicals
+    0x2FF0,     0x2FFF,     // Ideographic Description Characters
+    0x3000,     0x303F,     // CJK Symbols and Punctuation
+    0x3040,     0x309F,     // Hiragana
+    0x30A0,     0x30FF,     // Katakana
+    0x3100,     0x312F,     // Bopomofo
+    0x3130,     0x318F,     // Hangul Compatibility Jamo
+    0x3190,     0x319F,     // Kanbun
+    0x31A0,     0x31BF,     // Bopomofo Extended
+    0x31C0,     0x31EF,     // CJK Strokes
+    0x31F0,     0x31FF,     // Katakana Phonetic Extensions
+    0x3200,     0x32FF,     // Enclosed CJK Letters and Months
+    0x3300,     0x33FF,     // CJK Compatibility
+    0x3400,     0x4DBF,     // CJK Unified Ideographs Extension A
+    0x4DC0,     0x4DFF,     // Yijing Hexagram Symbols
+    0x4E00,     0x9FFF,     // CJK Unified Ideographs
+    0xA000,     0xA48F,     // Yi Syllables
+    0xA490,     0xA4CF,     // Yi Radicals
+    0xA4D0,     0xA4FF,     // Lisu
+    0xA500,     0xA63F,     // Vai
+    0xA640,     0xA69F,     // Cyrillic Extended-B
+    0xA6A0,     0xA6FF,     // Bamum
+    0xA700,     0xA71F,     // Modifier Tone Letters
+    0xA720,     0xA7FF,     // Latin Extended-D
+    0xA800,     0xA82F,     // Syloti Nagri
+    0xA830,     0xA83F,     // Common Indic Number Forms
+    0xA840,     0xA87F,     // Phags-pa
+    0xA880,     0xA8DF,     // Saurashtra
+    0xA8E0,     0xA8FF,     // Devanagari Extended
+    0xA900,     0xA92F,     // Kayah Li
+    0xA930,     0xA95F,     // Rejang
+    0xA960,     0xA97F,     // Hangul Jamo Extended-A
+    0xA980,     0xA9DF,     // Javanese
+    0xAA00,     0xAA5F,     // Cham
+    0xAA60,     0xAA7F,     // Myanmar Extended-A
+    0xAA80,     0xAADF,     // Tai Viet
+    0xAB00,     0xAB2F,     // Ethiopic Extended-A
+    0xABC0,     0xABFF,     // Meetei Mayek
+    0xAC00,     0xD7AF,     // Hangul Syllables
+    0xD7B0,     0xD7FF,     // Hangul Jamo Extended-B
+    //0xD800,       0xDB7F,     // High Surrogates
+    //0xDB80,       0xDBFF,     // High Private Use Surrogates
+    //0xDC00,       0xDFFF,     // Low Surrogates
+    0xE000,     0xF8FF,     // Private Use Area
+    0xF900,     0xFAFF,     // CJK Compatibility Ideographs
+    0xFB00,     0xFB4F,     // Alphabetic Presentation Forms
+    0xFB50,     0xFDFF,     // Arabic Presentation Forms-A
+    0xFE00,     0xFE0F,     // Variation Selectors
+    0xFE10,     0xFE1F,     // Vertical Forms
+    0xFE20,     0xFE2F,     // Combining Half Marks
+    0xFE30,     0xFE4F,     // CJK Compatibility Forms
+    0xFE50,     0xFE6F,     // Small Form Variants
+    0xFE70,     0xFEFF,     // Arabic Presentation Forms-B
+    0xFF00,     0xFFEF,     // Halfwidth and Fullwidth Forms
+    0xFFF0,     0xFFFF,     // Specials
+    0x10000,    0x1007F,    // Linear B Syllabary
+    0x10080,    0x100FF,    // Linear B Ideograms
+    0x10100,    0x1013F,    // Aegean Numbers
+    0x10140,    0x1018F,    // Ancient Greek Numbers
+    0x10190,    0x101CF,    // Ancient Symbols
+    0x101D0,    0x101FF,    // Phaistos Disc
+    0x10280,    0x1029F,    // Lycian
+    0x102A0,    0x102DF,    // Carian
+    0x10300,    0x1032F,    // Old Italic
+    0x10330,    0x1034F,    // Gothic
+    0x10380,    0x1039F,    // Ugaritic
+    0x103A0,    0x103DF,    // Old Persian
+    0x10400,    0x1044F,    // Deseret
+    0x10450,    0x1047F,    // Shavian
+    0x10480,    0x104AF,    // Osmanya
+    0x10800,    0x1083F,    // Cypriot Syllabary
+    0x10840,    0x1085F,    // Imperial Aramaic
+    0x10900,    0x1091F,    // Phoenician
+    0x10920,    0x1093F,    // Lydian
+    0x10A00,    0x10A5F,    // Kharoshthi
+    0x10A60,    0x10A7F,    // Old South Arabian
+    0x10B00,    0x10B3F,    // Avestan
+    0x10B40,    0x10B5F,    // Inscriptional Parthian
+    0x10B60,    0x10B7F,    // Inscriptional Pahlavi
+    0x10C00,    0x10C4F,    // Old Turkic
+    0x10E60,    0x10E7F,    // Rumi Numeral Symbols
+    0x11000,    0x1107F,    // Brahmi
+    0x11080,    0x110CF,    // Kaithi
+    0x12000,    0x123FF,    // Cuneiform
+    0x12400,    0x1247F,    // Cuneiform Numbers and Punctuation
+    0x13000,    0x1342F,    // Egyptian Hieroglyphs
+    0x16800,    0x16A3F,    // Bamum Supplement
+    0x1B000,    0x1B0FF,    // Kana Supplement
+    0x1D000,    0x1D0FF,    // Byzantine Musical Symbols
+    0x1D100,    0x1D1FF,    // Musical Symbols
+    0x1D200,    0x1D24F,    // Ancient Greek Musical Notation
+    0x1D300,    0x1D35F,    // Tai Xuan Jing Symbols
+    0x1D360,    0x1D37F,    // Counting Rod Numerals
+    0x1D400,    0x1D7FF,    // Mathematical Alphanumeric Symbols
+    0x1F000,    0x1F02F,    // Mahjong Tiles
+    0x1F030,    0x1F09F,    // Domino Tiles
+    0x1F0A0,    0x1F0FF,    // Playing Cards
+    0x1F100,    0x1F1FF,    // Enclosed Alphanumeric Supplement
+    0x1F200,    0x1F2FF,    // Enclosed Ideographic Supplement
+    0x1F300,    0x1F5FF,    // Miscellaneous Symbols And Pictographs
+    0x1F600,    0x1F64F,    // Emoticons
+    0x1F680,    0x1F6FF,    // Transport And Map Symbols
+    0x1F700,    0x1F77F,    // Alchemical Symbols
+    0x20000,    0x2A6DF,    // CJK Unified Ideographs Extension B
+    0x2A700,    0x2B73F,    // CJK Unified Ideographs Extension C
+    0x2B740,    0x2B81F,    // CJK Unified Ideographs Extension D
+    0x2F800,    0x2FA1F,    // CJK Compatibility Ideographs Supplement
+    0xE0000,    0xE007F,    // Tags
+    0xE0100,    0xE01EF,    // Variation Selectors Supplement
+    0xF0000,    0xFFFFF,    // Supplementary Private Use Area-A
+    0x100000,   0x10FFFF,   // Supplementary Private Use Area-B
+    0xFFFFFFFF
+};
+
+// Copyright (c) 2008-2010 Bjoern Hoehrmann <bjoern@hoehrmann.de>
+// See http://bjoern.hoehrmann.de/utf-8/decoder/dfa/ for details.
+
+#define UTF8_ACCEPT 0u
+
+static const unsigned char utf8d[] = {
+    // The first part of the table maps bytes to character classes that
+    // to reduce the size of the transition table and create bitmasks.
+    0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,  0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+    0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,  0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+    0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,  0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+    0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,  0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+    1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,  9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,
+    7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,  7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,
+    8,8,2,2,2,2,2,2,2,2,2,2,2,2,2,2,  2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,
+    10,3,3,3,3,3,3,3,3,3,3,3,3,4,3,3, 11,6,6,6,5,8,8,8,8,8,8,8,8,8,8,8,
+
+    // The second part is a transition table that maps a combination
+    // of a state of the automaton and a character class to a state.
+    0,12,24,36,60,96,84,12,12,12,48,72, 12,12,12,12,12,12,12,12,12,12,12,12,
+    12, 0,12,12,12,12,12, 0,12, 0,12,12, 12,24,12,12,12,12,12,24,12,24,12,12,
+    12,12,12,12,12,12,12,24,12,12,12,12, 12,24,12,12,12,12,12,12,12,24,12,12,
+    12,12,12,12,12,12,12,36,12,36,12,12, 12,36,12,12,12,12,12,36,12,36,12,12,
+    12,36,12,12,12,12,12,12,12,12,12,12, 
+};
+
+static unsigned inline decode(unsigned* state, unsigned* codep, unsigned byte) {
+    unsigned type = utf8d[byte];
+
+    *codep = (*state != UTF8_ACCEPT) ?
+        (byte & 0x3fu) | (*codep << 6) :
+    (0xff >> type) & (byte);
+
+    *state = utf8d[256 + *state + type];
+    return *state;
+}
+
+//static bool IsUTF8(unsigned char* s) {
+//  unsigned codepoint, state = 0;
+//
+//  while (*s)
+//      decode(&state, &codepoint, *s++);
+//
+//  return state == UTF8_ACCEPT;
+//}
+
+TEST(EncodingsTest, UTF8) {
+    StringBuffer os, os2;
+    for (const unsigned* range = kCodepointRanges; *range != 0xFFFFFFFF; range += 2) {
+        for (unsigned codepoint = range[0]; codepoint <= range[1]; ++codepoint) {
+            os.Clear();
+            UTF8<>::Encode(os, codepoint);
+            const char* encodedStr = os.GetString();
+
+            // Decode with Hoehrmann
+            {
+                unsigned decodedCodepoint = 0;
+                unsigned state = 0;
+
+                unsigned decodedCount = 0;
+                for (const char* s = encodedStr; *s; ++s)
+                    if (!decode(&state, &decodedCodepoint, static_cast<unsigned char>(*s))) {
+                        EXPECT_EQ(codepoint, decodedCodepoint);
+                        decodedCount++;
+                    }
+
+                if (*encodedStr) {                  // This decoder cannot handle U+0000
+                    EXPECT_EQ(1u, decodedCount);    // Should only contain one code point
+                }
+
+                EXPECT_EQ(UTF8_ACCEPT, state);
+                if (UTF8_ACCEPT != state)
+                    std::cout << std::hex << codepoint << " " << decodedCodepoint << std::endl;
+            }
+
+            // Decode
+            {
+                StringStream is(encodedStr);
+                unsigned decodedCodepoint;
+                bool result = UTF8<>::Decode(is, &decodedCodepoint);
+                EXPECT_TRUE(result);
+                EXPECT_EQ(codepoint, decodedCodepoint);
+                if (!result || codepoint != decodedCodepoint)
+                    std::cout << std::hex << codepoint << " " << decodedCodepoint << std::endl;
+            }
+
+            // Validate
+            {
+                StringStream is(encodedStr);
+                os2.Clear();
+                bool result = UTF8<>::Validate(is, os2);
+                EXPECT_TRUE(result);
+                EXPECT_EQ(0, StrCmp(encodedStr, os2.GetString()));
+            }
+        }
+    }
+}
+
+TEST(EncodingsTest, UTF16) {
+    GenericStringBuffer<UTF16<> > os, os2;
+    GenericStringBuffer<UTF8<> > utf8os;
+    for (const unsigned* range = kCodepointRanges; *range != 0xFFFFFFFF; range += 2) {
+        for (unsigned codepoint = range[0]; codepoint <= range[1]; ++codepoint) {
+            os.Clear();
+            UTF16<>::Encode(os, codepoint);
+            const UTF16<>::Ch* encodedStr = os.GetString();
+
+            // Encode with Hoehrmann's code
+            if (codepoint != 0) // cannot handle U+0000
+            {
+                // encode with UTF8<> first
+                utf8os.Clear();
+                UTF8<>::Encode(utf8os, codepoint);
+
+                // transcode from UTF8 to UTF16 with Hoehrmann's code
+                unsigned decodedCodepoint = 0;
+                unsigned state = 0;
+                UTF16<>::Ch buffer[3], *p = &buffer[0];
+                for (const char* s = utf8os.GetString(); *s; ++s) {
+                    if (!decode(&state, &decodedCodepoint, static_cast<unsigned char>(*s)))
+                        break;
+                }
+
+                if (codepoint <= 0xFFFF)
+                    *p++ = static_cast<UTF16<>::Ch>(decodedCodepoint);
+                else {
+                    // Encode code points above U+FFFF as surrogate pair.
+                    *p++ = static_cast<UTF16<>::Ch>(0xD7C0 + (decodedCodepoint >> 10));
+                    *p++ = static_cast<UTF16<>::Ch>(0xDC00 + (decodedCodepoint & 0x3FF));
+                }
+                *p++ = '\0';
+
+                EXPECT_EQ(0, StrCmp(buffer, encodedStr));
+            }
+
+            // Decode
+            {
+                GenericStringStream<UTF16<> > is(encodedStr);
+                unsigned decodedCodepoint;
+                bool result = UTF16<>::Decode(is, &decodedCodepoint);
+                EXPECT_TRUE(result);
+                EXPECT_EQ(codepoint, decodedCodepoint);         
+                if (!result || codepoint != decodedCodepoint)
+                    std::cout << std::hex << codepoint << " " << decodedCodepoint << std::endl;
+            }
+
+            // Validate
+            {
+                GenericStringStream<UTF16<> > is(encodedStr);
+                os2.Clear();
+                bool result = UTF16<>::Validate(is, os2);
+                EXPECT_TRUE(result);
+                EXPECT_EQ(0, StrCmp(encodedStr, os2.GetString()));
+            }
+        }
+    }
+}
+
+TEST(EncodingsTest, UTF32) {
+    GenericStringBuffer<UTF32<> > os, os2;
+    for (const unsigned* range = kCodepointRanges; *range != 0xFFFFFFFF; range += 2) {
+        for (unsigned codepoint = range[0]; codepoint <= range[1]; ++codepoint) {
+            os.Clear();
+            UTF32<>::Encode(os, codepoint);
+            const UTF32<>::Ch* encodedStr = os.GetString();
+
+            // Decode
+            {
+                GenericStringStream<UTF32<> > is(encodedStr);
+                unsigned decodedCodepoint;
+                bool result = UTF32<>::Decode(is, &decodedCodepoint);
+                EXPECT_TRUE(result);
+                EXPECT_EQ(codepoint, decodedCodepoint);         
+                if (!result || codepoint != decodedCodepoint)
+                    std::cout << std::hex << codepoint << " " << decodedCodepoint << std::endl;
+            }
+
+            // Validate
+            {
+                GenericStringStream<UTF32<> > is(encodedStr);
+                os2.Clear();
+                bool result = UTF32<>::Validate(is, os2);
+                EXPECT_TRUE(result);
+                EXPECT_EQ(0, StrCmp(encodedStr, os2.GetString()));
+            }
+        }
+    }
+}
+
+TEST(EncodingsTest, ASCII) {
+    StringBuffer os, os2;
+    for (unsigned codepoint = 0; codepoint < 128; codepoint++) {
+        os.Clear();
+        ASCII<>::Encode(os, codepoint);
+        const ASCII<>::Ch* encodedStr = os.GetString();
+        {
+            StringStream is(encodedStr);
+            unsigned decodedCodepoint;
+            bool result = ASCII<>::Decode(is, &decodedCodepoint);
+            if (!result || codepoint != decodedCodepoint)
+                std::cout << std::hex << codepoint << " " << decodedCodepoint << std::endl;
+        }
+
+        // Validate
+        {
+            StringStream is(encodedStr);
+            os2.Clear();
+            bool result = ASCII<>::Validate(is, os2);
+            EXPECT_TRUE(result);
+            EXPECT_EQ(0, StrCmp(encodedStr, os2.GetString()));
+        }
+    }
+}