unicode.cpp

Go to the documentation of this file.

// Unicode.cpp : Defines the entry point for the console application.
//

// Includes
//#include "unicode.h"

//-----------------------------------------------------------------------------
// Private declarations of this module
//-----------------------------------------------------------------------------

namespace {

class TVectorBuilder2
{
public:
        TIntV v;
        TVectorBuilder2(int i) { v.Add(i); }
        operator TIntV() const { return v; }
        TVectorBuilder2& operator ,(int i) { v.Add(i); return *this; }
};

class TVectorBuilder
{
public:
        operator TIntV() const { return TIntV(); }
        TVectorBuilder2 operator ,(int i) { return TVectorBuilder2(i); }
};

TVectorBuilder VB;

TStr CombinePath(const TStr& s, const TStr& t)
{
        int n = s.Len(); if (n <= 0) return t;
        if (s[n - 1] == '\\' || s[n - 1] == '/' || s[n - 1] == ':') return s + t;
        return s + "\\" + t;
}

void AssertEq(const TIntV& v1, const TIntV& v2, const TStr& explanation, FILE *f)
{
        const int n = v1.Len();
        bool ok = (n == v2.Len());
        if (ok) for (int i = 0; i < n && ok; i++) ok = ok && (v1[i] == v2[i]);
        if (! ok)
        {
                if (! f) f = stderr;
                fprintf(f, "%s: [", explanation.CStr());
                for (int i = 0; i < v1.Len(); i++) fprintf(f, "%s%04x", (i == 0 ? "" : " "), int(v1[i]));
                fprintf(f, "] != [");
                for (int i = 0; i < v2.Len(); i++) fprintf(f, "%s%04x", (i == 0 ? "" : " "), int(v2[i]));
                fprintf(f, "]\n");
                Fail;
        }
}

};

//-----------------------------------------------------------------------------
// TUniCodec -- miscellaneous declarations
//-----------------------------------------------------------------------------

uint TUniCodec::GetRndUint(TRnd& rnd)
{
        uint u = rnd.GetUniDevUInt(256) & 0xff;
        u <<= 8; u |= (rnd.GetUniDevUInt(256) & 0xff);
        u <<= 8; u |= (rnd.GetUniDevUInt(256) & 0xff);
        u <<= 8; u |= (rnd.GetUniDevUInt(256) & 0xff);
        return u;
}

uint TUniCodec::GetRndUint(TRnd& rnd, uint minVal, uint maxVal)
{
        if (minVal == TUInt::Mn && maxVal == TUInt::Mx) return GetRndUint(rnd);
        uint range = maxVal - minVal + 1;
        if (range > (uint(1) << (8 * sizeof(uint) - 1)))
                while (true) { uint u = GetRndUint(rnd); if (u < range) return minVal + u; }
        uint mask = 1;
        while (mask < range) mask <<= 1;
        mask -= 1;
        while (true) { uint u = GetRndUint(rnd) & mask; if (u < range) return minVal + u; }
}

bool TUniCodec::IsMachineLittleEndian()
{
        static bool isLE, initialized = false;
        if (initialized) return isLE;
        int i = 0x0201;
        char *p = (char *) (&i);
        char c1, c2;
        memcpy(&c1, p, 1); memcpy(&c2, p + 1, 1);
        if (c1 == 1 && c2 == 2) isLE = true;
        else if (c1 == 2 && c2 == 1) isLE = false;
        else {
                FailR(("TUniCodec::IsMachineLittleEndian: c1 = " + TInt::GetStr(int(uchar(c1)), "%02x") + ", c2 = " + TInt::GetStr(int(uchar(c2)), "%02x") + ".").CStr());
                isLE = true; }
        initialized = true; return isLE;
}

//-----------------------------------------------------------------------------
// TUniCodec -- UTF-8 test driver
//-----------------------------------------------------------------------------

void TUniCodec::TestUtf8(bool decode, size_t expectedRetVal, bool expectedThrow, const TIntV& src, const TIntV& expectedDest, FILE *f)
{
        TIntV dest;
        if (f) {
                fprintf(f, "Settings: %s  %s  %s   replacementChar = %x\n",
                        (errorHandling == uehAbort ? "abort" : errorHandling == uehThrow ? "throw" : errorHandling == uehIgnore ? "ignore" : errorHandling == uehReplace ? "replace" : "????"),
                        (strict ? "STRICT" : ""), (skipBom ? "skipBom" : ""), uint(replacementChar));
                fprintf(f, "src: "); for (int i = 0; i < src.Len(); i++) fprintf(f, (decode ? " %02x" : " %x"), uint(src[i])); }
        try
        {
                size_t retVal = (decode ? DecodeUtf8(src, 0, src.Len(), dest, true) : EncodeUtf8(src, 0, src.Len(), dest, true));
                if (f) {
                        fprintf(f, "\n -> dest:    "); for (int i = 0; i < dest.Len(); i++) fprintf(f, (decode ? " %x" :  " %02x"), uint(dest[i]));
                        fprintf(f, "\n    expDest  "); for (int i = 0; i < expectedDest.Len(); i++) fprintf(f, (decode ? " %x" :  " %02x"), uint(expectedDest[i]));
                        fprintf(f, "\n    retVal = %llu (expected %llu)\n", static_cast<long long unsigned int> (retVal), static_cast<long long unsigned int> (expectedRetVal)); }
                if (retVal != expectedRetVal)
                        printf("!!!");
                IAssert(retVal == expectedRetVal); IAssert(! expectedThrow);
                if (dest.Len() != expectedDest.Len())
                        printf("!!!");
                IAssert(dest.Len() == expectedDest.Len());
                for (int i = 0; i < dest.Len(); i++) IAssert(dest[i] == expectedDest[i]);
        }
        catch (TUnicodeException e)
        {
                if (f) {
                        fprintf(f, "\n -> expDest  "); for (int i = 0; i < expectedDest.Len(); i++) fprintf(f, " %x", uint(expectedDest[i]));
                        fprintf(f, "\n    exception \"%s\" at %d (char 0x%02x)\n", e.message.CStr(), int(e.srcIdx), uint(e.srcChar)); }
                IAssert(expectedThrow);
        }
}

// Generates a random UTF-8-encoded stream according to the specifications in 'testCaseDesc',
// then calls TestUtf8 to make sure that DecodeUtf8 reacts as expected.
void TUniCodec::TestDecodeUtf8(TRnd& rnd, const TStr& testCaseDesc)
{
        TIntV src; TIntV expectedDest; int expectedRetVal = 0;
        bool expectedAbort = false;
        FILE *f = 0; // stderr
        // testCaseDesc should consist of pairs or triples of characters, 'cd[e]', where:
        // - 'c' defines the range from which the codepoint should be taken ('A'..'H', 'X'..'Z');
        // - 'd' defines how many bytes the codepoint should be encoded with ('1'..'6');
        // - 'e' defines how many bytes will be removed from the end of the encoded sequence for this codepoint.
        //   (absent = 0, 'a' = 1, 'b' = 2 and so on).
        for (int i = 0; i < testCaseDesc.Len(); )
        {
                IAssert(i + 2 <= testCaseDesc.Len());
                const char c = testCaseDesc[i], d = testCaseDesc[i + 1]; i += 2;
                uint cp = 0; int nBytes = -1, minBytes = -1; bool eighties = false;
                IAssert('1' <= d && d <= '6'); nBytes = d - '0';
                if (c == 'A') { cp = GetRndUint(rnd, 0u, 0x7fu); minBytes = 1; } // 1 byte
                else if (c == 'B') { cp = GetRndUint(rnd, 0x80u, 0x7ffu); minBytes = 2; } // 2 bytes
                else if (c == 'C') { cp = GetRndUint(rnd, 0x800u, 0xffffu); minBytes = 3; } // 3 bytes
                else if (c == 'D') { cp = GetRndUint(rnd, 0x10000u, 0x10ffffu); minBytes = 4; } // 4 bytes, valid Unicode
                else if (c == 'E') { cp = GetRndUint(rnd, 0x110000u, 0x1fffffu); minBytes = 4; } // 4 bytes, invalid Unicode
                else if (c == 'F') { cp = GetRndUint(rnd, 0x200000u, 0x3ffffffu); minBytes = 5; } // 5 bytes
                else if (c == 'G') { cp = GetRndUint(rnd, 0x4000000u, 0x7fffffffu); minBytes = 6; } // 6 bytes, 31 bits
                else if (c == 'H') { cp = GetRndUint(rnd, 0x80000000u, 0xffffffffu); minBytes = 6; } // 6 bytes, 32 bits
                else if (c == 'X') { cp = 0xfffe; minBytes = 3; }
                else if (c == 'Y') { cp = 0xfeff; minBytes = 3; }
                else if (c == 'Z') { eighties = true; minBytes = 1; } // insert several random 10xxxxxx bytes (= 0x80 | random(0..0x3f))
                else Fail;
                IAssert(nBytes >= minBytes);
                // Process 'e'.
                int nToDel = 0;
                if (i < testCaseDesc.Len()) {
                        const char e = testCaseDesc[i];
                        if (e >= 'a' && e <= 'e') { i += 1; nToDel = e - 'a' + 1; }}
                IAssert(nToDel < nBytes);
                // Will an error occur during the decoding of this codepoint?
                bool errHere = false;
                if (eighties) errHere = true;
                else if (nToDel > 0) errHere = true;
                else if (strict && (cp >= 0x10ffff || nBytes > minBytes)) errHere = true;
                // Update 'expectedDest' and 'expetedRetVal'.
                if (! expectedAbort) {
                        if (! errHere) {
                                if (src.Len() == 0 && (cp == 0xfffe || cp == 0xfeff) && skipBom) { }
                                else { expectedDest.Add(cp); expectedRetVal += 1; } }
                        else if (errorHandling == uehReplace) {
                                if (eighties) for (int j = 0; j < nBytes; j++) expectedDest.Add(replacementChar);
                                else expectedDest.Add(replacementChar); }
                        if (errHere && (errorHandling == uehAbort || errorHandling == uehThrow)) expectedAbort = true; }
                // Update 'src'.
                if (eighties) for (int j = 0; j < nBytes; j++) src.Add(GetRndUint(rnd, 0x80, 0xff));
                else if (nBytes == 1) src.Add(cp);
                else {
                        int mask = (1 << nBytes) - 1; mask <<= (8 - nBytes);
                        src.Add(mask | (uint(cp) >> (6 * (nBytes - 1))));
                        for (int j = 1; j < nBytes - nToDel; j++) src.Add(0x80 | ((cp >> (6 * (nBytes - j - 1))) & _0011_1111)); }
        }
        if (f) fprintf(f, "Test case: \"%s\"\n", testCaseDesc.CStr());
        TestUtf8(true, expectedRetVal, expectedAbort && (errorHandling == uehThrow), src, expectedDest, f);
}

void TUniCodec::TestUtf8()
{
        TIntV utf8ReplCh; EncodeUtf8((TVectorBuilder(), replacementChar).v, 0, 1, utf8ReplCh, true);
        for (int skipBom_ = 0; skipBom_ < 2; skipBom_++)
        for (int strict_ = 0; strict_ < 2; strict_++)
        for (int errMode_ = 0; errMode_ < 4; errMode_++)
        {
                strict = (strict_ == 1); errorHandling = TUnicodeErrorHandling(errMode_); skipBom = (skipBom_ == 1);
                TRnd rnd = TRnd(123);
                // Test DecodeUtf8 on various random UTF-8-encoded sequences.
                for (int i = 0; i < 10; i++)
                {
                        TestDecodeUtf8(rnd, "X3A1A2A3A4A5A6B2B3B4B5B6C3C4C5C6D4D5D6E5E6F6G6");
                        TestDecodeUtf8(rnd, "X3A5dA6d");
                        TestDecodeUtf8(rnd, "X3A1B2C3D4E4F5A1G6H6Y3X3A1");
                        TestDecodeUtf8(rnd, "X3A1B2C3D4E4F5A2G6H6Y3X3A1");
                        TestDecodeUtf8(rnd, "Y3A1B2C3D4E4F5A1G6H6Y3X3A1");
                        TestDecodeUtf8(rnd, "A1B2C3D4E4F5A1G6H6Y3X3A1");
                        TestDecodeUtf8(rnd, "G6A1A1D4E4A1B2");
                        TestDecodeUtf8(rnd, "D4A1A1C3A1B2A1B2");
                        TestDecodeUtf8(rnd, "D4A1A1C3A1B2A1B2D4a");
                        TestDecodeUtf8(rnd, "X3A1B2C3D5E4F5A1G6H6Y3X3A1");
                        TestDecodeUtf8(rnd, "X3A1B2C3D4E5F5A1G6H6Y3X3A1");
                        TestDecodeUtf8(rnd, "X3A1B2C3D4aE4F5A1G6H6Y3X3A1");
                        TestDecodeUtf8(rnd, "X3A1B2C3D4bE4F5A1G6H6Y3X3A1");
                        TestDecodeUtf8(rnd, "X3A2aA3aA4aA5aA6aB2aB3aB4aB5aB6aC3aC4aC5aC6aD4aD5aD6aE5aE6aF6aG6a");
                        TestDecodeUtf8(rnd, "X3A3bA4bA5bA6aB3bB4bB5bB6bC3bC4bC5bC6bD4bD5bD6bE5bE6bF6bG6b");
                        TestDecodeUtf8(rnd, "X3A4cA5cA6cB4cB5cB6cC4cC5cC6cD4cD5cD6cE5cE6cF6cG6c");
                        TestDecodeUtf8(rnd, "X3A5dA6dB5dB6dC5dC6dD5dD6dE5dE6dF6dG6d");
                        TestDecodeUtf8(rnd, "X3A6eB6eC6eD6eE6eF6eG6e");
                }
                // Test both DecodeUtf8 and EncodeUtf8 systematically on various characters
                // close to powers of 2.
                TIntV src, expectedDest, src2;
                expectedDest.Gen(1); src.Reserve(6); src2.Gen(1);
                for (int pow = 8; pow <= 32; pow++)
                {
                        uint uFrom, uTo;
                        if (pow == 8) uFrom = 0, uTo = 1u << pow;
                        else if (pow == 32) uFrom = TUInt::Mx - (1u << 8), uTo = TUInt::Mx;
                        else uFrom = (1u << pow) - (1u << 8), uTo = (1u << pow) + (1u << 8);
                        printf("%u..%u          \r", uFrom, uTo);
                        for (uint u = uFrom; ; u++)
                        {
                                int nBytes = 0;
                                if (u < (1u << 7)) nBytes = 1;
                                else if (u < (1u << 11)) nBytes = 2;
                                else if (u < (1u << 16)) nBytes = 3;
                                else if (u < (1u << 21)) nBytes = 4;
                                else if (u < (1u << 26)) nBytes = 5;
                                else nBytes = 6;
                                src.Gen(6, nBytes);
                                if (nBytes == 1) src[0] = u;
                                else {
                                        src[0] = (((1 << nBytes) - 1) << (8 - nBytes)) | (u >> (6 * (nBytes - 1)));
                                        for (int i = 1; i < nBytes; i++) src[i] = 0x80 | ((u >> (6 * (nBytes - i - 1))) & _0011_1111); }
                                bool err = (strict && u > 0x10ffff);
                                expectedDest.Reserve(1, 0);
                                if (err && errorHandling == uehReplace) expectedDest.Add(replacementChar);
                                else if (! err) expectedDest.Add(u);
                                int erv = (err ? 0 : 1);
                                if (skipBom && (u == 0xfeff || u == 0xfffe)) expectedDest.Clr(), erv = 0;
                                TestUtf8(true, erv, (err && errorHandling == uehThrow), src, expectedDest, 0);
                                // We can also test the UTF-8 encoder.
                                src2[0] = u;
                                if (err) {
                                        if (errorHandling == uehReplace) src = utf8ReplCh;
                                        else src.Clr(false); }
                                TestUtf8(false, (err ? 0 : 1), (err && errorHandling == uehThrow), src2, src, 0);
                                //
                                if (u == uTo) break;
                        }
                }
        }
}

//-----------------------------------------------------------------------------
// TUniCodec -- UTF-16 test driver
//-----------------------------------------------------------------------------

void TUniCodec::WordsToBytes(const TIntV& src, TIntV& dest)
{
        dest.Clr();
        bool isLE = IsMachineLittleEndian();
        for (int i = 0; i < src.Len(); i++) {
                int c = src[i] & 0xffff;
                if (isLE) { dest.Add(c & 0xff); dest.Add((c >> 8) & 0xff); }
                else { dest.Add((c >> 8) & 0xff); dest.Add(c & 0xff); } }
}

void TUniCodec::TestUtf16(bool decode, size_t expectedRetVal, bool expectedThrow, const TIntV& src, const TIntV& expectedDest,
        const TUtf16BomHandling bomHandling, const TUniByteOrder defaultByteOrder, const bool insertBom,
        FILE *f)
{
        TIntV srcBytes, expectedDestBytes;
        WordsToBytes(src, srcBytes); WordsToBytes(expectedDest, expectedDestBytes);
        TIntV dest;
        if (f) {
                fprintf(f, "Settings: %s  %s  %s  %s  %s replacementChar = %x  \n",
                        (errorHandling == uehAbort ? "abort" : errorHandling == uehThrow ? "throw" : errorHandling == uehIgnore ? "ignore" : errorHandling == uehReplace ? "replace" : "????"),
                        (strict ? "STRICT" : ""), (decode ? (skipBom ? "skipBom" : "") : (insertBom ? "insrtBom" : "")),
                        (bomHandling == bomAllowed ? "bomAllowed" : bomHandling == bomRequired ? "bomRequired" : "bomIgnored"),
                        (defaultByteOrder == boBigEndian ? "boBigEndian" : defaultByteOrder == boLittleEndian ? "boLittleEndian" : "boMachineEndian"),
                        uint(replacementChar));
                fprintf(f, "src: "); for (int i = 0; i < src.Len(); i++) fprintf(f, (decode ? " %04x" : " %x"), uint(src[i])); }
        for (int useBytes = 0; useBytes < 2; useBytes++)
        {
                const char *fmt = (useBytes ? " %02x" : " %04x");
                try
                {
                        dest.Clr();
                        size_t retVal;
                        if (! useBytes) {
                                if (decode) retVal = DecodeUtf16FromWords(src, 0, src.Len(), dest, true, bomHandling, defaultByteOrder);
                                else retVal = EncodeUtf16ToWords(src, 0, src.Len(), dest, true, insertBom, defaultByteOrder); }
                        else {
                                if (decode) retVal = DecodeUtf16FromBytes(srcBytes, 0, srcBytes.Len(), dest, true, bomHandling, defaultByteOrder);
                                else retVal = EncodeUtf16ToBytes(src, 0, src.Len(), dest, true, insertBom, defaultByteOrder); }
                        const TIntV& ed = (useBytes && ! decode ? expectedDestBytes : expectedDest);
                        if (f) {
                                fprintf(f, "\n -> dest:    "); for (int i = 0; i < dest.Len(); i++) fprintf(f, (decode ? " %x" :  fmt), uint(dest[i]));
                                fprintf(f, "\n    expDest  "); for (int i = 0; i < ed.Len(); i++) fprintf(f, (decode ? " %x" :  fmt), uint(ed[i]));
                                fprintf(f, "\n    retVal = %llu (expected %llu)\n", static_cast<long long unsigned int> (retVal), static_cast<long long unsigned int> (expectedRetVal)); }
                        bool ok = true;
                        if (retVal != expectedRetVal) ok = false;
                        if (dest.Len() != ed.Len()) ok = false;
                        if (ok) for (int i = 0; i < dest.Len(); i++) if (dest[i] != ed[i]) ok = false;
                        if (! ok)
                        {
                                printf("!!!\n");
                        }
                        IAssert(retVal == expectedRetVal); IAssert(! expectedThrow);
                        IAssert(dest.Len() == ed.Len());
                        for (int i = 0; i < dest.Len(); i++) IAssert(dest[i] == ed[i]);
                }
                catch (TUnicodeException e)
                {
                        if (f) {
                                fprintf(f, "\n -> expDest  "); for (int i = 0; i < expectedDest.Len(); i++) fprintf(f, (decode ? " %x" : fmt), uint(expectedDest[i]));
                                fprintf(f, "\n    exception \"%s\" at %d (char 0x%02x)\n", e.message.CStr(), int(e.srcIdx), uint(e.srcChar)); }
                        IAssert(expectedThrow);
                }
        }
}

// Generates a random UTF-16-encoded stream according to the specifications in 'testCaseDesc',
// then calls TestUtf16 to make sure that DecodeUtf16 reacts as expected.
void TUniCodec::TestDecodeUtf16(TRnd& rnd, const TStr& testCaseDesc,
        const TUtf16BomHandling bomHandling,
        const TUniByteOrder defaultByteOrder,
        const bool insertBom)
{
        TIntV src; TIntV expectedDest; int expectedRetVal = 0;
        bool expectedAbort = false;
        FILE *f = 0;
        bool isMachineLe = IsMachineLittleEndian();
        bool isDefaultLe = (defaultByteOrder == boLittleEndian || (defaultByteOrder == boMachineEndian && isMachineLe));
        bool swap = (isMachineLe != isDefaultLe);
        if (insertBom) {
                src.Add(swap ? 0xfffe : 0xfeff);
                if (! skipBom) { expectedRetVal += 1; expectedDest.Add(0xfeff); } }
        else if (bomHandling == bomRequired) {
                expectedAbort = true; expectedRetVal = -1; }
        // testCaseDesc should consist single characters or pairs of characters, 'c[e]', where:
        // - 'c' defines the range from which the codepoint should be taken ('A'..'E', 'X'..'Y');
        // - 'e' defines how many words will be removed from the end of the encoded sequence for this codepoint.
        //   (absent = 0, 'a' = 1).
        for (int i = 0; i < testCaseDesc.Len(); )
        {
                const char c = testCaseDesc[i++];
                uint cp = 0; int nWords = -1;
                if (c == 'X' || c == 'Y') IAssert(i > 1); // if you want a BOM at the beginning of your data, use insertBom -- if we permit X and Y here, predicting the expectedDest and expectedRetVal gets more complicated
                if (c == 'A') { cp = GetRndUint(rnd, 0u, Utf16FirstSurrogate - 1); nWords = 1; } // characters below the first surrogate range
                else if (c == 'B') { cp = GetRndUint(rnd, Utf16FirstSurrogate, Utf16FirstSurrogate + 1023); nWords = 1; } // the first surrogate range
                else if (c == 'C') { cp = GetRndUint(rnd, Utf16SecondSurrogate, Utf16SecondSurrogate + 1023); nWords = 1; } // the second surrogate range
                else if (c == 'D') { do { cp = GetRndUint(rnd, Utf16SecondSurrogate + 1024, 0xffffu); } while (cp == 0xfffe || cp == 0xfeff); nWords = 1; } // above the second surrogate range, but still in the BMP
                else if (c == 'E') { cp = GetRndUint(rnd, 0x10000u, 0x10ffffu); nWords = 2; } // above the BMP, but still within the range for UTF-16
                else if (c == 'X') { cp = 0xfffe; nWords = 1; }
                else if (c == 'Y') { cp = 0xfeff; nWords = 1; }
                else Fail;
                if (c == 'B' && i < testCaseDesc.Len()) IAssert(testCaseDesc[i] != 'C');
                // Process 'e'.
                int nToDel = 0;
                if (i < testCaseDesc.Len()) {
                        const char e = testCaseDesc[i];
                        if (e >= 'a') { i += 1; nToDel = 1; }}
                IAssert((nWords == 1 && nToDel == 0) || (nWords == 2 && (nToDel == 0 || nToDel == 1)));
                if (nWords == 2 && nToDel == 1 && i < testCaseDesc.Len()) IAssert(testCaseDesc[i] != 'C');
                // Will an error occur during the decoding of this codepoint?
                bool errHere = false;
                if (Utf16FirstSurrogate <= cp && cp <= Utf16FirstSurrogate + 1023) errHere = true;
                else if (cp > 0x10ffff) { Fail; errHere = true; }
                else if (nToDel > 0) errHere = true;
                else if (strict && (Utf16SecondSurrogate <= cp && cp <= Utf16SecondSurrogate + 1023)) errHere = true;
                // Update 'expectedDest' and 'expectedRetVal'.
                if (! expectedAbort) {
                        if (! errHere) {
                                if (src.Len() == 0 && (cp == 0xfffe || cp == 0xfeff) && skipBom) { }
                                else { expectedDest.Add(cp); expectedRetVal += 1; } }
                        else if (errorHandling == uehReplace) {
                                expectedDest.Add(replacementChar); }
                        if (errHere && (errorHandling == uehAbort || errorHandling == uehThrow)) expectedAbort = true; }
                // Update 'src'.
                if (nWords == 1) src.Add(swap ? SwapBytes(cp) : cp);
                else {
                        int c1 = ((cp - 0x10000) >> 10) & 1023; c1 += Utf16FirstSurrogate;
                        int c2 = (cp - 0x10000) & 1023; c2 += Utf16SecondSurrogate;
                        src.Add(swap ? SwapBytes(c1) : c1);
                        if (nToDel == 0) src.Add(swap ? SwapBytes(c2) : c2); }
        }
        if (f) fprintf(f, "Test case: \"%s\"\n", testCaseDesc.CStr());
        TestUtf16(true, expectedRetVal, expectedAbort && (errorHandling == uehThrow), src, expectedDest, bomHandling, defaultByteOrder, false, f);
}

void TUniCodec::TestUtf16()
{
        TIntV utf16ReplCh; utf16ReplCh.Add(replacementChar);
        for (int skipBom_ = 0; skipBom_ < 2; skipBom_++)
        for (int strict_ = 0; strict_ < 2; strict_++)
        for (int errMode_ = 0; errMode_ < 4; errMode_++)
        for (int bomHandling_ = 0; bomHandling_ < 3; bomHandling_++)
        for (int byteOrder_ = 0; byteOrder_ < 3; byteOrder_++)
        for (int insertBom_ = 0; insertBom_ < 2; insertBom_++)
        {
                strict = (strict_ == 1); errorHandling = TUnicodeErrorHandling(errMode_); skipBom = (skipBom_ == 1);
                bool insertBom = (insertBom_ == 1);
                TUniByteOrder byteOrder = (TUniByteOrder) byteOrder_;
                TUtf16BomHandling bomHandling = (TUtf16BomHandling) bomHandling_;
                TRnd rnd = TRnd(123);
                // Test DecodeUtf16 on various random UTF-16-encoded sequences.
                for (int i = 0; i < 10; i++)
                {
                        TestDecodeUtf16(rnd, "A", bomHandling, byteOrder, insertBom);
                        TestDecodeUtf16(rnd, "AAA", bomHandling, byteOrder, insertBom);
                        TestDecodeUtf16(rnd, "B", bomHandling, byteOrder, insertBom);
                        TestDecodeUtf16(rnd, "DDAADADAAADDDAA", bomHandling, byteOrder, insertBom);
                        TestDecodeUtf16(rnd, "DEEEDAAEEDADEEAAEEADEEDDAA", bomHandling, byteOrder, insertBom);
                        TestDecodeUtf16(rnd, "DEaEaEDAAEaEDADEaEAAEEADEEDDAA", bomHandling, byteOrder, insertBom);
                        TestDecodeUtf16(rnd, "CABDEBACCEaB", bomHandling, byteOrder, insertBom);
                        TestDecodeUtf16(rnd, "EaEEEEaBBACABXABYXXEaYDDXBDCEA", bomHandling, byteOrder, insertBom);
                        TestDecodeUtf16(rnd, "EaEEEEaBDCAAXADYXXEaYDDXDCEA", bomHandling, byteOrder, insertBom);
                }
                //continue;
                // Test both DecodeUtf16 and EncodeUtf16 systematically on various characters
                // close to powers of 2.
                TIntV src, expectedDest, src2;
                expectedDest.Gen(1); src.Reserve(6); src2.Gen(1);
                for (int pow = 8; pow <= 32; pow++)
                {
                        uint uFrom, uTo;
                        if (pow == 8) uFrom = 0, uTo = 1u << pow;
                        else if (pow == 32) uFrom = TUInt::Mx - (1u << 8), uTo = TUInt::Mx;
                        else uFrom = (1u << pow) - (1u << 8), uTo = (1u << pow) + (1u << 8);
                        printf("%u..%u          \r", uFrom, uTo);
                        for (uint u = uFrom; ; u++)
                        {
                                int nWords = 0;
                                if (u < 0x10000) nWords = 1;
                                else nWords = 2;
                                bool isMachineLe = IsMachineLittleEndian(), isDestLe = (byteOrder == boLittleEndian || (byteOrder == boMachineEndian && isMachineLe));
                                bool swap = (isMachineLe != isDestLe);
                                bool err = (u > 0x10ffff) || (Utf16FirstSurrogate <= u && u <= Utf16FirstSurrogate + 1023) || (strict && Utf16SecondSurrogate <= u && u <= Utf16SecondSurrogate + 1023);
                                src.Gen(3, (err ? 0 : nWords) + (insertBom ? 1 : 0));
                                if (insertBom) src[0] = (swap ? 0xfffe : 0xfeff);
                                if (! ((u > 0x10ffff) || (Utf16FirstSurrogate <= u && u <= Utf16FirstSurrogate + 1023)))
                                {
                                        // Try to encode 'u' and see if it gets decoded correctly.
                                        if (nWords == 1) src[insertBom ? 1 : 0] = (swap ? SwapBytes(u) : u);
                                        else {
                                                int u1 = Utf16FirstSurrogate + (((u - 0x10000) >> 10) & 1023);
                                                int u2 = Utf16SecondSurrogate + ((u - 0x10000) & 1023);
                                                src[insertBom ? 1 : 0] = (swap ? SwapBytes(u1) : u1);
                                                src[insertBom ? 2 : 1] = (swap ? SwapBytes(u2) : u2); }
                                        if (! ((u == 0xfffe || u == 0xfeff) && bomHandling == bomAllowed && ! insertBom)) // this will just create a mess when decoding
                                        {
                                                expectedDest.Reserve(2, 0);
                                                if (insertBom && ! skipBom) expectedDest.Add(0xfeff);
                                                if (err && errorHandling == uehReplace) expectedDest.Add(replacementChar);
                                                else if (! err) expectedDest.Add(u);
                                                int erv = (err ? 0 : expectedDest.Len());
                                                if (skipBom && (u == 0xfeff || u == 0xfffe) && ! insertBom) expectedDest.Clr(), erv = 0;
                                                bool errD = err;
                                                if (bomHandling == bomRequired && ! insertBom) {
                                                        expectedDest.Clr(false);
                                                        if (u == 0xfeff || u == 0xfffe) { erv = (skipBom ? 0 : 1); if (! skipBom) expectedDest.Add(0xfeff); }
                                                        else { erv = -1; errD = true;
                                                                /*if (errorHandling == uehReplace) expectedDest.Add(replacementChar);*/ }}
                                                TestUtf16(true, erv, (errD && errorHandling == uehThrow), src, expectedDest, bomHandling, byteOrder, insertBom, 0);
                                        }
                                }
                                // We can also test the UTF-16 encoder.
                                src2[0] = u;
                                if (err) {
                                        src.Clr(false); if (insertBom) src.Add(swap ? 0xfffe : 0xfeff);
                                        if (errorHandling == uehReplace) {
                                                src.Add(swap ? SwapBytes(replacementChar) : replacementChar);
                                                /*if (byteOrder == boBigEndian || (byteOrder == boMachineEndian && ! TUniCodec::IsMachineLittleEndian())) { src.Add((replacementChar >> 8) & 0xff); src.Add(replacementChar & 0xff); }
                                                else { src.Add(replacementChar & 0xff); src.Add((replacementChar >> 8) & 0xff); } */
                                        }}
                                TestUtf16(false, (err ? 0 : 1) + (insertBom ? 1 : 0), (err && errorHandling == uehThrow), src2, src, bomHandling, byteOrder, insertBom, 0);
                                //
                                if (u == uTo) break;
                        }
                }
        }
}

//-----------------------------------------------------------------------------
// TUniCaseFolding
//-----------------------------------------------------------------------------

void TUniCaseFolding::LoadTxt(const TStr& fileName)
{
        Clr();
        TUniChDb::TUcdFileReader reader; reader.Open(fileName);
        TStrV fields;
        while (reader.GetNextLine(fields))
        {
                int cp = reader.ParseCodePoint(fields[0]);
                const TStr status = fields[1], mapsTo = fields[2];
                if (status == "C" || status == "S" || status == "T") {
                        TIntH &dest = (status == "C" ? cfCommon : status == "S" ? cfSimple : cfTurkic);
                        IAssert(! dest.IsKey(cp));
                        int cp2 = reader.ParseCodePoint(mapsTo);
                        dest.AddDat(cp, cp2); }
                else if (status == "F") {
                        TIntIntVH &dest = cfFull;
                        IAssert(! dest.IsKey(cp));
                        TIntV cps; reader.ParseCodePointList(mapsTo, cps); IAssert(cps.Len() > 0);
                        dest.AddDat(cp, cps); }
                else
                        FailR(status.CStr());
        }
        printf("TUniCaseFolding(\"%s\"): %d common, %d simple, %d full, %d Turkic.\n",
                fileName.CStr(), cfCommon.Len(), cfSimple.Len(), cfFull.Len(), cfTurkic.Len());
}

void TUniCaseFolding::Test(const TIntV& src, const TIntV& expectedDest, const bool full, const bool turkic, FILE *f)
{
        fprintf(f, "TUniCaseFolding(%s%s): ", (full ? "full" : "simple"), (turkic ? ", turkic" : ""));
        for (int i = 0; i < src.Len(); i++) fprintf(f, " %04x", int(src[i]));
        TIntV dest; Fold(src, 0, src.Len(), dest, true, full, turkic);
        fprintf(f, "\n  -> ");
        for (int i = 0; i < dest.Len(); i++) fprintf(f, " %04x", int(dest[i]));
        fprintf(f, "\n");
        IAssert(dest.Len() == expectedDest.Len());
        for (int i = 0; i < dest.Len(); i++) IAssert(dest[i] == expectedDest[i]);
}

/*
void TUniCaseFolding::Test(const TIntV& src, FILE *f) {
        Test(src, false, false, f); Test(src, false, true, f);
        Test(src, true, false, f); Test(src, true, true, f); }
*/

void TUniCaseFolding::Test()
{
        FILE *f = stderr;
        TVectorBuilder VB;
        // simple
        Test((VB, 0x41, 0x62, 0x49, 0x43, 0xdf), (VB, 0x61, 0x62, 0x69, 0x63, 0xdf), false, false, f);
        // simple + turkic
        Test((VB, 0x41, 0x62, 0x49, 0x43, 0xdf), (VB, 0x61, 0x62, 0x131, 0x63, 0xdf), false, true, f);
        // full
        Test((VB, 0x41, 0x62, 0x49, 0x43, 0xdf), (VB, 0x61, 0x62, 0x69, 0x63, 0x73, 0x73), true, false, f);
        // full + turkic
        Test((VB, 0x41, 0x62, 0x49, 0x43, 0xdf), (VB, 0x61, 0x62, 0x131, 0x63, 0x73, 0x73), true, true, f);
}

//-----------------------------------------------------------------------------
// TUniChInfo
//-----------------------------------------------------------------------------

// UAX #14
const ushort TUniChInfo::LineBreak_Unknown = TUniChInfo::GetLineBreakCode('X', 'X');
const ushort TUniChInfo::LineBreak_ComplexContext = TUniChInfo::GetLineBreakCode('S', 'A');
const ushort TUniChInfo::LineBreak_Numeric = TUniChInfo::GetLineBreakCode('N', 'U');
const ushort TUniChInfo::LineBreak_InfixNumeric = TUniChInfo::GetLineBreakCode('I', 'S');
const ushort TUniChInfo::LineBreak_Quotation = TUniChInfo::GetLineBreakCode('Q', 'U');

//-----------------------------------------------------------------------------
// TUniChDb -- word breaking
//-----------------------------------------------------------------------------

// Test driver for WbFind*NonIgnored.
void TUniChDb::TestWbFindNonIgnored(const TIntV& src) const
{
        int n = src.Len();
        TBoolV isIgnored; isIgnored.Gen(n);
        for (int i = 0; i < n; i++) isIgnored[i] = IsWbIgnored(src[i]);
        TIntV prevNonIgnored, nextNonIgnored, curOrNextNonIgnored;
        prevNonIgnored.Gen(n); nextNonIgnored.Gen(n); curOrNextNonIgnored.Gen(n);
        FILE *f = 0; // stderr;
        for (int srcIdx = 0; srcIdx < n; srcIdx++) for (int srcLen = 1; srcLen < n - srcIdx; srcLen++)
        {
                int prev = -1;
                for (int i = 0; i < srcLen; i++) {
                        prevNonIgnored[i] = prev;
                        if (! isIgnored[srcIdx + i]) prev = srcIdx + i; }
                int next = srcIdx + srcLen;
                for (int i = srcLen - 1; i >= 0; i--) {
                        nextNonIgnored[i] = next;
                        if (! isIgnored[srcIdx + i]) next = srcIdx + i;
                        curOrNextNonIgnored[i] = next; }
                if (f) {
                        fprintf(f, "\nIndex:     "); for (int i = 0; i < srcLen; i++) fprintf(f, " %2d", srcIdx + i);
                        fprintf(f, "\nNonIgn:    "); for (int i = 0; i < srcLen; i++) fprintf(f, " %s", (isIgnored[srcIdx + i] ? " ." : " Y"));
                        fprintf(f, "\nPrevNI:    "); for (int i = 0; i < srcLen; i++) fprintf(f, " %2d", int(prevNonIgnored[i]));
                        fprintf(f, "\nNextNI:    "); for (int i = 0; i < srcLen; i++) fprintf(f, " %2d", int(nextNonIgnored[i]));
                        fprintf(f, "\nCurNextNI: "); for (int i = 0; i < srcLen; i++) fprintf(f, " %2d", int(curOrNextNonIgnored[i]));
                        fprintf(f, "\n"); }
                for (int i = 0; i < srcLen; i++)
                {
                        size_t s;
                        s = size_t(srcIdx + i); WbFindNextNonIgnored(src, s, size_t(srcIdx + srcLen));
                        IAssert(s == size_t(nextNonIgnored[i]));
                        s = size_t(srcIdx + i); WbFindCurOrNextNonIgnored(src, s, size_t(srcIdx + srcLen));
                        IAssert(s == size_t(curOrNextNonIgnored[i]));
                        s = size_t(srcIdx + i); bool ok = WbFindPrevNonIgnored(src, size_t(srcIdx), s);
                        if (prevNonIgnored[i] < 0) { IAssert(! ok); IAssert(s == size_t(srcIdx)); }
                        else { IAssert(ok); IAssert(s == size_t(prevNonIgnored[i])); }
                }
        }
}

void TUniChDb::TestWbFindNonIgnored() const
{
        TIntV chIgnored, chNonIgnored;
        FILE *f = 0; // stderr;
        for (int i = h.FFirstKeyId(); h.FNextKeyId(i); ) {
                const int cp = h.GetKey(i); const TUniChInfo& ci = h[i];
                if (f) fprintf(f, "%04x: flags %08x props %08x %08x script \"%s\"\n", cp,
                        ci.flags, ci.properties, ci.propertiesX, GetScriptName(ci.script).CStr());
                (IsWbIgnored(h[i]) ? chIgnored : chNonIgnored).Add(h.GetKey(i));
        }
        chIgnored.Sort(); chNonIgnored.Sort();
        printf("TUniChDb::TestWbNonIgnored: %d ignored, %d nonignored chars.\n", chIgnored.Len(), chNonIgnored.Len());
        TRnd rnd = TRnd(123);
        for (int iter = 0; iter <= 50; iter++)
        {
                int percIgnored = 2 * iter;
                for (int n = 0; n <= 20; n++)
                {
                        // Prepare a random sequence of 'n' codepoints.
                        TIntV v; v.Gen(n);
                        for (int i = 0; i < n; i++) {
                                TIntV& chars = (rnd.GetUniDevInt(100) < percIgnored) ? chIgnored : chNonIgnored;
                                int j = rnd.GetUniDevInt(chars.Len());
                                v.Add(chars[j]); }
                        // Run the tests with this sequence.
                        TestWbFindNonIgnored(v);
                }
        }
}

void TUniChDb::TestFindNextWordOrSentenceBoundary(const TStr& basePath, bool sentence)
{
        TUcdFileReader reader; TStrV fields;
        reader.Open(CombinePath(CombinePath(basePath, GetAuxiliaryDir()), (sentence ? GetSentenceBreakTestFn() : GetWordBreakTestFn())));
        int nLines = 0; TRnd rnd = TRnd(123);
        while (reader.GetNextLine(fields))
        {
                nLines += 1;
                IAssert(fields.Len() == 1);
                TStrV parts; fields[0].SplitOnWs(parts);
                const int n = parts.Len(); IAssert((n % 2) == 1);
                TIntV chars; TBoolV isBreak, isPredicted, isPredicted2;
                // Each line is a sequence of codepoints, with a \times or \div in between each
                // pair of codepoints (as well as at the beginning and the end of the sequence) to
                // indicate whether a boundary exists there or not.
                for (int i = 0; i < n; i++)
                {
                        const TStr& s = parts[i];
                        if ((i % 2) == 0) {
                                if (s == "\xc3\x97") // multiplication sign (U+00D7) in UTF-8
                                        isBreak.Add(false);
                                else if (s == "\xc3\xb7") // division sign (U+00F7) in UTF-8
                                        isBreak.Add(true);
                                else FailR(s.CStr()); }
                        else chars.Add(reader.ParseCodePoint(s));
                }
                const int m = n / 2; IAssert(chars.Len() == m); IAssert(isBreak.Len() == m + 1);
                IAssert(isBreak[0]); IAssert(isBreak[m]);
                isPredicted.Gen(m + 1); isPredicted.PutAll(false);
                if (AlwaysFalse()) { printf("%3d", nLines); for (int i = 0; i < m; i++) printf(" %04x", int(chars[i])); printf("\n"); }
                // We'll insert a few random characters at the beginning of the sequence
                // so that srcPos doesn't always begin at 0.
                for (int nBefore = 0; nBefore < 5; nBefore++)
                {
                        TIntV chars2; for (int i = 0; i < nBefore; i++) chars2.Add(0, rnd.GetUniDevInt(0x10ffff + 1));
                        chars2.AddV(chars);
                        // Use FindNextBoundary to find all the word boundaries.
                        size_t position = (nBefore > 0 ? nBefore - 1 : nBefore); size_t prevPosition = position;
                        while (sentence ? FindNextSentenceBoundary(chars2, nBefore, m, position) : FindNextWordBoundary(chars2, nBefore, m, position))
                        {
                                IAssert(prevPosition < position);
                                IAssert(position <= size_t(nBefore + m));
                                isPredicted[int(position) - nBefore] = true;
                                prevPosition = position;
                        }
                        IAssert(position == size_t(nBefore + m));
                        if (sentence) FindSentenceBoundaries(chars2, nBefore, m, isPredicted2);
                        else FindWordBoundaries(chars2, nBefore, m, isPredicted2);
                        IAssert(isPredicted2.Len() == m + 1);
                        bool ok = true;
                        // If we start at 0, the word boundary at the beginning of the sequence was
                        // not found explicitly, so we'll add it now.
                        if (nBefore == 0) isPredicted[0] = true;
                        // Compare the predicted and the true boundaries.
                        for (int i = 0; i <= m; i++) {
                                if (isBreak[i] != isPredicted[i]) ok = false;
                                IAssert(isPredicted2[i] == isPredicted[i]); }
                        FILE *f = stderr;
                        if (! ok)
                        {
                                fprintf(f, "\nError in line %d:\n", nLines);
                                fprintf(f, "True:      ");
                                for (int i = 0; i <= m; i++) {
                                        fprintf(f, "%s ", (isBreak[i] ? "|" : "."));
                                        if (i < m) fprintf(f, "%04x ", int(chars[i + nBefore])); }
                                fprintf(f, "\nPredicted: ");
                                for (int i = 0; i <= m; i++) {
                                        fprintf(f, "%s ", (isPredicted[i] ? "|" : "."));
                                        if (i < m) {
                                                const int cp = chars[i + nBefore];
                                                TStr s = sentence ? TUniChInfo::GetSbFlagsStr(GetSbFlags(cp)) : TUniChInfo::GetWbFlagsStr(GetWbFlags(cp));
                                                if (IsWbIgnored(cp)) s = "*" + s;
                                                fprintf(f, "%4s ", s.CStr()); }}
                                fprintf(f, "\n");
                                Fail;
                        }
                        // Test FindNextBoundary if we start in the middle of the sequence,
                        // i.e. not at an existing boundary.
                        for (int i = 0; i < m; i++) {
                                position = i + nBefore; bool ok = sentence ? FindNextSentenceBoundary(chars2, nBefore, m, position) : FindNextWordBoundary(chars2, nBefore, m, position);
                                IAssert(ok); // at the very least, there should be the 'boundary' at nBefore + m
                                IAssert(size_t(i + nBefore) < position); IAssert(position <= size_t(nBefore + m));
                                position -= nBefore;
                                for (int j = i + 1; j < int(position); j++)
                                        IAssert(! isBreak[j]);
                                IAssert(isBreak[int(position)]); }
                }
        }
        reader.Close();
        printf("TUniChDb::TestFindNext%sBoundary: %d lines processed.\n", (sentence ? "Sentence" : "Word"), nLines);
}

//-----------------------------------------------------------------------------
// TUniChDb -- composition and decomposition
//-----------------------------------------------------------------------------

void TUniChDb::TestComposition(const TStr& basePath)
{
        TUcdFileReader reader; TStrV fields; int nLines = 0;
        reader.Open(CombinePath(basePath, GetNormalizationTestFn()));
        bool inPart1 = false; TIntH testedInPart1;
        while (reader.GetNextLine(fields))
        {
                nLines += 1;
                if (fields.Len() == 1) {
                        IAssert(fields[0].IsPrefix("@Part"));
                        inPart1 = (fields[0] == "@Part1"); continue; }
                IAssert(fields.Len() == 6);
                IAssert(fields[5].Len() == 0);
                TIntV c1, c2, c3, c4, c5;
                reader.ParseCodePointList(fields[0], c1);
                reader.ParseCodePointList(fields[1], c2);
                reader.ParseCodePointList(fields[2], c3);
                reader.ParseCodePointList(fields[3], c4);
                reader.ParseCodePointList(fields[4], c5);
                TIntV v;
#define AssE_(v1, v2, expl) AssertEq(v1, v2, TStr(expl) + " (line " + TInt::GetStr(nLines) + ")", 0)
#define NFC_(cmpWith, operand) DecomposeAndCompose(operand, 0, operand.Len(), v, false); AssE_(cmpWith, v, #cmpWith " == NFC(" #operand ")")
#define NFD_(cmpWith, operand) Decompose(operand, 0, operand.Len(), v, false); AssE_(cmpWith, v, #cmpWith " == NFD(" #operand ")")
#define NFKC_(cmpWith, operand) DecomposeAndCompose(operand, 0, operand.Len(), v, true); AssE_(cmpWith, v, #cmpWith " == NFKC(" #operand ")")
#define NFKD_(cmpWith, operand) Decompose(operand, 0, operand.Len(), v, true); AssE_(cmpWith, v, #cmpWith " == NFKD(" #operand ")")
                // NFD:
                NFD_(c3, c1);   // c3 == NFD(c1)
                NFD_(c3, c2);   // c3 == NFD(c2)
                NFD_(c3, c3);   // c3 == NFD(c3)
                NFD_(c5, c4);   // c5 == NFD(c4)
                NFD_(c5, c5);   // c5 == NFD(c5)
                // NFC:
                NFC_(c2, c1);   // c2 == NFC(c1)
                NFC_(c2, c2);   // c2 == NFC(c2)
                NFC_(c2, c3);   // c2 == NFC(c3)
                NFC_(c4, c4);   // c4 == NFC(c4)
                NFC_(c4, c5);   // c4 == NFC(c5)
                // NFKD:
                NFKD_(c5, c1);   // c5 == NFKD(c1)
                NFKD_(c5, c2);   // c5 == NFKD(c2)
                NFKD_(c5, c3);   // c5 == NFKD(c3)
                NFKD_(c5, c4);   // c5 == NFKD(c4)
                NFKD_(c5, c5);   // c5 == NFKD(c5)
                // NFKC:
                NFKC_(c4, c1);   // c4 == NFKC(c1)
                NFKC_(c4, c2);   // c4 == NFKC(c2)
                NFKC_(c4, c3);   // c4 == NFKC(c3)
                NFKC_(c4, c4);   // c4 == NFKC(c4)
                NFKC_(c4, c5);   // c4 == NFKC(c5)
                //
                if (inPart1) {
                        IAssert(c1.Len() == 1);
                        testedInPart1.AddKey(c1[0]); }
        }
        reader.Close();
        // Test other individual codepoints that were not mentioned in part 1.
        int nOther = 0;
        for (int i = h.FFirstKeyId(); h.FNextKeyId(i); )
        {
                const int cp = h.GetKey(i), nLines = -1;
                if (testedInPart1.IsKey(cp)) continue;
                TIntV x, v; x.Add(cp);
                NFC_(x, x);    // x == NFC(x)
                NFD_(x, x);    // x == NFD(x)
                NFKC_(x, x);   // x == NFKC(x)
                NFKD_(x, x);   // x == NFKD(x)
                nOther += 1;
        }
#undef AssE_
#undef NFC_
#undef NFD_
#undef NFKC_
#undef NFKD_
        printf("TUniChDb::TestComposition: %d lines processed + %d other individual codepoints.\n", nLines, nOther);
}

//-----------------------------------------------------------------------------
// TUniChDb -- case conversion tests
//-----------------------------------------------------------------------------

void TUniChDb::TestCaseConversion(const TStr& source, const TStr& trueLc,
                                                                  const TStr& trueTc, const TStr& trueUc,
                                                                  bool turkic, bool lithuanian)
{
        TIntV src;
        TUcdFileReader::ParseCodePointList(source, src);
        FILE *f = stderr;
        for (int i = 0; i < 3; i++)
        {
                TCaseConversion how = (i == 0) ? ccLower : (i == 1) ? ccTitle : ccUpper;
                const TStr &trueDestS = (how == ccLower ? trueLc : how == ccTitle ? trueTc : trueUc);
                TIntV trueDest; TUcdFileReader::ParseCodePointList(trueDestS, trueDest);
                TIntV dest;
                GetCaseConverted(src, 0, src.Len(), dest, true, how, turkic, lithuanian);
                bool ok = (dest.Len() == trueDest.Len());
                if (ok) for (int i = 0; i < dest.Len() && ok; i++) ok = ok && (dest[i] == trueDest[i]);
                if (ok) continue;
                fprintf(f, "%s(", (how == ccLower ? "toLowercase" : how == ccTitle ? "toTitlecase" : "toUppercase"));
                for (int i = 0; i < src.Len(); i++) fprintf(f, "%s%04x", (i == 0 ? "" : " "), int(src[i]));
                fprintf(f, ")\nCorrect:   (");
                for (int i = 0; i < trueDest.Len(); i++) fprintf(f, "%s%04x", (i == 0 ? "" : " "), int(trueDest[i]));
                fprintf(f, ")\nOur output:(");
                for (int i = 0; i < dest.Len(); i++) fprintf(f, "%s%04x", (i == 0 ? "" : " "), int(dest[i]));
                fprintf(f, ")\n");
                IAssert(ok);
        }
}

void TUniChDb::TestCaseConversions()
{
        // Because no thorough case-conversion test files have been provided as part
        // of the Unicode standard, we'll have to test things on a few test cases of our own.
        // - First, test some unconditional special mappings, such as 'ss', 'ffl', 'dz', etc.
        const TStr F = "0046 ", L = "004C ", S = "0053 ", T = "0054 ", W = "0057 ";
        const TStr f = "0066 ", l = "006c ", s = "0073 ", t = "0074 ", w = "0077 ";
        const TStr ss = "00df ", ffl = "fb04 ", longs = "017f ", longst = "fb05 ", wRing = "1e98 ", Ring = "030a ";
        const TStr DZ = "01c4 ", Dz = "01c5 ", dz = "01c6 ";
        const TStr space = "0020 ", Grave = "0300 ";
        TestCaseConversion(
                F + L + s + t + space + Dz + w + T + ss + wRing + space + longs + DZ + space + dz + longst,  // source
                f + l + s + t + space + dz + w + t + ss + wRing + space + longs + dz + space + dz + longst,  // lowercase
                F + l + s + t + space + Dz + w + t + ss + wRing + space + S + dz + space + Dz + longst,      // titlecase
                F + L + S + T + space + DZ + W + T + S + S + W + Ring + space + S + DZ + space + DZ + S + T, // uppercase
                false, false);
        // - Dotted I, dotless i, etc., but with turkic == false.
        const TStr I = "0049 ", J = "004a ", i = "0069 ", j = "006a ", iDotless = "0131 ", IDot = "0130 ", DotA = "0307 ";
        TestCaseConversion(
                s + I + t + i + w + iDotless + f + IDot + l + space + iDotless + DotA + f + I + DotA + s, // source
                s + i + t + i + w + iDotless + f + i + DotA + l + space + iDotless + DotA + f + i + DotA + s, // lowercase
                S + i + t + i + w + iDotless + f + i + DotA + l + space + I + DotA + f + i + DotA + s, // titlecase
                S + I + T + I + W + I + F + IDot + L + space + I + DotA + F + I + DotA + S, // uppercase
                false, false);
        // - Sigma (final vs. non-final forms).
        const TStr Sigma = "03a3 ", sigma = "03c3 ", fsigma = "03c2 ";
        TestCaseConversion(
                Sigma + s + space + s + Sigma  + space + s + Sigma + s + space + Sigma + S + Sigma  + space + Sigma, // source
                sigma + s + space + s + fsigma + space + s + sigma + s + space + sigma + s + fsigma + space + sigma, // lowercase
                Sigma + s + space + S + fsigma + space + S + sigma + s + space + Sigma + s + fsigma + space + Sigma, // titlecase
                Sigma + S + space + S + Sigma  + space + S + Sigma + S + space + Sigma + S + Sigma  + space + Sigma, // uppercase
                false, false);
        TestCaseConversion(
                sigma + s + space + s + sigma  + space + s + sigma + s + space + sigma + S + sigma  + space + sigma, // source
                sigma + s + space + s + sigma  + space + s + sigma + s + space + sigma + s + sigma  + space + sigma, // lowercase
                Sigma + s + space + S + sigma  + space + S + sigma + s + space + Sigma + s + sigma  + space + Sigma, // titlecase
                Sigma + S + space + S + Sigma  + space + S + Sigma + S + space + Sigma + S + Sigma  + space + Sigma, // uppercase
                false, false);
        TestCaseConversion(
                fsigma + s + space + s + fsigma + space + s + fsigma + s + space + fsigma + S + fsigma  + space + fsigma, // source
                fsigma + s + space + s + fsigma + space + s + fsigma + s + space + fsigma + s + fsigma  + space + fsigma, // lowercase
                Sigma  + s + space + S + fsigma + space + S + fsigma + s + space + Sigma  + s + fsigma  + space + Sigma, // titlecase
                Sigma  + S + space + S + Sigma  + space + S + Sigma  + S + space + Sigma  + S + Sigma   + space + Sigma, // uppercase
                false, false);
        const TStr nonSA = "0315 0321 0322 "; // characters that are neither ccStarter nor ccAbove
        // Special case mappings for Turkic languages:
        // - After_I
        TestCaseConversion(
                s + I + t + i + w + iDotless + f + IDot + l + space + iDotless + DotA + f + I + DotA + J + DotA + I + Grave + DotA + I + DotA + DotA + I + nonSA + DotA + s, // source
                s + iDotless + t + i + w + iDotless + f + i + l + space + iDotless + DotA + f + i + j + DotA + iDotless + Grave + DotA + i + DotA + i + nonSA + s, // lowercase
                S + iDotless + t + i + w + iDotless + f + i + l + space + I + DotA + f + i + j + DotA + iDotless + Grave + DotA + i + DotA + i + nonSA + s, // titlecase
                S + I + T + IDot + W + I + F + IDot + L + space + I + DotA + F + I + DotA + J + DotA + I + Grave + DotA + I + DotA + DotA + I + nonSA + DotA + S, // uppercase
                true, false); // turkic
        // - Not_Before_Dot
        TestCaseConversion(
                I + Grave + t + I + DotA + f + I + nonSA + DotA + j + space + I + nonSA + DotA + space + I + Grave + t, // source
                iDotless + Grave + t + i + f + i + nonSA + j + space + i + nonSA + space + iDotless + Grave + t, // lowercase
                I + Grave + t + i + f + i + nonSA + j + space + I + nonSA + DotA + space + I + Grave + t, // titlecase
                I + Grave + T + I + DotA + F + I + nonSA + DotA + J + space + I + nonSA + DotA + space + I + Grave + T, // uppercase
                true, false); // turkic
        // Special case mappings for Lithuanian:
        // - After_Soft_Dotted  [note: I + DotA turns into i + DotA + DotA when lowercasing due to More_Above]
        TestCaseConversion(
                i + DotA + t + i + Grave + DotA + f + i + DotA + DotA + f + i + nonSA + DotA + I + DotA + t + DotA + i + DotA + Grave, // source
                i + DotA + t + i + Grave + DotA + f + i + DotA + DotA + f + i + nonSA + DotA + i + DotA + DotA + t + DotA + i + DotA + Grave, // lowercase
                I + t + i + Grave + DotA + f + i + DotA + DotA + f + i + nonSA + DotA + i + DotA + DotA + t + DotA + i + DotA + Grave, // titlecase
                I + T + I + Grave + DotA + F + I + DotA + F + I + nonSA + I + DotA + T + DotA + I + Grave, // uppercase
                false, true); // lithuanian
        // - More_Above  [note: j + DotA turns into just J when uppercasing due to After_Soft_Dotted]
        TestCaseConversion(
                J +        Grave + space + J +        nonSA + DotA + space + j + Grave + space + j + DotA + space + J + nonSA + J +        nonSA + Grave + space + j + nonSA, // source
                j + DotA + Grave + space + j + DotA + nonSA + DotA + space + j + Grave + space + j + DotA + space + j + nonSA + j + DotA + nonSA + Grave + space + j + nonSA, // lowercase
                J +        Grave + space + J +        nonSA + DotA + space + J + Grave + space + J +        space + J + nonSA + j + DotA + nonSA + Grave + space + J + nonSA, // titlecase
                J +        Grave + space + J +        nonSA + DotA + space + J + Grave + space + J +        space + J + nonSA + J +        nonSA + Grave + space + J + nonSA, // uppercase
                false, true); // lithuanian
        // SoftDotted [^ Starter Above]* 0307   --(uc,tc)-->  brez 0307
        // SoftDotted [^ Starter Above]* 0307   --(
        //TestCaseConversion("", "", "", "", false, false);
}

//-----------------------------------------------------------------------------
// TUniChDb -- initialization from the text files
//-----------------------------------------------------------------------------

void TUniChDb::LoadTxt_ProcessDecomposition(TUniChInfo& ci, TStr s)
{
        if (s.Empty()) return;
        if (s[0] == '<') {
                int i = s.SearchCh('>'); IAssert(i > 0);
                ci.flags |= ucfCompatibilityDecomposition;
                s = s.GetSubStr(i + 1, s.Len() - 1); s.ToTrunc(); }
        TIntV dec; TUcdFileReader::ParseCodePointList(s, dec);
        IAssert(dec.Len() > 0);
        ci.decompOffset = decompositions.Len();
        decompositions.AddV(dec); decompositions.Add(-1);
}

void TUniChDb::InitPropList(const TStr& basePath)
{
        TUcdFileReader reader; TStrV fields; int nCps = 0, nLines = 0;
        reader.Open(CombinePath(basePath, GetPropListFn()));
        TSubcatHelper helper(*this);
        while (reader.GetNextLine(fields))
        {
                IAssert(fields.Len() == 2);
                int from, to; reader.ParseCodePointRange(fields[0], from, to);
                TStr s = fields[1];
                TUniChProperties prop = TUniChProperties(0); TUniChPropertiesX propx = TUniChPropertiesX(0);
                if (s == "White_Space") prop = ucfPrWhiteSpace;
                else if (s == "Bidi_Control") prop = ucfPrBidiControl;
                else if (s == "Join_Control") prop = ucfPrJoinControl;
                else if (s == "Dash") prop = ucfPrDash;
                else if (s == "Hyphen") prop = ucfPrHyphen;
                else if (s == "Quotation_Mark") prop = ucfPrQuotationMark;
                else if (s == "Terminal_Punctuation") prop = ucfPrTerminalPunctuation;
                else if (s == "Other_Math") propx = ucfPxOtherMath;
                else if (s == "Hex_Digit") prop = ucfPrHexDigit;
                else if (s == "ASCII_Hex_Digit") prop = ucfPrAsciiHexDigit;
                else if (s == "Other_Alphabetic") propx = ucfPxOtherAlphabetic;
                else if (s == "Ideographic") prop = ucfPrIdeographic;
                else if (s == "Diacritic") prop = ucfPrDiacritic;
                else if (s == "Extender") prop = ucfPrExtender;
                else if (s == "Other_Lowercase") propx = ucfPxOtherLowercase;
                else if (s == "Other_Uppercase") propx = ucfPxOtherUppercase;
                else if (s == "Noncharacter_Code_Point") prop = ucfPrNoncharacterCodePoint;
                else if (s == "Other_Grapheme_Extend") propx = ucfPxOtherGraphemeExtend;
                else if (s == "IDS_Binary_Operator") propx = ucfPxIdsBinaryOperator;
                else if (s == "IDS_Trinary_Operator") propx = ucfPxIdsTrinaryOperator;
                else if (s == "Radical") propx = ucfPxRadical;
                else if (s == "Unified_Ideograph") propx = ucfPxUnifiedIdeograph;
                else if (s == "Other_Default_Ignorable_Code_Point") propx = ucfPxOtherDefaultIgnorableCodePoint;
                else if (s == "Deprecated") prop = ucfPrDeprecated;
                else if (s == "Soft_Dotted") prop = ucfPrSoftDotted;
                else if (s == "Logical_Order_Exception") prop = ucfPrLogicalOrderException;
                else if (s == "Other_ID_Start") propx = ucfPxOtherIdStart;
                else if (s == "Other_ID_Continue") propx = ucfPxOtherIdContinue;
                else if (s == "STerm") prop = ucfPrSTerm;
                else if (s == "Variation_Selector") prop = ucfPrVariationSelector;
                else if (s == "Pattern_White_Space") prop = ucfPrPatternWhiteSpace;
                else if (s == "Pattern_Syntax") prop = ucfPrPatternSyntax;
                else FailR(s.CStr());
                helper.ProcessComment(reader);
                for (int cp = from; cp <= to; cp++) {
                        int i = h.GetKeyId(cp); if (i < 0) { i = h.AddKey(cp); helper.SetCat(cp); }
                        TUniChInfo &ci = h[i]; helper.TestCat(cp);
                        if (prop) { IAssert(! ci.IsProperty(prop)); ci.SetProperty(prop); }
                        if (propx) { IAssert(! ci.IsPropertyX(propx)); ci.SetPropertyX(propx); }
                        nCps++; }
                nLines++;
        }
        reader.Close();
        printf("TUniChDb::InitPropList: %d lines, %d code points.\n", nLines, nCps);
}

void TUniChDb::InitDerivedCoreProperties(const TStr& basePath)
{
        TUcdFileReader reader; TStrV fields; int nCps = 0, nLines = 0;
        reader.Open(CombinePath(basePath, GetDerivedCorePropsFn()));
        TSubcatHelper helper(*this);
        while (reader.GetNextLine(fields))
        {
                IAssert(fields.Len() == 2);
                int from, to; reader.ParseCodePointRange(fields[0], from, to);
                TStr s = fields[1];
                TUniChFlags flag = ucfCompatibilityDecomposition;
                if (s == "Math") flag = ucfDcpMath;
                else if (s == "Alphabetic") flag = ucfDcpAlphabetic;
                else if (s == "Lowercase") flag = ucfDcpLowercase;
                else if (s == "Uppercase") flag = ucfDcpUppercase;
                else if (s == "ID_Start") flag = ucfDcpIdStart;
                else if (s == "ID_Continue") flag = ucfDcpIdContinue;
                else if (s == "XID_Start") flag = ucfDcpXidStart;
                else if (s == "XID_Continue") flag = ucfDcpXidContinue;
                else if (s == "Default_Ignorable_Code_Point") flag = ucfDcpDefaultIgnorableCodePoint;
                else if (s == "Grapheme_Extend") flag = ucfDcpGraphemeExtend;
                else if (s == "Grapheme_Base") flag = ucfDcpGraphemeBase;
                else if (s == "Grapheme_Link") continue; // this flag is deprecated; test for combClass == Virama instead
                else FailR(s.CStr());
                // If we add new codepoints to the hash table, we should also set their category.
                // This is supposed to be provided in the comment, e.g. "# Cf       SOFT HYPHEN".
                helper.ProcessComment(reader);
                //
                for (int cp = from; cp <= to; cp++) {
                        int i = h.GetKeyId(cp); if (i < 0) { i = h.AddKey(cp); helper.SetCat(cp); }
                        helper.TestCat(cp);
                        TUniChInfo &ci = h[i]; IAssert(! ci.IsDcpFlag(flag));
                        ci.SetDcpFlag(flag); nCps++; }
                nLines++;
        }
        reader.Close();
        printf("TUniChDb::InitDerivedCoreProperties: %d lines, %d code points.\n", nLines, nCps);
}

void TUniChDb::InitLineBreaks(const TStr& basePath)
{
        // Clear old linebreak values.
        ushort xx = TUniChInfo::LineBreak_Unknown;
        for (int i = h.FFirstKeyId(); h.FNextKeyId(i); ) h[i].lineBreak = xx;
        // Read LineBreak.txt.
        TUcdFileReader reader; TStrV fields;
        reader.Open(CombinePath(basePath, GetLineBreakFn()));
        int nLines = 0, nCps = 0;
        while (reader.GetNextLine(fields))
        {
                IAssert(fields.Len() == 2);
                int from, to; reader.ParseCodePointRange(fields[0], from, to);
                TStr s = fields[1]; IAssert(s.Len() == 2);
                ushort us = TUniChInfo::GetLineBreakCode(s[0], s[1]);
                if (us == xx) continue;
                for (int cp = from; cp <= to; cp++) {
                        int i = h.GetKeyId(cp); if (i < 0) { i = h.AddKey(cp);
                                printf("TUniChDb::InitLineBreaks: warning, adding codepoint %d, its category will remain unknown.\n", cp); }
                        IAssert(h[i].lineBreak == xx);
                        h[i].lineBreak = us; nCps++; }
                nLines++;
        }
        reader.Close();
        printf("TUniChDb::InitLineBreaks: %d lines, %d codepoints processed (excluding \'xx\' values).\n", nLines, nCps);
}

void TUniChDb::InitScripts(const TStr& basePath)
{
        TUcdFileReader reader; TStrV fields;
        reader.Open(CombinePath(basePath, GetScriptsFn()));
        TSubcatHelper helper(*this);
        while (reader.GetNextLine(fields))
        {
                int from, to; reader.ParseCodePointRange(fields[0], from, to);
                TStr scriptName = fields[1];
                int scriptNo = scripts.GetKeyId(scriptName);
                if (scriptNo < 0) { scriptNo = scripts.AddKey(scriptName); scripts[scriptNo] = 0; }
                IAssert(scriptNo >= 0 && scriptNo < SCHAR_MAX); // because TUniChInfo.script is a signed char
                scripts[scriptNo] += 1;
                helper.ProcessComment(reader);
                for (int cp = from; cp <= to; cp++) {
                        int i = h.GetKeyId(cp); if (i < 0) { i = h.AddKey(cp); helper.SetCat(cp); }
                        helper.TestCat(cp);
                        TUniChInfo &ci = h[i]; ci.script = scriptNo; }
        }
        reader.Close();
        scripts.AddDat(GetScriptNameUnknown()) = 0;
        printf("TUniChDb::InitScripts: %d scripts: ", scripts.Len());
        if (AlwaysFalse()) for (int i = scripts.FFirstKeyId(); scripts.FNextKeyId(i); )
                printf("  %d:%s (%d)", i, scripts.GetKey(i).CStr(), int(scripts[i]));
        printf("\n");
}

void TUniChDb::InitWordAndSentenceBoundaryFlags(const TStr& basePath)
{
        // UAX #29, sec. 4.1 and 5.1.
        // Note: these flags can also be initialized from auxiliary\\WordBreakProperty.txt.
        int katakana = GetScriptByName(GetScriptNameKatakana()); IAssert(katakana >= 0);
        int hiragana = GetScriptByName(GetScriptNameHiragana()); IAssert(hiragana >= 0);
        // Clear any existing word-boundary flags and initialize them again.
        for (int i = h.FFirstKeyId(); h.FNextKeyId(i); )
        {
                const int cp = h.GetKey(i); TUniChInfo& ci = h[i];
                ci.ClrWbAndSbFlags();
                // Word-boundary flags.
                if (ci.subCat  == ucOtherFormat && cp != 0x200c && cp != 0x200d) ci.SetWbFlag(ucfWbFormat);
                if (ci.script == katakana) ci.SetWbFlag(ucfWbKatakana);
                if (ci.lineBreak == TUniChInfo::LineBreak_InfixNumeric && cp != 0x3a) ci.SetWbFlag(ucfWbMidNum);
                if (ci.lineBreak == TUniChInfo::LineBreak_Numeric) ci.SetWbFlag(ucfWbNumeric);
                if (ci.subCat == ucPunctuationConnector) ci.SetWbFlag(ucfWbExtendNumLet);
                // Sentence-boundary flags.  Some are identical to some word-boundary flags.
                if (cp == 0xa || cp == 0xd || cp == 0x85 || cp == 0x2028 || cp == 0x2029) ci.SetSbFlag(ucfSbSep);
                if (ci.subCat == ucOtherFormat && cp != 0x200c && cp != 0x200d) ci.SetSbFlag(ucfSbFormat);
                if (ci.IsWhiteSpace() && ! ci.IsSbFlag(ucfSbSep) && cp != 0xa0) ci.SetSbFlag(ucfSbSp);
                if (ci.IsLowercase() && ! ci.IsGraphemeExtend()) ci.SetSbFlag(ucfSbLower);
                if (ci.IsUppercase() || ci.subCat == ucLetterTitlecase) ci.SetSbFlag(ucfSbUpper);
                if ((ci.IsAlphabetic() || cp == 0xa0 || cp == 0x5f3) && ! ci.IsSbFlag(ucfSbLower) && ! ci.IsSbFlag(ucfSbUpper) && ! ci.IsGraphemeExtend()) ci.SetSbFlag(ucfSbOLetter);
                if (ci.lineBreak == TUniChInfo::LineBreak_Numeric) ci.SetSbFlag(ucfSbNumeric);
                if (cp == 0x2e) ci.SetSbFlag(ucfSbATerm);
                // Note: UAX #29 says that if the property STerm = true, then the character should belong to the STerm class for
                // the purposes of sentence-boundary detection.  Now in PropList.txt there is no doubt that 002E has the STerm
                // property; thus, it should also belong to the STerm sentence-boundary class.  However, in
                // SentenceBreakProperty.txt, 002E is only listed in the ATerm class, but not in the STerm class.
                if (ci.IsSTerminal() && cp != 0x2e) ci.SetSbFlag(ucfSbSTerm);
                if ((ci.subCat == ucPunctuationOpen || ci.subCat == ucPunctuationClose || ci.lineBreak == TUniChInfo::LineBreak_Quotation) && cp != 0x5f3 && ! ci.IsSbFlag(ucfSbATerm) && ! ci.IsSbFlag(ucfSbSTerm)) ci.SetSbFlag(ucfSbClose);
        }
        // Some additional characters for Katakana and MidLetter.
        TIntV v = (VB, 0x3031, 0x3032, 0x3033, 0x3034, 0x3035, 0x309b, 0x309c, 0x30a0, 0x30fc, 0xff70, 0xff9e, 0xff9f);
        for (int i = 0; i < v.Len(); i++) h.GetDat(v[i]).SetWbFlag(ucfWbKatakana);
        v = (VB, 0x27, 0xb7, 0x5f4, 0x2019, 0x2027, 0x3a);
        for (int i = 0; i < v.Len(); i++) h.GetDat(v[i]).SetWbFlag(ucfWbMidLetter);
        // WbALetter depends on Katakana, so it cannot be initialized earlier.
        for (int i = h.FFirstKeyId(); h.FNextKeyId(i); )
        {
                const int cp = h.GetKey(i); TUniChInfo& ci = h[i];
                if ((ci.IsAlphabetic() || cp == 0x5f3) && ! ci.IsIdeographic() && ! ci.IsWbFlag(ucfWbKatakana) && ci.lineBreak != TUniChInfo::LineBreak_ComplexContext && ci.script != hiragana && ! ci.IsGraphemeExtend())
                        ci.SetWbFlag(ucfWbALetter);
        }
        // An alternative is to extract the flags from WordBreakProperty.txt.
        // The results should be the same.
        {TUcdFileReader reader; TStrV fields;
        reader.Open(CombinePath(CombinePath(basePath, GetAuxiliaryDir()), GetWordBreakPropertyFn()));
        THash<TInt, TInt> hh;
        while (reader.GetNextLine(fields))
        {
                IAssert(fields.Len() == 2);
                int from, to; reader.ParseCodePointRange(fields[0], from, to);
                TStr s = fields[1];
                TUniChFlags flag = ucfCompatibilityDecomposition;
                if (s == "Format") flag = ucfWbFormat;
                else if (s == "Katakana") flag = ucfWbKatakana;
                else if (s == "ALetter") flag = ucfWbALetter;
                else if (s == "MidLetter") flag = ucfWbMidLetter;
                else if (s == "MidNum") flag = ucfWbMidNum;
                else if (s == "Numeric") flag = ucfWbNumeric;
                else if (s == "ExtendNumLet") flag = ucfWbExtendNumLet;
                else FailR(s.CStr());
                for (int c = from; c <= to; c++) {
                        int i = hh.GetKeyId(c); if (i < 0) hh.AddDat(c, flag);
                        else hh[i].Val |= flag; }
        }
        reader.Close();
        TIntV cps; for (int i = h.FFirstKeyId(); h.FNextKeyId(i); ) cps.Add(h.GetKey(i));
        for (int i = hh.FFirstKeyId(); hh.FNextKeyId(i); ) cps.Add(hh.GetKey(i));
        cps.Sort(); cps.Merge();
        for (int i = 0; i < cps.Len(); i++)
        {
                int cp = cps[i];
                int flags1 = 0; if (h.IsKey(cp)) flags1 = h.GetDat(cp).GetWbFlags();
                int flags2 = 0; if (hh.IsKey(cp)) flags2 = hh.GetDat(cp);
                flags1 &= ~ucfSbSep; flags2 &= ~ucfSbSep;
                if (flags1 != flags2) {
                        printf("cp = %04x: flags1 = %08x flags2 = %08x xor = %08x\n", cp, flags1, flags2, flags1 ^ flags2);
                        Fail; }
        }}
        // Likewise, for sentence boundary flags we have SentenceBreakProperty.txt.
        {TUcdFileReader reader; TStrV fields;
        reader.Open(CombinePath(CombinePath(basePath, GetAuxiliaryDir()), GetSentenceBreakPropertyFn()));
        THash<TInt, TInt> hh;
        while (reader.GetNextLine(fields))
        {
                IAssert(fields.Len() == 2);
                int from, to; reader.ParseCodePointRange(fields[0], from, to);
                TStr s = fields[1];
                TUniChFlags flag = ucfCompatibilityDecomposition;
                if (s == "Sep") flag = ucfSbSep;
                else if (s == "Format") flag = ucfSbFormat;
                else if (s == "Sp") flag = ucfSbSp;
                else if (s == "Lower") flag = ucfSbLower;
                else if (s == "Upper") flag = ucfSbUpper;
                else if (s == "OLetter") flag = ucfSbOLetter;
                else if (s == "Numeric") flag = ucfSbNumeric;
                else if (s == "ATerm") flag = ucfSbATerm;
                else if (s == "STerm") flag = ucfSbSTerm;
                else if (s == "Close") flag = ucfSbClose;
                else FailR(s.CStr());
                for (int c = from; c <= to; c++) {
                        int i = hh.GetKeyId(c); if (i < 0) hh.AddDat(c, flag);
                        else hh[i].Val |= flag; }
        }
        reader.Close();
        TIntV cps; for (int i = h.FFirstKeyId(); h.FNextKeyId(i); ) cps.Add(h.GetKey(i));
        for (int i = hh.FFirstKeyId(); hh.FNextKeyId(i); ) cps.Add(hh.GetKey(i));
        cps.Sort(); cps.Merge();
        for (int i = 0; i < cps.Len(); i++)
        {
                int cp = cps[i];
                int flags1 = 0; if (h.IsKey(cp)) flags1 = h.GetDat(cp).GetSbFlags();
                int flags2 = 0; if (hh.IsKey(cp)) flags2 = hh.GetDat(cp);
                if (flags1 != flags2) {
                        printf("cp = %04x: flags1 = %08x [%s] flags2 = %08x [%s] xor = %08x\n", cp,
                                flags1, TUniChInfo::GetSbFlagsStr(flags1).CStr(),
                                flags2, TUniChInfo::GetSbFlagsStr(flags2).CStr(),
                                flags1 ^ flags2);
                        Fail; }
        }}
}

void TUniChDb::InitSpecialCasing(const TStr& basePath)
{
        TUcdFileReader reader; TStrV fields;
        reader.Open(CombinePath(basePath, GetSpecialCasingFn()));
        while (reader.GetNextLine(fields))
        {
                IAssert(fields.Len() == 5 || fields.Len() == 6);
                IAssert(fields.Last().Empty());
                // Skip conditional mappings -- they will be hardcoded in the GetCaseConverted method.
                TStr conditions = "";
                if (fields.Len() == 6) conditions = fields[4];
                conditions.ToTrunc(); if (! conditions.Empty()) continue;
                // Keep the other mappings.
                const int cp = reader.ParseCodePoint(fields[0]);
                TIntV v; reader.ParseCodePointList(fields[1], v);
                specialCasingLower.AddDat(cp, v);
                reader.ParseCodePointList(fields[2], v);
                specialCasingTitle.AddDat(cp, v);
                reader.ParseCodePointList(fields[3], v);
                specialCasingUpper.AddDat(cp, v);
        }
        reader.Close();
}

void TUniChDb::LoadTxt(const TStr& basePath)
{
        Clr();
        // Set up a hash table with enough ports that there will be more or less no chains longer than 1 element.
        h = THash<TInt, TUniChInfo>(196613, true);
        //
        caseFolding.LoadTxt(CombinePath(basePath, GetCaseFoldingFn()));
        //
        TUcdFileReader reader; TStrV fields; TIntH seen;
        reader.Open(CombinePath(basePath, GetUnicodeDataFn()));
        while (reader.GetNextLine(fields))
        {
                // Codepoint.
                int cp = reader.ParseCodePoint(fields[0]);
                IAssert(! seen.IsKey(cp)); seen.AddKey(cp);
                TUniChInfo& ci = h.AddDat(cp);
                // Name.
                ci.nameOffset = charNames.AddStr(fields[1]);
                // Category.
                TStr& s = fields[2]; IAssert(s.Len() == 2);
                ci.chCat = s[0]; ci.chSubCat = s[1];
                // Canonical combining class.
                s = fields[3]; IAssert(s.Len() > 0);
                int i; bool ok = s.IsInt(true, TUCh::Mn, TUCh::Mx, i); IAssertR(ok, s);
                ci.combClass = (uchar) i;
                // Decomposition type and mapping.
                LoadTxt_ProcessDecomposition(ci, fields[5]);
                // Simple case mappings.
                s = fields[12]; ci.simpleUpperCaseMapping = (! s.Empty() ? reader.ParseCodePoint(s) : -1);
                s = fields[13]; ci.simpleLowerCaseMapping = (! s.Empty() ? reader.ParseCodePoint(s) : -1);
                s = fields[14]; ci.simpleTitleCaseMapping = (! s.Empty() ? reader.ParseCodePoint(s) : -1);
                //
                ci.InitAfterLoad(); // initializes ci.cat, ci.subCat
        }
        reader.Close();
        //
        InitScripts(basePath);
        //
        InitPropList(basePath);
        InitDerivedCoreProperties(basePath);
        InitLineBreaks(basePath);
        InitSpecialCasing(basePath);
        // Process the composition exclusions (UAX #15, sec. 6).
        for (int i = h.FFirstKeyId(); h.FNextKeyId(i); )
        {
                TUniChInfo& ci = h[i];
                int ofs = ci.decompOffset; if (ofs < 0) continue;
                int n = 0; while (decompositions[ofs + n] >= 0) n++;
                IAssert(n > 0);
                // Singleton decompositions.
                if (n == 1) { ci.flags |= ucfCompositionExclusion; continue; }
                // Non-starter decompositions.
                int cp1 = decompositions[ofs];
                IAssert(h.IsKey(cp1));
                uchar ccc = h.GetDat(cp1).combClass;
                if (ccc != TUniChInfo::ccStarter) { ci.flags |= ucfCompositionExclusion; continue; }
        }
        // Process the composition exclusion table.
        reader.Open(CombinePath(basePath, GetCompositionExclusionsFn()));
        int nExclusionTable = 0;
        while (reader.GetNextLine(fields))
        {
                IAssert(fields.Len() == 1);
                int cp = reader.ParseCodePoint(fields[0]);
                int i = h.GetKeyId(cp); IAssert(i >= 0);
                h[i].flags |= ucfCompositionExclusion;
                nExclusionTable++;
        }
        reader.Close();
        // Prepare the inverted index for composition pairs.
        for (int i = h.FFirstKeyId(); h.FNextKeyId(i); )
        {
                int cp = h.GetKey(i);
                TUniChInfo& ci = h[i];
                int ofs = ci.decompOffset; if (ofs < 0) continue;
                if (ci.IsCompositionExclusion()) continue;
                if (ci.IsCompatibilityDecomposition()) continue;
                int n = 0; while (decompositions[ofs + n] >= 0) n++;
                if (n != 2) continue;
                TIntPr pr = TIntPr(decompositions[ofs], decompositions[ofs + 1]);
                IAssert(! inverseDec.IsKey(pr));
                IAssert(ci.combClass == TUniChInfo::ccStarter);
                inverseDec.AddDat(pr, cp);
        }
        printf("TUniChDb(%s): %d chars in h, %d in decomp inverse index; %d in decomp vector; %d in exclusion table\n",
                basePath.CStr(), h.Len(), inverseDec.Len(), decompositions.Len(), nExclusionTable);
        // Before calling InitWordBoundaryFlags(), scripts must have been initialized, as well as
        // flags such as Alphabetic, Word_Break, and Grapheme_Extend.
        InitWordAndSentenceBoundaryFlags(basePath); // Note: scripts must have been initialized by this point.
        // Make sure that Hangul combined characters are treated as stareters.
        for (int cp = HangulSBase; cp < HangulSBase + HangulSCount; cp++)
        {
                int j = h.GetKeyId(cp); if (j < 0) continue;
                TUniChInfo& ci = h[j];
                if (ci.combClass == TUniChInfo::ccInvalid) ci.combClass = TUniChInfo::ccStarter;
                IAssert(ci.combClass == TUniChInfo::ccStarter);
        }
        // There should be no more additions to 'h' beyond this point.
        const int oldHLen = h.Len();
        // Provide default (identity) case mappings if any were missing from UnicodeData.txt
        // (or if any entirely new characters were added later, e.g. while reading LineBreaks.txt).
        int scriptUnknown = GetScriptByName(GetScriptNameUnknown());
        for (int i = h.FFirstKeyId(); h.FNextKeyId(i); )
        {
                int cp = h.GetKey(i); TUniChInfo &ci = h[i];
                if (ci.simpleLowerCaseMapping < 0) ci.simpleLowerCaseMapping = cp;
                if (ci.simpleUpperCaseMapping < 0) ci.simpleUpperCaseMapping = cp;
                if (ci.simpleTitleCaseMapping < 0) ci.simpleTitleCaseMapping = cp;
                if (ci.script < 0) ci.script = scriptUnknown;
        }
        IAssert(h.Len() == oldHLen);
}

void TUniChDb::SaveBin(const TStr& fnBinUcd)
{
        PSOut SOut=TFOut::New(fnBinUcd);
        Save(*SOut);
}

void TUniChDb::InitAfterLoad()
{
        scriptUnknown = GetScriptByName(GetScriptNameUnknown()); IAssert(scriptUnknown >= 0);
}

//-----------------------------------------------------------------------------
// TUniChDb -- main test driver
//-----------------------------------------------------------------------------

void TUniChDb::Test(const TStr& basePath)
{
        TStr fnBin = CombinePath(basePath, GetBinFn());
        if (true || ! TFile::Exists(fnBin))
        {
                // Test LoadTxt.
                LoadTxt(basePath);
                // Test Save.
                {PSOut SOut = TFOut::New(fnBin);
                Save(*SOut);}
        }
        // Test Load.
        this->~TUniChDb();
        new(this) TUniChDb();
        {PSIn SIn = TFIn::New(fnBin);
        Load(*SIn);}
        // Test the case folding.
        caseFolding.Test();
        // Test the word breaking.
        TestWbFindNonIgnored();
        // Test the sentence breaking.
        TestFindNextWordOrSentenceBoundary(basePath, true);
        TestFindNextWordOrSentenceBoundary(basePath, false);
        // Test composition and decomposition.
        TestComposition(basePath);
        // Test the case conversions.
        TestCaseConversions();
}

//-----------------------------------------------------------------------------
// T8BitCodec -- a class for converting between 8-bit encodings and Unicode
//-----------------------------------------------------------------------------

//-----------------------------------------------------------------------------
// ISO-8859-2
//-----------------------------------------------------------------------------

const int TEncoding_ISO8859_2::toUnicodeTable[6 * 16] =
{
        /* 0xa0 */ 0x00a0, 0x0104, 0x02d8, 0x0141, 0x00a4, 0x013d, 0x015a, 0x00a7, 0x00a8, 0x0160, 0x015e, 0x0164, 0x0179, 0x00ad, 0x017d, 0x017b,
        /* 0xb0 */ 0x00b0, 0x0105, 0x02db, 0x0142, 0x00b4, 0x013e, 0x015b, 0x02c7, 0x00b8, 0x0161, 0x015f, 0x0165, 0x017a, 0x02dd, 0x017e, 0x017c,
        /* 0xc0 */ 0x0154, 0x00c1, 0x00c2, 0x0102, 0x00c4, 0x0139, 0x0106, 0x00c7, 0x010c, 0x00c9, 0x0118, 0x00cb, 0x011a, 0x00cd, 0x00ce, 0x010e,
        /* 0xd0 */ 0x0110, 0x0143, 0x0147, 0x00d3, 0x00d4, 0x0150, 0x00d6, 0x00d7, 0x0158, 0x016e, 0x00da, 0x0170, 0x00dc, 0x00dd, 0x0162, 0x00df,
        /* 0xe0 */ 0x0155, 0x00e1, 0x00e2, 0x0103, 0x00e4, 0x013a, 0x0107, 0x00e7, 0x010d, 0x00e9, 0x0119, 0x00eb, 0x011b, 0x00ed, 0x00ee, 0x010f,
        /* 0xf0 */ 0x0111, 0x0144, 0x0148, 0x00f3, 0x00f4, 0x0151, 0x00f6, 0x00f7, 0x0159, 0x016f, 0x00fa, 0x0171, 0x00fc, 0x00fd, 0x0163, 0x02d9
};

const int TEncoding_ISO8859_2::fromUnicodeTable1[14 * 16] = {
        /* U+00a0 */ 0x00a0,     -1,     -1,     -1, 0x00a4,     -1,     -1, 0x00a7, 0x00a8,     -1,     -1,     -1,     -1, 0x00ad,     -1,     -1,
        /* U+00b0 */ 0x00b0,     -1,     -1,     -1, 0x00b4,     -1,     -1,     -1, 0x00b8,     -1,     -1,     -1,     -1,     -1,     -1,     -1,
        /* U+00c0 */     -1, 0x00c1, 0x00c2,     -1, 0x00c4,     -1,     -1, 0x00c7,     -1, 0x00c9,     -1, 0x00cb,     -1, 0x00cd, 0x00ce,     -1,
        /* U+00d0 */     -1,     -1,     -1, 0x00d3, 0x00d4,     -1, 0x00d6, 0x00d7,     -1,     -1, 0x00da,     -1, 0x00dc, 0x00dd,     -1, 0x00df,
        /* U+00e0 */     -1, 0x00e1, 0x00e2,     -1, 0x00e4,     -1,     -1, 0x00e7,     -1, 0x00e9,     -1, 0x00eb,     -1, 0x00ed, 0x00ee,     -1,
        /* U+00f0 */     -1,     -1,     -1, 0x00f3, 0x00f4,     -1, 0x00f6, 0x00f7,     -1,     -1, 0x00fa,     -1, 0x00fc, 0x00fd,     -1,     -1,
        /* U+0100 */     -1,     -1, 0x00c3, 0x00e3, 0x00a1, 0x00b1, 0x00c6, 0x00e6,     -1,     -1,     -1,     -1, 0x00c8, 0x00e8, 0x00cf, 0x00ef,
        /* U+0110 */ 0x00d0, 0x00f0,     -1,     -1,     -1,     -1,     -1,     -1, 0x00ca, 0x00ea, 0x00cc, 0x00ec,     -1,     -1,     -1,     -1,
        /* U+0120 */     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1 /* blank */,
        /* U+0130 */     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1, 0x00c5, 0x00e5,     -1,     -1, 0x00a5, 0x00b5,     -1,
        /* U+0140 */     -1, 0x00a3, 0x00b3, 0x00d1, 0x00f1,     -1,     -1, 0x00d2, 0x00f2,     -1,     -1,     -1,     -1,     -1,     -1,     -1,
        /* U+0150 */ 0x00d5, 0x00f5,     -1,     -1, 0x00c0, 0x00e0,     -1,     -1, 0x00d8, 0x00f8, 0x00a6, 0x00b6,     -1,     -1, 0x00aa, 0x00ba,
        /* U+0160 */ 0x00a9, 0x00b9, 0x00de, 0x00fe, 0x00ab, 0x00bb,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1, 0x00d9, 0x00f9,
        /* U+0170 */ 0x00db, 0x00fb,     -1,     -1,     -1,     -1,     -1,     -1,     -1, 0x00ac, 0x00bc, 0x00af, 0x00bf, 0x00ae, 0x00be,     -1
};

const int TEncoding_ISO8859_2::fromUnicodeTable2[2 * 16] = {
        /* U+02c0 */     -1,     -1,     -1,     -1,     -1,     -1,     -1, 0x00b7,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,
        /* U+02d0 */     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1, 0x00a2, 0x00ff,     -1, 0x00b2,     -1, 0x00bd,     -1,     -1
};

//-----------------------------------------------------------------------------
// ISO-8859-3
//-----------------------------------------------------------------------------

const int TEncoding_ISO8859_3::toUnicodeTable[6 * 16] = {
        /* 0xa0 */ 0x00a0, 0x0126, 0x02d8, 0x00a3, 0x00a4,     -1, 0x0124, 0x00a7, 0x00a8, 0x0130, 0x015e, 0x011e, 0x0134, 0x00ad,     -1, 0x017b,
        /* 0xb0 */ 0x00b0, 0x0127, 0x00b2, 0x00b3, 0x00b4, 0x00b5, 0x0125, 0x00b7, 0x00b8, 0x0131, 0x015f, 0x011f, 0x0135, 0x00bd,     -1, 0x017c,
        /* 0xc0 */ 0x00c0, 0x00c1, 0x00c2,     -1, 0x00c4, 0x010a, 0x0108, 0x00c7, 0x00c8, 0x00c9, 0x00ca, 0x00cb, 0x00cc, 0x00cd, 0x00ce, 0x00cf,
        /* 0xd0 */     -1, 0x00d1, 0x00d2, 0x00d3, 0x00d4, 0x0120, 0x00d6, 0x00d7, 0x011c, 0x00d9, 0x00da, 0x00db, 0x00dc, 0x016c, 0x015c, 0x00df,
        /* 0xe0 */ 0x00e0, 0x00e1, 0x00e2,     -1, 0x00e4, 0x010b, 0x0109, 0x00e7, 0x00e8, 0x00e9, 0x00ea, 0x00eb, 0x00ec, 0x00ed, 0x00ee, 0x00ef,
        /* 0xf0 */     -1, 0x00f1, 0x00f2, 0x00f3, 0x00f4, 0x0121, 0x00f6, 0x00f7, 0x011d, 0x00f9, 0x00fa, 0x00fb, 0x00fc, 0x016d, 0x015d, 0x02d9
};

const int TEncoding_ISO8859_3::fromUnicodeTable1[14 * 16] = {
        /* U+00a0 */ 0x00a0,     -1,     -1, 0x00a3, 0x00a4,     -1,     -1, 0x00a7, 0x00a8,     -1,     -1,     -1,     -1, 0x00ad,     -1,     -1,
        /* U+00b0 */ 0x00b0,     -1, 0x00b2, 0x00b3, 0x00b4, 0x00b5,     -1, 0x00b7, 0x00b8,     -1,     -1,     -1,     -1, 0x00bd,     -1,     -1,
        /* U+00c0 */ 0x00c0, 0x00c1, 0x00c2,     -1, 0x00c4,     -1,     -1, 0x00c7, 0x00c8, 0x00c9, 0x00ca, 0x00cb, 0x00cc, 0x00cd, 0x00ce, 0x00cf,
        /* U+00d0 */     -1, 0x00d1, 0x00d2, 0x00d3, 0x00d4,     -1, 0x00d6, 0x00d7,     -1, 0x00d9, 0x00da, 0x00db, 0x00dc,     -1,     -1, 0x00df,
        /* U+00e0 */ 0x00e0, 0x00e1, 0x00e2,     -1, 0x00e4,     -1,     -1, 0x00e7, 0x00e8, 0x00e9, 0x00ea, 0x00eb, 0x00ec, 0x00ed, 0x00ee, 0x00ef,
        /* U+00f0 */     -1, 0x00f1, 0x00f2, 0x00f3, 0x00f4,     -1, 0x00f6, 0x00f7,     -1, 0x00f9, 0x00fa, 0x00fb, 0x00fc,     -1,     -1,     -1,
        /* U+0100 */     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1, 0x00c6, 0x00e6, 0x00c5, 0x00e5,     -1,     -1,     -1,     -1,
        /* U+0110 */     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1, 0x00d8, 0x00f8, 0x00ab, 0x00bb,
        /* U+0120 */ 0x00d5, 0x00f5,     -1,     -1, 0x00a6, 0x00b6, 0x00a1, 0x00b1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,
        /* U+0130 */ 0x00a9, 0x00b9,     -1,     -1, 0x00ac, 0x00bc,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,
        /* U+0140 */     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1 /* blank */,
        /* U+0150 */     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1, 0x00de, 0x00fe, 0x00aa, 0x00ba,
        /* U+0160 */     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1, 0x00dd, 0x00fd,     -1,     -1,
        /* U+0170 */     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1, 0x00af, 0x00bf,     -1,     -1,     -1,
};
const int TEncoding_ISO8859_3::fromUnicodeTable2[2] = {
        /* U+02d8 */ 0x00a2, 0x00ff
};

//-----------------------------------------------------------------------------
// ISO-8859-4
//-----------------------------------------------------------------------------

const int TEncoding_ISO8859_4::toUnicodeTable[6 * 16] = {
        /* 0xa0 */ 0x00a0, 0x0104, 0x0138, 0x0156, 0x00a4, 0x0128, 0x013b, 0x00a7, 0x00a8, 0x0160, 0x0112, 0x0122, 0x0166, 0x00ad, 0x017d, 0x00af,
        /* 0xb0 */ 0x00b0, 0x0105, 0x02db, 0x0157, 0x00b4, 0x0129, 0x013c, 0x02c7, 0x00b8, 0x0161, 0x0113, 0x0123, 0x0167, 0x014a, 0x017e, 0x014b,
        /* 0xc0 */ 0x0100, 0x00c1, 0x00c2, 0x00c3, 0x00c4, 0x00c5, 0x00c6, 0x012e, 0x010c, 0x00c9, 0x0118, 0x00cb, 0x0116, 0x00cd, 0x00ce, 0x012a,
        /* 0xd0 */ 0x0110, 0x0145, 0x014c, 0x0136, 0x00d4, 0x00d5, 0x00d6, 0x00d7, 0x00d8, 0x0172, 0x00da, 0x00db, 0x00dc, 0x0168, 0x016a, 0x00df,
        /* 0xe0 */ 0x0101, 0x00e1, 0x00e2, 0x00e3, 0x00e4, 0x00e5, 0x00e6, 0x012f, 0x010d, 0x00e9, 0x0119, 0x00eb, 0x0117, 0x00ed, 0x00ee, 0x012b,
        /* 0xf0 */ 0x0111, 0x0146, 0x014d, 0x0137, 0x00f4, 0x00f5, 0x00f6, 0x00f7, 0x00f8, 0x0173, 0x00fa, 0x00fb, 0x00fc, 0x0169, 0x016b, 0x02d9
};

const int TEncoding_ISO8859_4::fromUnicodeTable1[14 * 16] = {
        /* U+00a0 */ 0x00a0,     -1,     -1,     -1, 0x00a4,     -1,     -1, 0x00a7, 0x00a8,     -1,     -1,     -1,     -1, 0x00ad,     -1, 0x00af,
        /* U+00b0 */ 0x00b0,     -1,     -1,     -1, 0x00b4,     -1,     -1,     -1, 0x00b8,     -1,     -1,     -1,     -1,     -1,     -1,     -1,
        /* U+00c0 */     -1, 0x00c1, 0x00c2, 0x00c3, 0x00c4, 0x00c5, 0x00c6,     -1,     -1, 0x00c9,     -1, 0x00cb,     -1, 0x00cd, 0x00ce,     -1,
        /* U+00d0 */     -1,     -1,     -1,     -1, 0x00d4, 0x00d5, 0x00d6, 0x00d7, 0x00d8,     -1, 0x00da, 0x00db, 0x00dc,     -1,     -1, 0x00df,
        /* U+00e0 */     -1, 0x00e1, 0x00e2, 0x00e3, 0x00e4, 0x00e5, 0x00e6,     -1,     -1, 0x00e9,     -1, 0x00eb,     -1, 0x00ed, 0x00ee,     -1,
        /* U+00f0 */     -1,     -1,     -1,     -1, 0x00f4, 0x00f5, 0x00f6, 0x00f7, 0x00f8,     -1, 0x00fa, 0x00fb, 0x00fc,     -1,     -1,     -1,
        /* U+0100 */ 0x00c0, 0x00e0,     -1,     -1, 0x00a1, 0x00b1,     -1,     -1,     -1,     -1,     -1,     -1, 0x00c8, 0x00e8,     -1,     -1,
        /* U+0110 */ 0x00d0, 0x00f0, 0x00aa, 0x00ba,     -1,     -1, 0x00cc, 0x00ec, 0x00ca, 0x00ea,     -1,     -1,     -1,     -1,     -1,     -1,
        /* U+0120 */     -1,     -1, 0x00ab, 0x00bb,     -1,     -1,     -1,     -1, 0x00a5, 0x00b5, 0x00cf, 0x00ef,     -1,     -1, 0x00c7, 0x00e7,
        /* U+0130 */     -1,     -1,     -1,     -1,     -1,     -1, 0x00d3, 0x00f3, 0x00a2,     -1,     -1, 0x00a6, 0x00b6,     -1,     -1,     -1,
        /* U+0140 */     -1,     -1,     -1,     -1,     -1, 0x00d1, 0x00f1,     -1,     -1,     -1, 0x00bd, 0x00bf, 0x00d2, 0x00f2,     -1,     -1,
        /* U+0150 */     -1,     -1,     -1,     -1,     -1,     -1, 0x00a3, 0x00b3,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,
        /* U+0160 */ 0x00a9, 0x00b9,     -1,     -1,     -1,     -1, 0x00ac, 0x00bc, 0x00dd, 0x00fd, 0x00de, 0x00fe,     -1,     -1,     -1,     -1,
        /* U+0170 */     -1,     -1, 0x00d9, 0x00f9,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1, 0x00ae, 0x00be,     -1,
};

const int TEncoding_ISO8859_4::fromUnicodeTable2[2 * 16] = {
        /* U+02c0 */     -1,     -1,     -1,     -1,     -1,     -1,     -1, 0x00b7,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,
        /* U+02d0 */     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1, 0x00ff,     -1, 0x00b2,     -1,     -1,     -1,     -1
};

//-----------------------------------------------------------------------------
// CP 437
//-----------------------------------------------------------------------------

const int TEncoding_CP437::toUnicodeTable[8 * 16] = {
        /* 0x80 */ 0x00c7, 0x00fc, 0x00e9, 0x00e2, 0x00e4, 0x00e0, 0x00e5, 0x00e7, 0x00ea, 0x00eb, 0x00e8, 0x00ef, 0x00ee, 0x00ec, 0x00c4, 0x00c5,
        /* 0x90 */ 0x00c9, 0x00e6, 0x00c6, 0x00f4, 0x00f6, 0x00f2, 0x00fb, 0x00f9, 0x00ff, 0x00d6, 0x00dc, 0x00a2, 0x00a3, 0x00a5, 0x20a7, 0x0192,
        /* 0xa0 */ 0x00e1, 0x00ed, 0x00f3, 0x00fa, 0x00f1, 0x00d1, 0x00aa, 0x00ba, 0x00bf, 0x2310, 0x00ac, 0x00bd, 0x00bc, 0x00a1, 0x00ab, 0x00bb,
        /* 0xb0 */ 0x2591, 0x2592, 0x2593, 0x2502, 0x2524, 0x2561, 0x2562, 0x2556, 0x2555, 0x2563, 0x2551, 0x2557, 0x255d, 0x255c, 0x255b, 0x2510,
        /* 0xc0 */ 0x2514, 0x2534, 0x252c, 0x251c, 0x2500, 0x253c, 0x255e, 0x255f, 0x255a, 0x2554, 0x2569, 0x2566, 0x2560, 0x2550, 0x256c, 0x2567,
        /* 0xd0 */ 0x2568, 0x2564, 0x2565, 0x2559, 0x2558, 0x2552, 0x2553, 0x256b, 0x256a, 0x2518, 0x250c, 0x2588, 0x2584, 0x258c, 0x2590, 0x2580,
        /* 0xe0 */ 0x03b1, 0x00df, 0x0393, 0x03c0, 0x03a3, 0x03c3, 0x00b5, 0x03c4, 0x03a6, 0x0398, 0x03a9, 0x03b4, 0x221e, 0x03c6, 0x03b5, 0x2229,
        /* 0xf0 */ 0x2261, 0x00b1, 0x2265, 0x2264, 0x2320, 0x2321, 0x00f7, 0x2248, 0x00b0, 0x2219, 0x00b7, 0x221a, 0x207f, 0x00b2, 0x25a0, 0x00a0
};

const int TEncoding_CP437::fromUnicodeTable1[6 * 16] = {
        /* U+00a0 */ 0x00ff, 0x00ad, 0x009b, 0x009c,     -1, 0x009d,     -1,     -1,     -1,     -1, 0x00a6, 0x00ae, 0x00aa,     -1,     -1,     -1,
        /* U+00b0 */ 0x00f8, 0x00f1, 0x00fd,     -1,     -1, 0x00e6,     -1, 0x00fa,     -1,     -1, 0x00a7, 0x00af, 0x00ac, 0x00ab,     -1, 0x00a8,
        /* U+00c0 */     -1,     -1,     -1,     -1, 0x008e, 0x008f, 0x0092, 0x0080,     -1, 0x0090,     -1,     -1,     -1,     -1,     -1,     -1,
        /* U+00d0 */     -1, 0x00a5,     -1,     -1,     -1,     -1, 0x0099,     -1,     -1,     -1,     -1,     -1, 0x009a,     -1,     -1, 0x00e1,
        /* U+00e0 */ 0x0085, 0x00a0, 0x0083,     -1, 0x0084, 0x0086, 0x0091, 0x0087, 0x008a, 0x0082, 0x0088, 0x0089, 0x008d, 0x00a1, 0x008c, 0x008b,
        /* U+00f0 */     -1, 0x00a4, 0x0095, 0x00a2, 0x0093,     -1, 0x0094, 0x00f6,     -1, 0x0097, 0x00a3, 0x0096, 0x0081,     -1,     -1, 0x0098,
};

const int TEncoding_CP437::fromUnicodeTable2[4 * 16] = {
        /* U+0390 */     -1,     -1,     -1, 0x00e2,     -1,     -1,     -1,     -1, 0x00e9,     -1,     -1,     -1,     -1,     -1,     -1,     -1,
        /* U+03a0 */     -1,     -1,     -1, 0x00e4,     -1,     -1, 0x00e8,     -1,     -1, 0x00ea,     -1,     -1,     -1,     -1,     -1,     -1,
        /* U+03b0 */     -1, 0x00e0,     -1,     -1, 0x00eb, 0x00ee,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,
        /* U+03c0 */ 0x00e3,     -1,     -1, 0x00e5, 0x00e7,     -1, 0x00ed,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,
};

const int TEncoding_CP437::fromUnicodeTable3[6 * 16] = {
        /* U+2210 */     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1, 0x00f9, 0x00fb,     -1,     -1,     -1, 0x00ec,     -1,
        /* U+2220 */     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1, 0x00ef,     -1,     -1,     -1,     -1,     -1,     -1,
        /* U+2230 */     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1 /* blank */,
        /* U+2240 */     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1, 0x00f7,     -1,     -1,     -1,     -1,     -1,     -1,     -1,
        /* U+2250 */     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1 /* blank */,
        /* U+2260 */     -1, 0x00f0,     -1,     -1, 0x00f3, 0x00f2,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,
};

const int TEncoding_CP437::fromUnicodeTable4[11 * 16] = {
        /* U+2500 */ 0x00c4,     -1, 0x00b3,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1, 0x00da,     -1,     -1,     -1,
        /* U+2510 */ 0x00bf,     -1,     -1,     -1, 0x00c0,     -1,     -1,     -1, 0x00d9,     -1,     -1,     -1, 0x00c3,     -1,     -1,     -1,
        /* U+2520 */     -1,     -1,     -1,     -1, 0x00b4,     -1,     -1,     -1,     -1,     -1,     -1,     -1, 0x00c2,     -1,     -1,     -1,
        /* U+2530 */     -1,     -1,     -1,     -1, 0x00c1,     -1,     -1,     -1,     -1,     -1,     -1,     -1, 0x00c5,     -1,     -1,     -1,
        /* U+2540 */     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1 /* blank */,
        /* U+2550 */ 0x00cd, 0x00ba, 0x00d5, 0x00d6, 0x00c9, 0x00b8, 0x00b7, 0x00bb, 0x00d4, 0x00d3, 0x00c8, 0x00be, 0x00bd, 0x00bc, 0x00c6, 0x00c7,
        /* U+2560 */ 0x00cc, 0x00b5, 0x00b6, 0x00b9, 0x00d1, 0x00d2, 0x00cb, 0x00cf, 0x00d0, 0x00ca, 0x00d8, 0x00d7, 0x00ce,     -1,     -1,     -1,
        /* U+2570 */     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1 /* blank */,
        /* U+2580 */ 0x00df,     -1,     -1,     -1, 0x00dc,     -1,     -1,     -1, 0x00db,     -1,     -1,     -1, 0x00dd,     -1,     -1,     -1,
        /* U+2590 */ 0x00de, 0x00b0, 0x00b1, 0x00b2,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,
        /* U+25a0 */ 0x00fe,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1
};
//      /* U+0190 */     -1,     -1, 0x009f,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,
//      /* U+2070 */     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1, 0x00fc,
//      /* U+20a0 */     -1,     -1,     -1,     -1,     -1,     -1,     -1, 0x009e,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,
//      /* U+2310 */ 0x00a9,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,
//      /* U+2320 */ 0x00f4, 0x00f5,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,

//-----------------------------------------------------------------------------
// CP 852
//-----------------------------------------------------------------------------

const int TEncoding_CP852::toUnicodeTable[8 * 16] = {
        /* 0x80 */ 0x00c7, 0x00fc, 0x00e9, 0x00e2, 0x00e4, 0x016f, 0x0107, 0x00e7, 0x0142, 0x00eb, 0x0150, 0x0151, 0x00ee, 0x0179, 0x00c4, 0x0106,
        /* 0x90 */ 0x00c9, 0x0139, 0x013a, 0x00f4, 0x00f6, 0x013d, 0x013e, 0x015a, 0x015b, 0x00d6, 0x00dc, 0x0164, 0x0165, 0x0141, 0x00d7, 0x010d,
        /* 0xa0 */ 0x00e1, 0x00ed, 0x00f3, 0x00fa, 0x0104, 0x0105, 0x017d, 0x017e, 0x0118, 0x0119, 0x00ac, 0x017a, 0x010c, 0x015f, 0x00ab, 0x00bb,
        /* 0xb0 */ 0x2591, 0x2592, 0x2593, 0x2502, 0x2524, 0x00c1, 0x00c2, 0x011a, 0x015e, 0x2563, 0x2551, 0x2557, 0x255d, 0x017b, 0x017c, 0x2510,
        /* 0xc0 */ 0x2514, 0x2534, 0x252c, 0x251c, 0x2500, 0x253c, 0x0102, 0x0103, 0x255a, 0x2554, 0x2569, 0x2566, 0x2560, 0x2550, 0x256c, 0x00a4,
        /* 0xd0 */ 0x0111, 0x0110, 0x010e, 0x00cb, 0x010f, 0x0147, 0x00cd, 0x00ce, 0x011b, 0x2518, 0x250c, 0x2588, 0x2584, 0x0162, 0x016e, 0x2580,
        /* 0xe0 */ 0x00d3, 0x00df, 0x00d4, 0x0143, 0x0144, 0x0148, 0x0160, 0x0161, 0x0154, 0x00da, 0x0155, 0x0170, 0x00fd, 0x00dd, 0x0163, 0x00b4,
        /* 0xf0 */ 0x00ad, 0x02dd, 0x02db, 0x02c7, 0x02d8, 0x00a7, 0x00f7, 0x00b8, 0x00b0, 0x00a8, 0x02d9, 0x0171, 0x0158, 0x0159, 0x25a0, 0x00a0
};

const int TEncoding_CP852::fromUnicodeTable1[14 * 16] = {
        /* U+00a0 */ 0x00ff,     -1,     -1,     -1, 0x00cf,     -1,     -1, 0x00f5, 0x00f9,     -1,     -1, 0x00ae, 0x00aa, 0x00f0,     -1,     -1,
        /* U+00b0 */ 0x00f8,     -1,     -1,     -1, 0x00ef,     -1,     -1,     -1, 0x00f7,     -1,     -1, 0x00af,     -1,     -1,     -1,     -1,
        /* U+00c0 */     -1, 0x00b5, 0x00b6,     -1, 0x008e,     -1,     -1, 0x0080,     -1, 0x0090,     -1, 0x00d3,     -1, 0x00d6, 0x00d7,     -1,
        /* U+00d0 */     -1,     -1,     -1, 0x00e0, 0x00e2,     -1, 0x0099, 0x009e,     -1,     -1, 0x00e9,     -1, 0x009a, 0x00ed,     -1, 0x00e1,
        /* U+00e0 */     -1, 0x00a0, 0x0083,     -1, 0x0084,     -1,     -1, 0x0087,     -1, 0x0082,     -1, 0x0089,     -1, 0x00a1, 0x008c,     -1,
        /* U+00f0 */     -1,     -1,     -1, 0x00a2, 0x0093,     -1, 0x0094, 0x00f6,     -1,     -1, 0x00a3,     -1, 0x0081, 0x00ec,     -1,     -1,
        /* U+0100 */     -1,     -1, 0x00c6, 0x00c7, 0x00a4, 0x00a5, 0x008f, 0x0086,     -1,     -1,     -1,     -1, 0x00ac, 0x009f, 0x00d2, 0x00d4,
        /* U+0110 */ 0x00d1, 0x00d0,     -1,     -1,     -1,     -1,     -1,     -1, 0x00a8, 0x00a9, 0x00b7, 0x00d8,     -1,     -1,     -1,     -1,
        /* U+0120 */     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1 /* blank */,
        /* U+0130 */     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1, 0x0091, 0x0092,     -1,     -1, 0x0095, 0x0096,     -1,
        /* U+0140 */     -1, 0x009d, 0x0088, 0x00e3, 0x00e4,     -1,     -1, 0x00d5, 0x00e5,     -1,     -1,     -1,     -1,     -1,     -1,     -1,
        /* U+0150 */ 0x008a, 0x008b,     -1,     -1, 0x00e8, 0x00ea,     -1,     -1, 0x00fc, 0x00fd, 0x0097, 0x0098,     -1,     -1, 0x00b8, 0x00ad,
        /* U+0160 */ 0x00e6, 0x00e7, 0x00dd, 0x00ee, 0x009b, 0x009c,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1, 0x00de, 0x0085,
        /* U+0170 */ 0x00eb, 0x00fb,     -1,     -1,     -1,     -1,     -1,     -1,     -1, 0x008d, 0x00ab, 0x00bd, 0x00be, 0x00a6, 0x00a7,     -1
};

const int TEncoding_CP852::fromUnicodeTable2[2* 16] = {
        /* U+02c0 */     -1,     -1,     -1,     -1,     -1,     -1,     -1, 0x00f3,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,
        /* U+02d0 */     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1, 0x00f4, 0x00fa,     -1, 0x00f2,     -1, 0x00f1,     -1,     -1
};

const int TEncoding_CP852::fromUnicodeTable3[11 * 16] = {
        /* U+2500 */ 0x00c4,     -1, 0x00b3,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1, 0x00da,     -1,     -1,     -1,
        /* U+2510 */ 0x00bf,     -1,     -1,     -1, 0x00c0,     -1,     -1,     -1, 0x00d9,     -1,     -1,     -1, 0x00c3,     -1,     -1,     -1,
        /* U+2520 */     -1,     -1,     -1,     -1, 0x00b4,     -1,     -1,     -1,     -1,     -1,     -1,     -1, 0x00c2,     -1,     -1,     -1,
        /* U+2530 */     -1,     -1,     -1,     -1, 0x00c1,     -1,     -1,     -1,     -1,     -1,     -1,     -1, 0x00c5,     -1,     -1,     -1,
        /* U+2540 */     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1 /* blank */,
        /* U+2550 */ 0x00cd, 0x00ba,     -1,     -1, 0x00c9,     -1,     -1, 0x00bb,     -1,     -1, 0x00c8,     -1,     -1, 0x00bc,     -1,     -1,
        /* U+2560 */ 0x00cc,     -1,     -1, 0x00b9,     -1,     -1, 0x00cb,     -1,     -1, 0x00ca,     -1,     -1, 0x00ce,     -1,     -1,     -1,
        /* U+2570 */     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1 /* blank */,
        /* U+2580 */ 0x00df,     -1,     -1,     -1, 0x00dc,     -1,     -1,     -1, 0x00db,     -1,     -1,     -1,     -1,     -1,     -1,     -1,
        /* U+2590 */     -1, 0x00b0, 0x00b1, 0x00b2,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,
        /* U+25a0 */ 0x00fe,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1
};

//-----------------------------------------------------------------------------
// Windows-1250
//-----------------------------------------------------------------------------

const int TEncoding_CP1250::toUnicodeTable[8 * 16] = {
        /* 0x80 */ 0x20ac,     -1, 0x201a,     -1, 0x201e, 0x2026, 0x2020, 0x2021,     -1, 0x2030, 0x0160, 0x2039, 0x015a, 0x0164, 0x017d, 0x0179,
        /* 0x90 */     -1, 0x2018, 0x2019, 0x201c, 0x201d, 0x2022, 0x2013, 0x2014,     -1, 0x2122, 0x0161, 0x203a, 0x015b, 0x0165, 0x017e, 0x017a,
        /* 0xa0 */ 0x00a0, 0x02c7, 0x02d8, 0x0141, 0x00a4, 0x0104, 0x00a6, 0x00a7, 0x00a8, 0x00a9, 0x015e, 0x00ab, 0x00ac, 0x00ad, 0x00ae, 0x017b,
        /* 0xb0 */ 0x00b0, 0x00b1, 0x02db, 0x0142, 0x00b4, 0x00b5, 0x00b6, 0x00b7, 0x00b8, 0x0105, 0x015f, 0x00bb, 0x013d, 0x02dd, 0x013e, 0x017c,
        /* 0xc0 */ 0x0154, 0x00c1, 0x00c2, 0x0102, 0x00c4, 0x0139, 0x0106, 0x00c7, 0x010c, 0x00c9, 0x0118, 0x00cb, 0x011a, 0x00cd, 0x00ce, 0x010e,
        /* 0xd0 */ 0x0110, 0x0143, 0x0147, 0x00d3, 0x00d4, 0x0150, 0x00d6, 0x00d7, 0x0158, 0x016e, 0x00da, 0x0170, 0x00dc, 0x00dd, 0x0162, 0x00df,
        /* 0xe0 */ 0x0155, 0x00e1, 0x00e2, 0x0103, 0x00e4, 0x013a, 0x0107, 0x00e7, 0x010d, 0x00e9, 0x0119, 0x00eb, 0x011b, 0x00ed, 0x00ee, 0x010f,
        /* 0xf0 */ 0x0111, 0x0144, 0x0148, 0x00f3, 0x00f4, 0x0151, 0x00f6, 0x00f7, 0x0159, 0x016f, 0x00fa, 0x0171, 0x00fc, 0x00fd, 0x0163, 0x02d9
};

const int TEncoding_CP1250::fromUnicodeTable1[14 * 16] = {
        /* U+00a0 */ 0x00a0,     -1,     -1,     -1, 0x00a4,     -1, 0x00a6, 0x00a7, 0x00a8, 0x00a9,     -1, 0x00ab, 0x00ac, 0x00ad, 0x00ae,     -1,
        /* U+00b0 */ 0x00b0, 0x00b1,     -1,     -1, 0x00b4, 0x00b5, 0x00b6, 0x00b7, 0x00b8,     -1,     -1, 0x00bb,     -1,     -1,     -1,     -1,
        /* U+00c0 */     -1, 0x00c1, 0x00c2,     -1, 0x00c4,     -1,     -1, 0x00c7,     -1, 0x00c9,     -1, 0x00cb,     -1, 0x00cd, 0x00ce,     -1,
        /* U+00d0 */     -1,     -1,     -1, 0x00d3, 0x00d4,     -1, 0x00d6, 0x00d7,     -1,     -1, 0x00da,     -1, 0x00dc, 0x00dd,     -1, 0x00df,
        /* U+00e0 */     -1, 0x00e1, 0x00e2,     -1, 0x00e4,     -1,     -1, 0x00e7,     -1, 0x00e9,     -1, 0x00eb,     -1, 0x00ed, 0x00ee,     -1,
        /* U+00f0 */     -1,     -1,     -1, 0x00f3, 0x00f4,     -1, 0x00f6, 0x00f7,     -1,     -1, 0x00fa,     -1, 0x00fc, 0x00fd,     -1,     -1,
        /* U+0100 */     -1,     -1, 0x00c3, 0x00e3, 0x00a5, 0x00b9, 0x00c6, 0x00e6,     -1,     -1,     -1,     -1, 0x00c8, 0x00e8, 0x00cf, 0x00ef,
        /* U+0110 */ 0x00d0, 0x00f0,     -1,     -1,     -1,     -1,     -1,     -1, 0x00ca, 0x00ea, 0x00cc, 0x00ec,     -1,     -1,     -1,     -1,
        /* U+0120 */     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1 /* blank */,
        /* U+0130 */     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1, 0x00c5, 0x00e5,     -1,     -1, 0x00bc, 0x00be,     -1,
        /* U+0140 */     -1, 0x00a3, 0x00b3, 0x00d1, 0x00f1,     -1,     -1, 0x00d2, 0x00f2,     -1,     -1,     -1,     -1,     -1,     -1,     -1,
        /* U+0150 */ 0x00d5, 0x00f5,     -1,     -1, 0x00c0, 0x00e0,     -1,     -1, 0x00d8, 0x00f8, 0x008c, 0x009c,     -1,     -1, 0x00aa, 0x00ba,
        /* U+0160 */ 0x008a, 0x009a, 0x00de, 0x00fe, 0x008d, 0x009d,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1, 0x00d9, 0x00f9,
        /* U+0170 */ 0x00db, 0x00fb,     -1,     -1,     -1,     -1,     -1,     -1,     -1, 0x008f, 0x009f, 0x00af, 0x00bf, 0x008e, 0x009e,     -1,
};

const int TEncoding_CP1250::fromUnicodeTable2[2 * 16] = {
        /* U+02c0 */     -1,     -1,     -1,     -1,     -1,     -1,     -1, 0x00a1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,
        /* U+02d0 */     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1, 0x00a2, 0x00ff,     -1, 0x00b2,     -1, 0x00bd,     -1,     -1,
};

const int TEncoding_CP1250::fromUnicodeTable3[3 * 16] = {
        /* U+2010 */     -1,     -1,     -1, 0x0096, 0x0097,     -1,     -1,     -1, 0x0091, 0x0092, 0x0082,     -1, 0x0093, 0x0094, 0x0084,     -1,
        /* U+2020 */ 0x0086, 0x0087, 0x0095,     -1,     -1,     -1, 0x0085,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,
        /* U+2030 */ 0x0089,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1, 0x008b, 0x009b,     -1,     -1,     -1,     -1,     -1,
};
//      /* U+20a0 */     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1, 0x0080,     -1,     -1,     -1,
//      /* U+2120 */     -1,     -1, 0x0099,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1,     -1

//-----------------------------------------------------------------------------
// YU-ASCII
//-----------------------------------------------------------------------------

//                                                C acute c acute C caron c caron S caron s caron Z caron z caron D stroke d stroke
const int TEncoding_YuAscii::uniChars[10] =     {  0x106,  0x107,  0x10c,  0x10d,  0x160,  0x161,  0x17d,  0x17e,   0x110,  0x111  };
const int TEncoding_YuAscii::yuAsciiChars[10] = {   0x5d,   0x7d,   0x5e,   0x7e,   0x5b,   0x7b,   0x40,   0x60,    0x5c,   0x7c  };
//                                                   ']'     '}'     '^'    '~'     '['     '{'     '@'     '`'      '\\'    '|'


//-----------------------------------------------------------------------------
// TUnicode - codec registry
//-----------------------------------------------------------------------------

void TUnicode::InitCodecs()
{
        ClrCodecs();
        RegisterCodec("ISO-8859-1 ISO_8859-1 ISO_8859-1:1987 ISO-IR-100 CP819 IBM819 LATIN1 L1 csISOLatin1 ISO8859-1 ISO8859_1 CP28591", TCodecBase::New<TCodec_ISO8859_1>());
        RegisterCodec("ISO-8859-2 ISO_8859-2 ISO_8859-2:1987 ISO-IR-101 LATIN2 L2 csISOLatin2 ISO8859-2 ISO8859_2 CP28592", TCodecBase::New<TCodec_ISO8859_2>());
        RegisterCodec("ISO-8859-3 ISO_8859-3 ISO_8859-3:1988 ISO-IR-109 LATIN3 L3 csISOLatin3 ISO8859-3 ISO8859_3 CP28593", TCodecBase::New<TCodec_ISO8859_3>());
        RegisterCodec("ISO-8859-4 ISO_8859-4 ISO_8859-4:1988 ISO-IR-110 LATIN4 L4 csISOLatin4 ISO8859-4 ISO8859_4 CP28594", TCodecBase::New<TCodec_ISO8859_4>());
        RegisterCodec("YUASCII YU-ASCII YU_ASCII", TCodecBase::New<TCodec_YuAscii>());
        RegisterCodec("CP1250 Windows-1250 MS-EE", TCodecBase::New<TCodec_CP1250>());
        RegisterCodec("CP852 cp852_DOSLatin2 DOSLatin2", TCodecBase::New<TCodec_CP852>());
        RegisterCodec("CP437 cp437_DOSLatinUS DOSLatinUS", TCodecBase::New<TCodec_CP437>());
}

void TUnicode::EncodeUtf8(const uint& c, TChA& dest) {
        if (c > 0x10ffff) {
                throw TExcept::New(TStr::Fmt("Unkown Unicode character %u", c)); }
        if (c < 0x80u)
                dest.AddCh(char(c & 0xffu));
        else if (c < 0x800u) {
                dest.AddCh(char(TUniCodec::_1100_0000 | ((c >> 6) & TUniCodec::_0001_1111)));
                dest.AddCh(char(TUniCodec::_1000_0000 | (c & TUniCodec::_0011_1111))); }
        else if (c < 0x10000u) {
                dest.AddCh(char(TUniCodec::_1110_0000 | ((c >> 12) & TUniCodec::_0000_1111)));
                dest.AddCh(char(TUniCodec::_1000_0000 | ((c >> 6) & TUniCodec::_0011_1111)));
                dest.AddCh(char(TUniCodec::_1000_0000 | (c & TUniCodec::_0011_1111))); }
        else if (c < 0x200000u) {
                dest.AddCh(char(TUniCodec::_1111_0000 | ((c >> 18) & TUniCodec::_0000_0111)));
                dest.AddCh(char(TUniCodec::_1000_0000 | ((c >> 12) & TUniCodec::_0011_1111)));
                dest.AddCh(char(TUniCodec::_1000_0000 | ((c >> 6) & TUniCodec::_0011_1111)));
                dest.AddCh(char(TUniCodec::_1000_0000 | (c & TUniCodec::_0011_1111))); }
        else if (c < 0x4000000u) {
                dest.AddCh(char(TUniCodec::_1111_1000 | ((c >> 24) & TUniCodec::_0000_0011)));
                dest.AddCh(char(TUniCodec::_1000_0000 | ((c >> 18) & TUniCodec::_0011_1111)));
                dest.AddCh(char(TUniCodec::_1000_0000 | ((c >> 12) & TUniCodec::_0011_1111)));
                dest.AddCh(char(TUniCodec::_1000_0000 | ((c >> 6) & TUniCodec::_0011_1111)));
                dest.AddCh(char(TUniCodec::_1000_0000 | (c & TUniCodec::_0011_1111))); }
        else {
                dest.AddCh(char(TUniCodec::_1111_1100 | ((c >> 30) & TUniCodec::_0000_0011)));
                dest.AddCh(char(TUniCodec::_1000_0000 | ((c >> 24) & TUniCodec::_0011_1111)));
                dest.AddCh(char(TUniCodec::_1000_0000 | ((c >> 18) & TUniCodec::_0011_1111)));
                dest.AddCh(char(TUniCodec::_1000_0000 | ((c >> 12) & TUniCodec::_0011_1111)));
                dest.AddCh(char(TUniCodec::_1000_0000 | ((c >> 6) & TUniCodec::_0011_1111)));
                dest.AddCh(char(TUniCodec::_1000_0000 | (c & TUniCodec::_0011_1111))); }
}

TStr TUnicode::EncodeUtf8(const uint& Ch) {
        TChA ChA; EncodeUtf8(Ch, ChA); return ChA;
}