Files
RedBear-OS/local/recipes/kde/kf6-kcodecs/source/autotests/kencodingproberunittest.cpp
T
vasilito b8aac3c9bc D7: editor multi-cursor support
Add secondary_cursors field to Editor with insert_char_multi,
delete_back_multi, delete_forward_multi methods. Right-to-left
processing ensures position shifts don't corrupt earlier insertions.

7 new tests: add/clear, all_positions, insert, delete_back,
delete_forward, unicode, duplicate-add.
2026-07-05 22:29:19 +03:00

425 lines
17 KiB
C++
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
/*
SPDX-FileCopyrightText: 2025 Stefan Brüns <stefan.bruens@rwth-aachen.de>
SPDX-License-Identifier: GPL-2.0-or-later
*/
#include <QTest>
#include "../src/probers/nsEscSM.h"
#include "../src/probers/nsMBCSSM.h"
#include "../src/probers/nsCodingStateMachine.h"
#include "../src/probers/nsHebrewProber.h"
#include "../src/probers/nsLatin1Prober.h"
class KEncodingProberUnitTest : public QObject
{
Q_OBJECT
private Q_SLOTS:
void testUtf8();
void testUtf8_data();
void testUtf16BE();
void testUtf16BE_data();
void testUtf16LE();
void testUtf16LE_data();
void testUtf16_common_data();
void testWindows1252();
void testWindows1252_data();
void testWindows1255();
void testWindows1255_data();
void testEsc_common_data();
void testHzCharset();
void testHzCharset_data();
void testIso2022JPCharset();
void testIso2022JPCharset_data();
};
void KEncodingProberUnitTest::testUtf8()
{
QFETCH(QByteArray, data);
QFETCH(bool, utf8Valid);
using namespace kencodingprober;
nsCodingStateMachine stateMachine{&UTF8SMModel};
nsSMState state = eStart;
for (auto b : data) {
state = stateMachine.NextState(b);
}
QCOMPARE((state != eError), utf8Valid);
}
void KEncodingProberUnitTest::testUtf8_data()
{
using namespace Qt::StringLiterals;
QTest::addColumn<QByteArray>("data");
QTest::addColumn<bool>("utf8Valid");
QTest::addRow("UTF-8 Latin1") << "abcdxyzABCDXYZ 0129;,"_ba << true;
QTest::addRow("BOM UTF-8") << "\xef\xbb\xbfZ"_ba << true; // "<UTF-8 BOM>Z"
// multibyte sequences - length 2
QTest::addRow("UTF-8 Latin1 Supplement") //
<< "Latin1 Text \xC3\xA4\xC3\xB6\xC3\xBC\xC3\x9F"_ba // "Latin1 Text äöüß"
<< true;
QTest::addRow("UTF-8 len 2") << "Text \xC3\xA4 "_ba << true;
QTest::addRow("UTF-8 len 2 short") << "Text \xC3 "_ba << false;
QTest::addRow("UTF-8 len 2 invalid range") << "Text \xC0\x90 "_ba << false;
// multibyte sequences - length 3
QTest::addRow("UTF-8 CJK") //
<< QByteArray::fromHex("e998bfe5b094e58d91e696afe5b1b1e88489") // 阿尔卑斯山脉
<< true;
QTest::addRow("UTF-8 len 3 a") << "Text \xE2\x80\x90 "_ba << true; // "" (HYPHEN)
QTest::addRow("UTF-8 len 3-1 short") << "Text \xE2\x80 "_ba << false;
QTest::addRow("UTF-8 len 3-2 short") << "Text \xE2 "_ba << false;
QTest::addRow("UTF-8 len 3 b") << "Text \xE0\xbf\xbf "_ba << true; // "๏" (THAI CHARACTER FONGMAN)
QTest::addRow("UTF-8 len 3 invalid range") << "Text \xE0\x9f\x90 "_ba << false;
QTest::addRow("UTF-8 len 3 c") << "Text \xED\x80\x80 "_ba << true; // "퀀" (HANGUL SYLLABLE KWEON)
QTest::addRow("UTF-8 invalid CESU") << "Text \xED\xbf\x80 "_ba << false;
// multibyte sequences - length 4
QTest::addRow("UTF-8 SMP Symbols") << "\xF0\x9F\x82\xA1 "_ba << true; // "🂡 " (ACE OF SPADES)
QTest::addRow("UTF-8 len 4-1 short") << "\xF0\x9F\x82 "_ba << false;
QTest::addRow("UTF-8 len 4-2 short") << "\xF0\x9F "_ba << false;
QTest::addRow("UTF-8 len 4-3 short") << "\xF0 "_ba << false;
QTest::addRow("UTF-8 len 4 invalid long") << "\xF0\x8F\x90\x90 "_ba << false;
QTest::addRow("UTF-8 len 4 invalid range") << "\xF5\x90\x90\x90 "_ba << false;
// multibyte sequences - length 5/6 (invalid)
QTest::addRow("UTF-8 len 5 invalid") << "\xF8\x90\x90\x90\x90 "_ba << false;
QTest::addRow("UTF-8 len 6 invalid") << "\xFC\x90\x90\x90\x90\x90 "_ba << false;
QTest::addRow("UTF-8 0xFE invalid") << "\xFE "_ba << false;
QTest::addRow("UTF-8 0xFF invalid") << "\xFF "_ba << false;
// continuation without leading 2/3/4 byte start byte
QTest::addRow("UTF-8 invalid isolate high 0x80") << "\x80 "_ba << false;
QTest::addRow("UTF-8 invalid isolate high 0x92") << "\x92 "_ba << false;
QTest::addRow("UTF-8 invalid isolate high 0xAA") << "\xAA "_ba << false;
QTest::addRow("UTF-8 invalid isolate high 0xBF") << "\xBF "_ba << false;
// Either Windows-1252/-1254/-1255 (binary identical)
// "One pound, i.e. ½ a kilogramm of butter costs 2 £."
QTest::addRow("Windows-125x English") << "One pound, i.e. \xAF a kilogramm of butter costs 2 \xA3."_ba << false;
// Example texts with Windows-125x encoding which are definitely not UTF-8 -- see Wikipedia "Pangram"
// "Příliš žluťoučký kůň úpěl ďábelské ódy" - "A horse that was too yellow moaned devilish odes"
QTest::addRow("Windows-1250 Czech") << //
"P\xf8\xedli\x9a \x9elu\x9dou\xe8k\xfd k\xf9\xf2 \xfap\xecl \xef\xe1\x62\x65lsk\xe9 \xf3\x64y."_ba << false;
// "Под южно дърво, цъфтящо в синьо, бягаше малко пухкаво зайче" - "Under a southern tree, blooming in blue, ran a little fluffy bunny"
QTest::addRow("Windows-1251 Bulgarian") << QByteArray::fromHex( //
"cfeee420fee6edee20e4faf0e2ee2c20f6faf4f2fff9ee20"
"e220f1e8edfcee2c20e1ffe3e0f8e520ece0ebeaee20eff3"
"f5eae0e2ee20e7e0e9f7e5") << false;
// "Victor jagt zwölf Boxkämpfer quer über den großen Sylter Deich." - "Victor chases twelve boxers across the Great Levee of Sylt"
QTest::addRow("Windows-1252 German") << //
"Victor jagt zw\xf6lf Boxk\xe4mpfer quer \xfc\x62\x65r den gro\xdf\x65n Sylter Deich."_ba << false;
// "שפן אכל קצת גזר בטעם חסה, ודי" - "A bunny ate some lettuce-flavored carrots, and he had enough"
QTest::addRow("Windows-1255 Hebrew") //
<< QByteArray::fromHex("f9f4ef20e0ebec20f7f6fa20e2e6f820e1e8f2ed20e7f1e42c20e5e3e9") << false;
}
void KEncodingProberUnitTest::testUtf16BE()
{
QFETCH(QByteArray, data);
QFETCH(bool, utf16BEValid);
using namespace kencodingprober;
nsCodingStateMachine stateMachine{&UCS2BESMModel};
nsSMState state = eStart;
QEXPECT_FAIL("UTF16 ZWNBSP little", "zero width no-break space rejected", Abort);
for (auto b : data) {
state = stateMachine.NextState(b);
}
QCOMPARE((state != eError), utf16BEValid);
}
void KEncodingProberUnitTest::testUtf16LE()
{
QFETCH(QByteArray, data);
QFETCH(bool, utf16LEValid);
using namespace kencodingprober;
nsCodingStateMachine stateMachine{&UCS2LESMModel};
nsSMState state = eStart;
QEXPECT_FAIL("UTF16 ZWNBSP big", "zero width no-break space rejected", Abort);
for (auto b : data) {
state = stateMachine.NextState(b);
}
QCOMPARE((state != eError), utf16LEValid);
}
void KEncodingProberUnitTest::testUtf16_common_data()
{
QTest::addColumn<QByteArray>("data");
QTest::addColumn<bool>("utf16BEValid");
QTest::addColumn<bool>("utf16LEValid");
QTest::addRow("empty") << QByteArray() << true << true;
// BOM must be detected
QTest::addRow("BE BOM") << QByteArray("\xFE\xFF") << true << false;
QTest::addRow("LE BOM") << QByteArray("\xFF\xFE") << false << true;
// swapped endianess does not cause an error, as the codepoint is still valid
QTest::addRow("BE HS+LS") << QByteArray("\x01\x01\xD8\x00\xDC\x00", 6) << true << true;
QTest::addRow("LE HS+LS") << QByteArray("\x01\x01\x00\xD8\x00\xDC", 6) << true << true;
QTest::addRow("BE HS+LS at start") << QByteArray("\xD8\x00\xDC\x00", 4) << true << true;
QTest::addRow("LE HS+LS at start") << QByteArray("\x00\xD8\x00\xDC", 4) << true << true;
// if swapped endianess results in a low surrogate, the input is invalid
QTest::addRow("BE LS") << QByteArray("\x01\x01\xDC\x00", 4) << false << true;
QTest::addRow("LE LS") << QByteArray("\x01\x01\x00\xDC", 4) << true << false;
QTest::addRow("BE LS at start") << QByteArray("\xDC\x00", 2) << false << true;
QTest::addRow("LE LS at start") << QByteArray("\x00\xDC", 2) << true << false;
struct Utf16TestData {
const char *name;
const std::span<const char16_t> data;
bool validBig;
bool validLittle;
};
using namespace std::string_view_literals;
constexpr std::array<Utf16TestData, 7> utf16TestData = {
// syntactically correct even with wrong endianess
Utf16TestData{"UTF16 XY", u"XY"sv, true, true},
Utf16TestData{"UTF16 ab", u"ab"sv, true, true},
Utf16TestData{"UTF16 äöü", u"äöü"sv, true, true},
Utf16TestData{"UTF16 BOM", u"\xFEFF"sv, true, false},
// "" or "ᬠ" (U+1B20 BALINESE LETTER DA MURDA MAHAPRANA)
Utf16TestData{"UTF16 Interpunctuation", u"\x201B"sv, true, true},
// "" or "⼪" (U+2F2A KANGXI RADICAL LAME)
Utf16TestData{"UTF16 Math supplement", u"\x2A2F"sv, true, true},
// ZWNBSP aka BOM inside the document is deprecated, but valid
Utf16TestData{"UTF16 ZWNBSP", u" \xFEFF"sv, true, true},
};
for (const auto &tc : utf16TestData) {
QByteArray data;
data.resize(tc.data.size() * 2);
qToBigEndian<quint16>(tc.data.data(), tc.data.size(), data.data());
QTest::addRow("%s big", tc.name) << data << tc.validBig << tc.validLittle;
qToLittleEndian<quint16>(tc.data.data(), tc.data.size(), data.data());
QTest::addRow("%s little", tc.name) << data << tc.validLittle << tc.validBig;
}
}
void KEncodingProberUnitTest::testUtf16BE_data()
{
testUtf16_common_data();
}
void KEncodingProberUnitTest::testUtf16LE_data()
{
testUtf16_common_data();
}
void KEncodingProberUnitTest::testWindows1252()
{
using namespace kencodingprober;
QFETCH(QByteArray, data);
QFETCH(bool, win1252Valid);
QFETCH(double, minConfidence);
nsLatin1Prober win1252Prober{};
auto state = win1252Prober.HandleData(data.constData(), data.size());
QCOMPARE(win1252Valid, (state != eNotMe));
QEXPECT_FAIL("Windows-1252 German", "Valid vowel + sharp s considered unlikely", Abort);
QEXPECT_FAIL("Windows-1252 French", "Accented vowel pair considered unlikely", Abort);
QEXPECT_FAIL("Windows-1252 Spanish", "Accented vowel pair considered unlikely", Abort);
if (win1252Valid) {
QCOMPARE_GE(win1252Prober.GetConfidence(), minConfidence);
}
}
void KEncodingProberUnitTest::testWindows1252_data()
{
using namespace Qt::StringLiterals;
QTest::addColumn<QByteArray>("data");
QTest::addColumn<bool>("win1252Valid");
QTest::addColumn<double>("minConfidence");
// Verify "undefined" codes in CP1252 are rejected
// https://www.unicode.org/Public/MAPPINGS/VENDORS/MICSFT/WINDOWS/CP1252.TXT
for (auto c : std::array<unsigned char, 5>{0x81, 0x8D, 0x8F, 0x90, 0x9D}) {
QTest::addRow("Undefined 0x%x", c) << QByteArray(1, c) + " "_ba << false << 0.0;
}
// "Victor jagt zwölf Boxkämpfer quer über den großen Sylter Deich." - "Victor chases twelve boxers across the Great Levee of Sylt"
QTest::addRow("Windows-1252 German") << //
"Victor jagt zw\xf6lf Boxk\xe4mpfer quer \xfc\x62\x65r den gro\xdf\x65n Sylter Deich."_ba << true << 0.45;
// Same "Victor jagt zwölf Boxkämpfer quer über den großen Sylter Deich.", but using UTF-8
// Unfortunately, these are all valid codepoints for Windows-1252, but the confidence should be lower than for UTF-8,
// as it contains UTF-8 multibyte sequences, and several capital characters following small characters for Windows-1252.
QTest::addRow("UTF-8 German") << //
"Victor jagt zw\xc3\xb6lf Boxk\xc3\xa4mpfer quer \xc3\xbc\x62\x65r den gro\xc3\x9f\x65n Sylter Deich."_ba << true << 0.0;
// "L'ergothérapeute effectue des prestations de rééducation et de réadaptation."
// - "The occupational therapist provides rehabilitation and re-adaptation services."
QTest::addRow("Windows-1252 French") << //
"L'ergoth\xe9rapeute effectue des prestations de r\xe9\xe9\x64ucation et de r\xe9\x61\x64\x61ptation."_ba << true << 0.45;
// "La sociolingüística sincrónica se centra en la estructura sociolingüística y en las variaciones lingüísticas."
// - "Synchronic sociolinguistics focuses on sociolinguistic structure and linguistic variations."
QTest::addRow("Windows-1252 Spanish") << //
"La socioling\xfc\xedstica sincr\xf3nica se centra en la estructura socioling\xfc\xedstica y en las variaciones ling\xfc\xedsticas."_ba << true << 0.45;
// "שפן אכל קצת גזר בטעם חסה, ודי" - "A bunny ate some lettuce-flavored carrots, and he had enough"
QTest::addRow("Windows-1255 Hebrew") //
<< QByteArray::fromHex("f9f4ef20e0ebec20f7f6fa20e2e6f820e1e8f2ed20e7f1e42c20e5e3e9") << true << 0.0;
// "שפן אכל קצת גזר בטעם חסה, ודי", but using UTF-8
QTest::addRow("UTF-8 Hebrew") << QByteArray::fromHex( //
"d7a9d7a4d79f20d790d79bd79c20d7a7d7a6d7aa20d792d7"
"96d7a820d791d798d7a2d79d20d797d7a1d7942c20d795d7"
"93d799") << false << 0.0;
}
void KEncodingProberUnitTest::testWindows1255()
{
using namespace kencodingprober;
QFETCH(QByteArray, data);
QFETCH(bool, win1255Valid);
nsHebrewProber win1255Prober{};
if (QByteArray(QTest::currentDataTag(), -1).startsWith("Undefined 0x")) {
QEXPECT_FAIL("", "Invalid value accepted", Abort);
}
QEXPECT_FAIL("UTF-8 Hebrew", "Invalid 0x9f accepted", Abort);
auto state = win1255Prober.HandleData(data.constData(), data.size());
QCOMPARE(win1255Valid, (state != eNotMe));
}
void KEncodingProberUnitTest::testWindows1255_data()
{
using namespace Qt::StringLiterals;
QTest::addColumn<QByteArray>("data");
QTest::addColumn<bool>("win1255Valid");
// Verify "undefined" codes in CP1255 are rejected
// https://www.unicode.org/Public/MAPPINGS/VENDORS/MICSFT/WINDOWS/CP1255.TXT
for (auto c : std::array<unsigned char, 23>{
0x81, 0x8A, 0x8C, 0x8D, 0x8E, 0x8F, 0x90, 0x9A, 0x9C, 0x9D, 0x9E, 0x9F, //
0xCA, 0xD9, 0xDA, 0xDB, 0xDC, 0xDD, 0xDE, 0xDF, 0xFB, 0xFC, 0xFF,
}) {
QTest::addRow("Undefined 0x%x", c) << QByteArray(1, c) + " "_ba << false;
}
// "שפן אכל קצת גזר בטעם חסה, ודי" - "A bunny ate some lettuce-flavored carrots, and he had enough"
QTest::addRow("Windows-1255 Hebrew") //
<< QByteArray::fromHex("f9f4ef20e0ebec20f7f6fa20e2e6f820e1e8f2ed20e7f1e42c20e5e3e9") << true;
// Same "שפן אכל קצת גזר בטעם חסה, ודי", but using UTF-8
QTest::addRow("UTF-8 Hebrew") << QByteArray::fromHex( //
"d7a9d7a4d79f20d790d79bd79c20d7a7d7a6d7aa20d792d7"
"96d7a820d791d798d7a2d79d20d797d7a1d7942c20d795d7"
"93d799") << false;
}
void KEncodingProberUnitTest::testHzCharset()
{
QFETCH(QByteArray, data);
QFETCH(bool, hzValid);
using namespace kencodingprober;
nsCodingStateMachine stateMachine{&HZSMModel};
nsSMState state = eStart;
QEXPECT_FAIL("HZ odd", "0x7d '}' accepted as lead byte", Abort);
for (auto b : data) {
state = stateMachine.NextState(b);
}
QCOMPARE((state != eError), hzValid);
}
void KEncodingProberUnitTest::testIso2022JPCharset()
{
QFETCH(QByteArray, data);
QFETCH(bool, iso2022JPValid);
using namespace kencodingprober;
nsCodingStateMachine stateMachine{&ISO2022JPSMModel};
nsSMState state = eStart;
for (auto b : data) {
state = stateMachine.NextState(b);
}
qDebug() << data;
QCOMPARE((state != eError), iso2022JPValid);
}
void KEncodingProberUnitTest::testEsc_common_data()
{
using namespace Qt::StringLiterals;
QTest::addColumn<QByteArray>("data");
QTest::addColumn<bool>("hzValid");
QTest::addColumn<bool>("iso2022JPValid");
// ASCII without escape sequence is always valid
QTest::addRow("empty") << QByteArray() << true << true;
QTest::addRow("Latin1") << "abcdxyzABCDXYZ 0129;,"_ba << true << true;
// All codes >= 0x80 are invalid
QTest::addRow("UTF-8 BOM") << "\xef\xbb\xbfZ"_ba << false << false; // "<UTF-8 BOM>Z"
QTest::addRow("UTF-16BE BOM") << "\xFE\xFF"_ba << false << false;
QTest::addRow("UTF-16LE BOM") << "\xFF\xFE"_ba << false << false;
QTest::addRow("UTF-8 Latin1 Supplement") //
<< "Latin1 Text \xC3\xA4\xC3\xB6\xC3\xBC\xC3\x9F"_ba // "Latin1 Text äöüß"
<< false << false;
// HZ encoding is (a subset of) plain ASCII, i.e. also valid ISO-2022
QTest::addRow("HZ empty") << "~{~}"_ba << true << true;
QTest::addRow("HZ odd") << "~{R~}"_ba << false << true;
QTest::addRow("HZ valid") << "~{R;~}"_ba << true << true; // "一" (one)
QTest::addRow("HZ valid with ASCII") << "R~{R;~}R"_ba << true << true; // "R一R"
// "こんにちは" / "Kon'nichiwa" - "Hello"
// ESC $ B <JIS X 208-1983 Double Byte> ESC ( B
QTest::addRow("ISO-2022-JP") << QByteArray::fromHex( //
"1b 24 42 24 33 24 73 24 4b 24 41 24 4f 1b 28 42"
"") << false << true;
}
void KEncodingProberUnitTest::testHzCharset_data()
{
testEsc_common_data();
}
void KEncodingProberUnitTest::testIso2022JPCharset_data()
{
testEsc_common_data();
}
QTEST_MAIN(KEncodingProberUnitTest)
#include "kencodingproberunittest.moc"