diff --git a/kate/document/katebuffer.cpp b/kate/document/katebuffer.cpp index a236b28..ede0633 100644 --- a/kate/document/katebuffer.cpp +++ b/kate/document/katebuffer.cpp @@ -122,7 +122,7 @@ class KateFileLoader kDebug (13020) << "PROBER TYPE: " << KEncodingProber::nameForProberType(m_prober->proberType()); m_prober->feed(m_buffer.data(), c); if (m_prober->confidence() > 0.5) - m_codec = QTextCodec::codecForName(m_prober->encodingName()); + m_codec = QTextCodec::codecForName(m_prober->encoding()); m_utf8Borked=errorsIfUtf8(m_buffer.data(), c); m_binary=processNull(m_buffer.data(), c); m_text = decoder()->toUnicode(m_buffer, c); diff --git a/kdecore/localization/kencodingprober.cpp b/kdecore/localization/kencodingprober.cpp index ad4eebd..89838fa 100644 --- a/kdecore/localization/kencodingprober.cpp +++ b/kdecore/localization/kencodingprober.cpp @@ -34,12 +34,10 @@ #include -#define MINIMUM_THRESHOLD (float)0.2 - class KEncodingProberPrivate { public: - KEncodingProberPrivate(): prober(NULL), mStart(true) {}; + KEncodingProberPrivate(): encoding(NULL), prober(NULL), mStart(true) {}; ~KEncodingProberPrivate() { delete prober; @@ -51,7 +49,10 @@ public: * and have to use some Stastics methods. * for single-byte encodings (most western encodings), nsSBCSGroupProber is ok, * because encoding state machine can detect many such encodings. - */ + */ + + delete prober; + switch (proberType) { case KEncodingProber::None: prober = NULL; @@ -129,17 +130,13 @@ public: break; } // switch - if (!encoding.isEmpty()) - { + if (encoding) proberState = KEncodingProber::FoundIt; - currentConfidence = 0.99f; - } } } KEncodingProber::ProberType proberType; KEncodingProber::ProberState proberState; - float currentConfidence; - QString encoding; + const char *encoding; kencodingprober::nsCharSetProber *prober; bool mStart; }; @@ -158,6 +155,7 @@ void KEncodingProber::reset() { d->proberState = KEncodingProber::Probing; d->mStart = true; + d->encoding = NULL; } KEncodingProber::ProberState KEncodingProber::feed(const QByteArray &data) @@ -203,11 +201,15 @@ KEncodingProber::ProberState KEncodingProber::state() const //DEPRECATED, do *not* use const char* KEncodingProber::encodingName() const { - return 0; + return strdup(encoding().constData()); } -QByteArray KEncodingProber::encodingNameByteArray() const +QByteArray KEncodingProber::encoding() const { + // detected by unicodeTest + if (d->encoding) + return QByteArray(d->encoding); + if (!d->prober) return QByteArray("UTF-8"); @@ -216,6 +218,10 @@ QByteArray KEncodingProber::encodingNameByteArray() const float KEncodingProber::confidence() const { + // detected by unicodeTest + if (d->encoding) + return 0.99f; + if (!d->prober) return 0.0; diff --git a/kdecore/localization/kencodingprober.h b/kdecore/localization/kencodingprober.h index 2ccceb5..d36ca2f 100644 --- a/kdecore/localization/kencodingprober.h +++ b/kdecore/localization/kencodingprober.h @@ -54,7 +54,7 @@ class KEncodingProberPrivate; * prober.feed(data); * prober.feed(moredata); * if (prober.confidence() > 0.6) - * QString out = QTextCodec::codeForName(prober.encodingName())->toUnicode(data); + * QString out = QTextCodec::codecForName(prober.encoding())->toUnicode(data); * \endcode * * at least 256 characters are needed to change the ProberState from Probing to FoundIt. @@ -132,7 +132,7 @@ public: /** * @returns a QByteArray with the name of the encoding */ - QByteArray encodingNameByteArray() const; + QByteArray encoding() const; /** * @returns the confidence(sureness) of encoding it guessed so far (0.0 ~ 0.99), not very reliable for single byte encodings diff --git a/kdecore/tests/CMakeLists.txt b/kdecore/tests/CMakeLists.txt index e191381..b7b4c81 100644 --- a/kdecore/tests/CMakeLists.txt +++ b/kdecore/tests/CMakeLists.txt @@ -57,6 +57,7 @@ KDECORE_UNIT_TESTS( kconfigafterkglobaltest2 ktcpsockettest ksycocathreadtest + kencodingprobertest ) if(UNIX) diff --git a/kdecore/tests/kencodingprobertest.cpp b/kdecore/tests/kencodingprobertest.cpp new file mode 100644 index 0000000..09861fd --- /dev/null +++ b/kdecore/tests/kencodingprobertest.cpp @@ -0,0 +1,126 @@ +/* This file is part of the KDE libraries + Copyright (c) 2009 Peter Oberndorfer + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Library General Public + License version 2 as published by the Free Software Foundation. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Library General Public License for more details. + + You should have received a copy of the GNU Library General Public License + along with this library; see the file COPYING.LIB. If not, write to + the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + Boston, MA 02110-1301, USA. +*/ + +#include "kencodingprobertest.h" +#include "qtest_kde.h" + +#include +#include +#include "kencodingprober.h" + +#include "kencodingprobertest.moc" + +void KEncodingProberTest::testDefaults() +{ + KEncodingProber prober; + + QCOMPARE(prober.state(), KEncodingProber::Probing); + QCOMPARE(prober.confidence(), 0.2f); + QCOMPARE(prober.proberType(), KEncodingProber::Universal); + QCOMPARE(prober.encoding(), QByteArray("UTF-8")); + QCOMPARE(prober.encodingName(), "UTF-8"); +} + +void KEncodingProberTest::testAscii() +{ + KEncodingProber prober; + const char text[] = "A easy test string without any special characters."; + + prober.feed(text, sizeof(text)); + // we have no definitive anser here, since there could be special characters in the string later + QCOMPARE(prober.state(), KEncodingProber::Probing); + QVERIFY(prober.confidence() > 0.9f); + QCOMPARE(prober.encoding(), QByteArray("UTF-8")); +} + +void KEncodingProberTest::testUtf8() +{ + KEncodingProber prober; + // created by notepad + const char text[] = "\xef\xbb\xbf" "A easy test string without any special characters, but with a UTF-8 BOM."; + + prober.feed(text, sizeof(text)); + QCOMPARE(prober.state(), KEncodingProber::FoundIt); + QVERIFY(prober.confidence() > 0.9f); + QCOMPARE(prober.encoding(), QByteArray("UTF-8")); +} + +void KEncodingProberTest::testUtf16LE() +{ + KEncodingProber prober; + const char text[] = "\xff\xfe" "A\0 \0t\0e\0s\0t\0 \0s\0t\0r\0i\0n\0g\0 \0w\0i\0t\0h\0o\0u\0t\na\0n\0y\0" + "s\0p\0e\0c\0i\0a\0l\0 \0c\0h\0a\0r\0a\0c\0t\0e\0r\0s\0,\0 \0b\0u\0t\0 \0a\0 \0U\0T\0F\0 \0B\0O\0M\0.\0"; + + prober.feed(text, sizeof(text)); + QCOMPARE(prober.state(), KEncodingProber::FoundIt); + QVERIFY(prober.confidence() > 0.9f); + QCOMPARE(prober.encoding(), QByteArray("UTF-16LE")); + + //check settings after reset + prober.reset(); + QCOMPARE(prober.state(), KEncodingProber::Probing); + QCOMPARE(prober.confidence(), 0.2f); + QCOMPARE(prober.encoding(), QByteArray("UTF-8")); +} + +void KEncodingProberTest::testUtf16BE() +{ + KEncodingProber prober; + const char text[] = "\xfe\xff" "\0A\0 \0t\0e\0s\0t\0 \0s\0t\0r\0i\0n\0g\0 \0w\0i\0t\0h\0o\0u\0t\na\0n\0y" + "\0s\0p\0e\0c\0i\0a\0l\0 \0c\0h\0a\0r\0a\0c\0t\0e\0r\0s\0,\0 \0b\0u\0t\0 \0a\0 \0U\0T\0F\0 \0B\0O\0M\0."; + + prober.feed(text, sizeof(text)); + QCOMPARE(prober.state(), KEncodingProber::FoundIt); + QVERIFY(prober.confidence() > 0.9f); + QCOMPARE(prober.encoding(), QByteArray("UTF-16BE")); +} + +void KEncodingProberTest::testQTextCodec() +{ + // make sure codecs returned by KEncodingProber are recognized by Qt + QTextCodec *utf8 = QTextCodec::codecForName("UTF-8"); + QVERIFY(utf8); + + QTextCodec *utf16 = QTextCodec::codecForName("UTF-16"); + QVERIFY(utf16); + + QTextCodec *utf16LE = QTextCodec::codecForName("UTF-16LE"); + QVERIFY(utf16LE); + + QTextCodec *utf16BE = QTextCodec::codecForName("UTF-16BE"); + QVERIFY(utf16BE); + + QTextCodec *utf32 = QTextCodec::codecForName("UTF-32"); + QVERIFY(utf32); + + QTextCodec *utf32LE = QTextCodec::codecForName("UTF-32LE"); + QVERIFY(utf32LE); + + QTextCodec *utf32BE = QTextCodec::codecForName("UTF-32BE"); + QVERIFY(utf32BE); + + QEXPECT_FAIL("", "KEncodingProber can return ISO-10646-UCS-4 which is not supported by QTextCodec::codecForName", Continue); + QTextCodec *iso10646ucs4 = QTextCodec::codecForName("ISO-10646-UCS-4"); + QVERIFY(iso10646ucs4); + + // test a invalid codec name + QTextCodec *invalidcodec = QTextCodec::codecForName("invalidcodec"); + QVERIFY(!invalidcodec); +} + +QTEST_KDEMAIN_CORE(KEncodingProberTest) diff --git a/kdecore/tests/kencodingprobertest.h b/kdecore/tests/kencodingprobertest.h new file mode 100644 index 0000000..21e3354 --- /dev/null +++ b/kdecore/tests/kencodingprobertest.h @@ -0,0 +1,36 @@ +/* This file is part of the KDE libraries + Copyright (c) 2009 Peter Oberndorfer + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Library General Public + License version 2 as published by the Free Software Foundation. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Library General Public License for more details. + + You should have received a copy of the GNU Library General Public License + along with this library; see the file COPYING.LIB. If not, write to + the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + Boston, MA 02110-1301, USA. +*/ + +#ifndef KENCODINGPROBERTEST_H +#define KENCODINGPROBERTEST_H + +#include + +class KEncodingProberTest : public QObject +{ + Q_OBJECT +private Q_SLOTS: + void testDefaults(); + void testAscii(); + void testUtf8(); + void testUtf16LE(); + void testUtf16BE(); + void testQTextCodec(); +}; + +#endif // KENCODINGPROBERTEST_H