[frameworks/kcodecs] src: [KEncodingProber] Refactor Unicode/UTF prober
Stefan Brüns <[email protected]>
| Newsgroups | gmane.comp.kde.cvs |
|---|---|
| Message-ID | <[email protected]> |
Git commit 38f9c8c267b345097e27665643425aaa1300c28d by Stefan Brüns.
Committed on 26/07/2026 at 17:00.
Pushed by bruns into branch 'master'.
[KEncodingProber] Refactor Unicode/UTF prober
The MBCS probers and SBCS probers are implemented as nsCharSetProber,
only the UTF-8 and UTF-16 probers directly wrap the nsCodingStateMachine
inside the UnicodeGroupProber.
This allows to remove the UnicodeGroupProber indirection when probing
for Unicode encodings from the SBCS and MBCS probers.
Also, the templated code removes the repeated loads of the model
variables from the inner loop.
M +2 -0 src/CMakeLists.txt
M +6 -2 src/kencodingprober.cpp
M +21 -13 src/probers/nsMBCSGroupProber.cpp
M +4 -2 src/probers/nsMBCSGroupProber.h
M +17 -13 src/probers/nsSBCSGroupProber.cpp
M +1 -1 src/probers/nsSBCSGroupProber.h
A +85 -0 src/probers/nsUtfProber.cpp [License: MIT]
A +50 -0 src/probers/nsUtfProber.h [License: MIT]
https://invent.kde.org/frameworks/kcodecs/-/commit/38f9c8c267b345097e27665643425aaa1300c28d
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index c2f8ad6..0c04a4a 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -65,6 +65,8 @@ target_sources(KF6Codecs PRIVATE
probers/nsSJISProber.h
probers/nsUniversalDetector.cpp
probers/nsUniversalDetector.h
+ probers/nsUtfProber.cpp
+ probers/nsUtfProber.h
probers/UnicodeGroupProber.cpp
probers/UnicodeGroupProber.h
)
diff --git a/src/kencodingprober.cpp b/src/kencodingprober.cpp
index bcaf32d..53cec6c 100644
--- a/src/kencodingprober.cpp
+++ b/src/kencodingprober.cpp
@@ -21,14 +21,18 @@ namespace
{
using Prober = kencodingprober::nsMBCSGroupProber::Prober;
static const std::array ChineseMSBCProbers{
- Prober::Unicode,
+ Prober::Utf8,
Prober::GB18030,
Prober::Big5,
+ Prober::Utf16LE,
+ Prober::Utf16BE,
};
static const std::array JapaneseMSBCProbers{
- Prober::Unicode,
+ Prober::Utf8,
Prober::SJIS,
Prober::EUCJP,
+ Prober::Utf16LE,
+ Prober::Utf16BE,
};
constexpr const char *checkBom(std::span<const char, 4> buf)
{
diff --git a/src/probers/nsMBCSGroupProber.cpp b/src/probers/nsMBCSGroupProber.cpp
index 7ae7b31..1bb736d 100644
--- a/src/probers/nsMBCSGroupProber.cpp
+++ b/src/probers/nsMBCSGroupProber.cpp
@@ -6,23 +6,24 @@
#include "nsMBCSGroupProber.h"
-#include "UnicodeGroupProber.h"
#include "nsBig5Prober.h"
#include "nsEUCJPProber.h"
#include "nsEUCKRProber.h"
#include "nsGB2312Prober.h"
#include "nsSJISProber.h"
+#include "nsUtfProber.h"
#include <format>
namespace kencodingprober
{
+
namespace
{
using Prober = nsMBCSGroupProber::Prober;
-constexpr std::array<bool, 6> fromSelectedList(std::span<const Prober> selected)
+constexpr std::array<bool, NUM_OF_PROBERS> fromSelectedList(std::span<const Prober> selected)
{
- std::array<bool, 6> isSelected{false};
+ std::array<bool, NUM_OF_PROBERS> isSelected{false};
for (auto p : selected) {
const auto i = static_cast<std::underlying_type_t<Prober>>(p);
if (i >= NUM_OF_PROBERS) {
@@ -34,8 +35,10 @@ constexpr std::array<bool, 6> fromSelectedList(std::span<const Prober> selected)
}
static_assert(fromSelectedList({})[0] == false);
static_assert(fromSelectedList({})[5] == false);
-static_assert(fromSelectedList(std::array{Prober::Unicode})[0] == true);
-static_assert(fromSelectedList(std::array{Prober::Unicode})[5] == false);
+static_assert(fromSelectedList(std::array{Prober::Utf8})[0] == true);
+static_assert(fromSelectedList(std::array{Prober::Utf8})[5] == false);
+static_assert(fromSelectedList(std::array{Prober::Utf16LE})[6] == true);
+static_assert(fromSelectedList(std::array{Prober::Utf16BE})[7] == true);
static_assert(fromSelectedList(std::array{Prober::SJIS, Prober::Big5})[0] == false);
static_assert(fromSelectedList(std::array{Prober::SJIS, Prober::Big5})[1] == true);
static_assert(fromSelectedList(std::array{Prober::SJIS, Prober::Big5})[2] == false);
@@ -46,13 +49,16 @@ static_assert(fromSelectedList(std::array{Prober::SJIS, Prober::Big5})[5] == tru
} // namespace <anonymous>
nsMBCSGroupProber::nsMBCSGroupProber(std::span<const Prober> selected)
- : mProbers{std::make_unique<UnicodeGroupProber>(),
- std::make_unique<nsSJISProber>(),
- std::make_unique<nsEUCJPProber>(),
- std::make_unique<nsGB18030Prober>(),
- std::make_unique<nsEUCKRProber>(),
- std::make_unique<nsBig5Prober>(),
- }
+ : mProbers{
+ std::make_unique<nsUtf8Prober>(),
+ std::make_unique<nsSJISProber>(),
+ std::make_unique<nsEUCJPProber>(),
+ std::make_unique<nsGB18030Prober>(),
+ std::make_unique<nsEUCKRProber>(),
+ std::make_unique<nsBig5Prober>(),
+ std::make_unique<nsUtf16LEProber>(),
+ std::make_unique<nsUtf16BEProber>(),
+ }
, mIsSelected(fromSelectedList(selected))
{
for (unsigned int i = 0; i < NUM_OF_PROBERS; i++) {
@@ -65,12 +71,14 @@ nsMBCSGroupProber::nsMBCSGroupProber(std::span<const Prober> selected)
nsMBCSGroupProber::nsMBCSGroupProber()
: nsMBCSGroupProber(std::array{
- Prober::Unicode,
+ Prober::Utf8,
Prober::SJIS,
Prober::EUCJP,
Prober::GB18030,
Prober::EUCKR,
Prober::Big5,
+ Prober::Utf16LE,
+ Prober::Utf16BE,
})
{
}
diff --git a/src/probers/nsMBCSGroupProber.h b/src/probers/nsMBCSGroupProber.h
index 34f72ad..6cc7dca 100644
--- a/src/probers/nsMBCSGroupProber.h
+++ b/src/probers/nsMBCSGroupProber.h
@@ -14,19 +14,21 @@
#include <memory>
#include <span>
-#define NUM_OF_PROBERS 6
+#define NUM_OF_PROBERS 8
namespace kencodingprober
{
class KCODECS_NO_EXPORT nsMBCSGroupProber : public nsCharSetProber
{
public:
enum class Prober : uint8_t {
- Unicode = 0,
+ Utf8 = 0,
SJIS = 1,
EUCJP = 2,
GB18030 = 3,
EUCKR = 4,
Big5 = 5,
+ Utf16LE = 6,
+ Utf16BE = 7,
};
nsMBCSGroupProber();
diff --git a/src/probers/nsSBCSGroupProber.cpp b/src/probers/nsSBCSGroupProber.cpp
index f4019b8..1cdddcd 100644
--- a/src/probers/nsSBCSGroupProber.cpp
+++ b/src/probers/nsSBCSGroupProber.cpp
@@ -6,9 +6,9 @@
#include "nsSBCSGroupProber.h"
-#include "UnicodeGroupProber.h"
#include "nsHebrewProber.h"
#include "nsSBCharSetProber.h"
+#include "nsUtfProber.h"
#include <format>
@@ -27,13 +27,15 @@ nsSBCSGroupProber::nsSBCSGroupProber()
std::make_unique<nsSingleByteCharSetProber<false>>(&Latin5BulgarianModel),
std::make_unique<nsSingleByteCharSetProber<false>>(&Win1251BulgarianModel),
std::make_unique<nsHebrewProber>(),
- std::make_unique<UnicodeGroupProber>(),
+ std::make_unique<nsUtf8Prober>(),
+ std::make_unique<nsUtf16BEProber>(),
+ std::make_unique<nsUtf16LEProber>(),
}
{
// disable latin2 before latin1/windows-1252 is available, otherwise all latin1
// will be detected as latin2 because of their similarity.
- // mProbers[12] = std::make_unique<nsSingleByteCharSetProber<false>>(&Latin2HungarianModel);
- // mProbers[13] = std::make_unique<nsSingleByteCharSetProber<false>>(&Win1250HungarianModel);
+ // mProbers[14] = std::make_unique<nsSingleByteCharSetProber<false>>(&Latin2HungarianModel);
+ // mProbers[15] = std::make_unique<nsSingleByteCharSetProber<false>>(&Win1250HungarianModel);
for (unsigned int i = 0; i < NUM_OF_SBCS_PROBERS; i++) {
if (mProbers[i]) { // not null
@@ -62,20 +64,22 @@ nsProbingState nsSBCSGroupProber::HandleData(const char *aBuf, unsigned int aLen
char *newBuf1 = nullptr;
unsigned int newLen1 = 0;
- int activeNum = NUM_OF_SBCS_PROBERS - 1;
+ int activeNum = NUM_OF_SBCS_PROBERS;
- // The UnicodeGroupProber (specifically the UTF16 subprobers) need unmangled data
- if (mIsActive[NUM_OF_SBCS_PROBERS - 1]) {
- if (const auto st = mProbers[NUM_OF_SBCS_PROBERS - 1]->HandleData(aBuf, aLen); st == eFoundIt) {
- mBestGuess = NUM_OF_SBCS_PROBERS - 1;
+ // The UTF16 probers need unmangled data
+ for (unsigned int i = NUM_OF_SBCS_PROBERS - 2; i < NUM_OF_SBCS_PROBERS; ++i) {
+ if (!mIsActive[i]) {
+ activeNum--;
+ continue;
+ }
+ if (const auto st = mProbers[i]->HandleData(aBuf, aLen); st == eFoundIt) {
+ mBestGuess = i;
mState = eFoundIt;
return mState;
} else if (st == eNotMe) {
- mIsActive[NUM_OF_SBCS_PROBERS - 1] = false;
+ mIsActive[i] = false;
activeNum--;
}
- } else {
- activeNum--;
}
// apply filter to original buffer, and we got new buffer back
@@ -92,7 +96,7 @@ nsProbingState nsSBCSGroupProber::HandleData(const char *aBuf, unsigned int aLen
goto done; // Nothing to see here, move on.
}
- for (unsigned int i = 0; i < NUM_OF_SBCS_PROBERS - 1; ++i) {
+ for (unsigned int i = 0; i < NUM_OF_SBCS_PROBERS - 2; ++i) {
if (!mIsActive[i]) {
activeNum--;
continue;
diff --git a/src/probers/nsSBCSGroupProber.h b/src/probers/nsSBCSGroupProber.h
index fafeafa..fec55f3 100644
--- a/src/probers/nsSBCSGroupProber.h
+++ b/src/probers/nsSBCSGroupProber.h
@@ -15,7 +15,7 @@
#include <array>
#include <memory>
-#define NUM_OF_SBCS_PROBERS 12
+#define NUM_OF_SBCS_PROBERS 14
namespace kencodingprober
{
diff --git a/src/probers/nsUtfProber.cpp b/src/probers/nsUtfProber.cpp
new file mode 100644
index 0000000..2d0f7ca
--- /dev/null
+++ b/src/probers/nsUtfProber.cpp
@@ -0,0 +1,85 @@
+/* -*- C++ -*-
+ SPDX-FileCopyrightText: 2026 Stefan Brüns <[email protected]>
+
+ SPDX-License-Identifier: MIT
+*/
+
+#include "nsUtfProber.h"
+#include "nsMBCSSM.h"
+
+namespace kencodingprober
+{
+constexpr auto modelForProber(UtfProberType type)
+{
+ if (type == Utf8) {
+ return kencodingprober::UTF8SMModel;
+ } else if (type == Utf16LE) {
+ return kencodingprober::UCS2LESMModel;
+ } else {
+ return kencodingprober::UCS2BESMModel;
+ }
+}
+
+template<UtfProberType TYPE>
+struct UtfSM {
+ static nsSMState advance(nsSMState oldState, const char c)
+ {
+ const uint8_t index = static_cast<uint8_t>(c);
+ const unsigned int rowIndex = oldState * sModel.classFactor;
+ const unsigned int byteCls = sModel.classTable[index];
+ return sModel.stateTable[rowIndex + byteCls];
+ }
+ static constexpr auto sModel = modelForProber(TYPE);
+};
+
+template<UtfProberType TYPE>
+nsProbingState nsUtfProber<TYPE>::HandleData(const char *aBuf, unsigned int aLen)
+{
+ if (mState == eNotMe) {
+ return mState;
+ }
+
+ for (unsigned int i = 0; i < aLen; ++i) {
+ // byte is feed to all active state machine
+ mCodingState = UtfSM<TYPE>::advance(mCodingState, aBuf[i]);
+ if (mCodingState == eError) {
+ // got negative answer for this state machine, make it inactive
+ mState = eNotMe;
+ return mState;
+ } else if (mCodingState == eItsMe) {
+ mState = eFoundIt;
+ return mState;
+ }
+ }
+ return mState;
+};
+
+template<UtfProberType TYPE>
+float nsUtfProber<TYPE>::GetConfidence()
+{
+ return mState == eNotMe ? 0.0f : mState == eFoundIt ? 0.8f : 0.1f;
+}
+
+template class nsUtfProber<UtfProberType::Utf8>;
+template class nsUtfProber<UtfProberType::Utf16LE>;
+template class nsUtfProber<UtfProberType::Utf16BE>;
+
+template<>
+const char *nsUtf8Prober::GetCharSetName()
+{
+ return "UTF-8";
+}
+
+template<>
+const char *nsUtf16LEProber::GetCharSetName()
+{
+ return "UTF-16LE";
+}
+
+template<>
+const char *nsUtf16BEProber::GetCharSetName()
+{
+ return "UTF-16BE";
+}
+
+} // namespace kencodingprober
diff --git a/src/probers/nsUtfProber.h b/src/probers/nsUtfProber.h
new file mode 100644
index 0000000..379ca72
--- /dev/null
+++ b/src/probers/nsUtfProber.h
@@ -0,0 +1,50 @@
+/* -*- C++ -*-
+ SPDX-FileCopyrightText: 2026 Stefan Brüns <[email protected]>
+
+ SPDX-License-Identifier: MIT
+*/
+
+#ifndef NSUTFPROBER_H
+#define NSUTFPROBER_H
+
+#include "nsCharSetProber.h"
+#include "nsCodingStateMachine.h"
+
+enum UtfProberType : uint8_t {
+ Utf8 = 0,
+ Utf16LE = 1,
+ Utf16BE = 2,
+};
+
+namespace kencodingprober
+{
+template<UtfProberType MODEL>
+class KCODECS_NO_EXPORT nsUtfProber : public nsCharSetProber
+{
+public:
+ nsUtfProber() = default;
+ ~nsUtfProber() override = default;
+
+ nsProbingState HandleData(const char *aBuf, unsigned int aLen) override;
+
+ const char *GetCharSetName() override;
+
+ nsProbingState GetState(void) override
+ {
+ return mState;
+ }
+
+ float GetConfidence() override;
+
+protected:
+ nsProbingState mState = eDetecting;
+ nsSMState mCodingState = 0;
+};
+
+using nsUtf8Prober = nsUtfProber<UtfProberType::Utf8>;
+using nsUtf16LEProber = nsUtfProber<UtfProberType::Utf16LE>;
+using nsUtf16BEProber = nsUtfProber<UtfProberType::Utf16BE>;
+
+} // namespace kencodingprober
+
+#endif // NSUTFPROBER_H