You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 
 
 
 
 

240 lines
8.3 KiB

///////////////////////////////////////////////////////////////////////////////////
// Copyright (C) 2026 Jon Beniston, M7RCE <jon@beniston.com> //
// Some code by AI //
// //
// This program is free software; you can redistribute it and/or modify //
// it under the terms of the GNU General Public License as published by //
// the Free Software Foundation as version 3 of the License, or //
// (at your option) any later version. //
// //
// This program is distributed in the hope that it will be useful, //
// but WITHOUT ANY WARRANTY; without even the implied warranty of //
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the //
// GNU General Public License V3 for more details. //
// //
// You should have received a copy of the GNU General Public License //
// along with this program. If not, see <http://www.gnu.org/licenses/>. //
///////////////////////////////////////////////////////////////////////////////////
#include "text.h"
namespace modemdr
{
namespace
{
// Complete EBU Latin based repertoire (TS 101 756 annex C) to Unicode. Codes 0x00, 0x0A,
// 0x0B and 0x1F are reserved (control codes in dynamic labels).
const std::uint16_t ebuLatin[256] = {
0x0000, 0x0118, 0x012E, 0x0172, 0x0102, 0x0116, 0x010E, 0x0218,
0x021A, 0x010A, 0x0000, 0x0000, 0x0120, 0x0139, 0x017B, 0x0143,
0x0105, 0x0119, 0x012F, 0x0173, 0x0103, 0x0117, 0x010F, 0x0219,
0x021B, 0x010B, 0x0147, 0x011A, 0x0121, 0x013A, 0x017C, 0x0000,
0x0020, 0x0021, 0x0022, 0x0023, 0x0142, 0x0025, 0x0026, 0x0027,
0x0028, 0x0029, 0x002A, 0x002B, 0x002C, 0x002D, 0x002E, 0x002F,
0x0030, 0x0031, 0x0032, 0x0033, 0x0034, 0x0035, 0x0036, 0x0037,
0x0038, 0x0039, 0x003A, 0x003B, 0x003C, 0x003D, 0x003E, 0x003F,
0x0040, 0x0041, 0x0042, 0x0043, 0x0044, 0x0045, 0x0046, 0x0047,
0x0048, 0x0049, 0x004A, 0x004B, 0x004C, 0x004D, 0x004E, 0x004F,
0x0050, 0x0051, 0x0052, 0x0053, 0x0054, 0x0055, 0x0056, 0x0057,
0x0058, 0x0059, 0x005A, 0x005B, 0x016E, 0x005D, 0x0141, 0x005F,
0x0104, 0x0061, 0x0062, 0x0063, 0x0064, 0x0065, 0x0066, 0x0067,
0x0068, 0x0069, 0x006A, 0x006B, 0x006C, 0x006D, 0x006E, 0x006F,
0x0070, 0x0071, 0x0072, 0x0073, 0x0074, 0x0075, 0x0076, 0x0077,
0x0078, 0x0079, 0x007A, 0x00AB, 0x016F, 0x00BB, 0x013D, 0x0126,
0x00E1, 0x00E0, 0x00E9, 0x00E8, 0x00ED, 0x00EC, 0x00F3, 0x00F2,
0x00FA, 0x00F9, 0x00D1, 0x00C7, 0x015E, 0x00DF, 0x00A1, 0x0178,
0x00E2, 0x00E4, 0x00EA, 0x00EB, 0x00EE, 0x00EF, 0x00F4, 0x00F6,
0x00FB, 0x00FC, 0x00F1, 0x00E7, 0x015F, 0x011F, 0x0131, 0x00FF,
0x0136, 0x0145, 0x00A9, 0x0122, 0x011E, 0x011B, 0x0148, 0x0151,
0x0150, 0x20AC, 0x00A3, 0x0024, 0x0100, 0x0112, 0x012A, 0x016A,
0x0137, 0x0146, 0x013B, 0x0123, 0x013C, 0x0130, 0x0144, 0x0171,
0x0170, 0x00BF, 0x013E, 0x00B0, 0x0101, 0x0113, 0x012B, 0x016B,
0x00C1, 0x00C0, 0x00C9, 0x00C8, 0x00CD, 0x00CC, 0x00D3, 0x00D2,
0x00DA, 0x00D9, 0x0158, 0x010C, 0x0160, 0x017D, 0x00D0, 0x013F,
0x00C2, 0x00C4, 0x00CA, 0x00CB, 0x00CE, 0x00CF, 0x00D4, 0x00D6,
0x00DB, 0x00DC, 0x0159, 0x010D, 0x0161, 0x017E, 0x0111, 0x0140,
0x00C3, 0x00C5, 0x00C6, 0x0152, 0x0177, 0x00DD, 0x00D5, 0x00D8,
0x00DE, 0x014A, 0x0154, 0x0106, 0x015A, 0x0179, 0x0164, 0x00F0,
0x00E3, 0x00E5, 0x00E6, 0x0153, 0x0175, 0x00FD, 0x00F5, 0x00F8,
0x00FE, 0x014B, 0x0155, 0x0107, 0x015B, 0x017A, 0x0165, 0x0127
};
void appendUTF8(std::string& out, uint32_t codePoint)
{
if (codePoint < 0x80)
{
out.push_back(static_cast<char>(codePoint));
}
else if (codePoint < 0x800)
{
out.push_back(static_cast<char>(0xc0 | (codePoint >> 6)));
out.push_back(static_cast<char>(0x80 | (codePoint & 0x3f)));
}
else if (codePoint < 0x10000)
{
out.push_back(static_cast<char>(0xe0 | (codePoint >> 12)));
out.push_back(static_cast<char>(0x80 | ((codePoint >> 6) & 0x3f)));
out.push_back(static_cast<char>(0x80 | (codePoint & 0x3f)));
}
else if (codePoint < 0x110000)
{
out.push_back(static_cast<char>(0xf0 | (codePoint >> 18)));
out.push_back(static_cast<char>(0x80 | ((codePoint >> 12) & 0x3f)));
out.push_back(static_cast<char>(0x80 | ((codePoint >> 6) & 0x3f)));
out.push_back(static_cast<char>(0x80 | (codePoint & 0x3f)));
}
}
// Control codes shared by every character set in labels: preferred line break (0x0a),
// end of headline (0x0b) and preferred word break (0x1f).
bool appendControl(std::string& out, uint32_t code)
{
if ((code == 0x0a) || (code == 0x0b))
{
out.push_back('\n');
return true;
}
if (code == 0x1f) {
return true;
}
if (code < 0x20) {
return true; // Other C0 controls are never displayed
}
return false;
}
// Keep only well formed UTF-8 (broadcast text is untrusted).
std::string sanitizeUTF8(const uint8_t *data, std::size_t length)
{
std::string out;
std::size_t i = 0;
while (i < length)
{
uint8_t c = data[i];
int extra;
uint32_t cp;
if (c < 0x80) {
extra = 0;
cp = c;
} else if ((c & 0xe0) == 0xc0) {
extra = 1;
cp = c & 0x1f;
} else if ((c & 0xf0) == 0xe0) {
extra = 2;
cp = c & 0x0f;
} else if ((c & 0xf8) == 0xf0) {
extra = 3;
cp = c & 0x07;
} else {
i++;
continue;
}
if (i + extra >= length) {
break; // Truncated sequence
}
bool ok = true;
for (int j = 1; ok && j <= extra; j++)
{
uint8_t cc = data[i + j];
if ((cc & 0xc0) != 0x80) {
ok = false;
} else {
cp = (cp << 6) | (cc & 0x3f);
}
}
if (!ok)
{
i++;
continue;
}
i += extra + 1;
if (!appendControl(out, cp)) {
appendUTF8(out, cp);
}
}
return out;
}
} // anonymous namespace
std::string Text::toUTF8(const std::uint8_t *data, std::size_t length, int charset)
{
// Labels are padded with zeros
while ((length > 0) && (data[length - 1] == 0)) {
length--;
}
std::string out;
switch (charset)
{
case 0: // Complete EBU Latin based repertoire
for (std::size_t i = 0; i < length; i++)
{
if (!appendControl(out, data[i]))
{
uint16_t cp = ebuLatin[data[i]];
if (cp) {
appendUTF8(out, cp);
}
}
}
break;
case 4: // ISO 8859-1 (MOT ContentName)
for (std::size_t i = 0; i < length; i++)
{
if (!appendControl(out, data[i])) {
appendUTF8(out, data[i]);
}
}
break;
case 6: // UCS-2 / UTF-16 BE, basic multilingual plane
for (std::size_t i = 0; i + 1 < length; i += 2)
{
uint32_t cp = (static_cast<uint32_t>(data[i]) << 8) | data[i + 1];
if ((cp >= 0xd800) && (cp <= 0xdfff)) {
continue;
}
if (!appendControl(out, cp)) {
appendUTF8(out, cp);
}
}
break;
case 15: // UTF-8
out = sanitizeUTF8(data, length);
break;
default:
break;
}
return out;
}
std::string Text::contentNameToUTF8(const std::uint8_t *data, std::size_t length, int charset)
{
if (charset != 0) {
return toUTF8(data, length, charset);
}
std::string out;
for (std::size_t i = 0; i < length; i++)
{
if (data[i] < 0x80)
{
if (data[i] >= 0x20) {
out.push_back(static_cast<char>(data[i]));
}
}
else if (ebuLatin[data[i]])
{
appendUTF8(out, ebuLatin[data[i]]);
}
}
return out;
}
} // namespace modemdr