Mirrors/fish-shell
2
0
Fork 0
mirror of https://github.com/fish-shell/fish-shell synced 2024-11-10 07:04:29 +00:00
Code Issues Projects Releases Packages Wiki Activity

Remove C++ UTF-8 bits

Browse source 
These are no longer used.
This commit is contained in:
ridiculousfish 2023-11-18 19:21:25 -08:00
parent a718852ad4
commit c1e4a447fd
6 changed files with 0 additions and 617 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
CMakeLists.txt
View file

@ -128,7 +128,6 @@ set(FISH_SRCS
src/reader.cpp
src/rustffi.cpp
src/screen.cpp
src/utf8.cpp
src/wcstringutil.cpp
src/wgetopt.cpp
src/wutil.cpp

12
doc_src/license.rst
View file

@ -231,18 +231,6 @@ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH RE
----
**License for UTF8**
``fish`` also contains small amounts of code under the ISC license, namely the UTF-8 conversion functions. This code is copyright © 2007 Alexey Vatchenko \<av@bsdua.org>.
The ISC license agreement follows.
Permission to use, copy, modify, and/or distribute this software for any purpose with or without fee is hereby granted, provided that the above copyright notice and this permission notice appear in all copies.
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
----
**License for flock**
``fish`` also contains small amounts of code from NetBSD, namely the ``flock`` fallback function. This code is copyright 2001 The NetBSD Foundation, Inc., and derived from software contributed to The NetBSD Foundation by Todd Vierling.

1
src/env_universal_common.cpp
View file

@ -40,7 +40,6 @@
#include "fd_readable_set.rs.h"
#include "flog.h"
#include "path.h"
#include "utf8.h"
#include "util.h" // IWYU pragma: keep
#include "wcstringutil.h"
#include "wutil.h"

210
src/fish_tests.cpp
View file

@ -96,7 +96,6 @@
#include "termsize.h"
#include "threads.rs.h"
#include "tokenizer.h"
#include "utf8.h"
#include "util.h"
#include "wcstringutil.h"
#include "wgetopt.h"
@ -994,214 +993,6 @@ static void test_utility_functions() {
test_const_strcmp();
}
// UTF8 tests taken from Alexey Vatchenko's utf8 library. See http://www.bsdua.org/libbsdua.html.
static void test_utf82wchar(const char *src, size_t slen, const wchar_t *dst, size_t dlen,
int flags, size_t res, const char *descr) {
size_t size;
wchar_t *mem = nullptr;
#if WCHAR_MAX == 0xffff
// Hack: if wchar is only UCS-2, and the UTF-8 input string contains astral characters, then
// tweak the expected size to 0.
if (src) {
// A UTF-8 code unit may represent an astral code point if it has 4 or more leading 1s.
const unsigned char astral_mask = 0xF0;
for (size_t i = 0; i < slen; i++) {
if ((src[i] & astral_mask) == astral_mask) {
// Astral char. We want this conversion to fail.
res = 0; //!OCLINT(parameter reassignment)
break;
}
}
}
#endif
if (!dst) {
size = utf8_to_wchar(src, slen, nullptr, flags);
} else {
mem = (wchar_t *)malloc(dlen * sizeof(*mem));
if (!mem) {
err(L"u2w: %s: MALLOC FAILED\n", descr);
return;
}
std::wstring buff;
size = utf8_to_wchar(src, slen, &buff, flags);
std::copy(buff.begin(), buff.begin() + std::min(dlen, buff.size()), mem);
}
if (res != size) {
err(L"u2w: %s: FAILED (rv: %lu, must be %lu)", descr, size, res);
} else if (mem && std::memcmp(mem, dst, size * sizeof(*mem)) != 0) {
err(L"u2w: %s: BROKEN", descr);
}
free(mem);
}
// Annoying variant to handle uchar to avoid narrowing conversion warnings.
static void test_utf82wchar(const unsigned char *usrc, size_t slen, const wchar_t *dst, size_t dlen,
int flags, size_t res, const char *descr) {
const char *src = reinterpret_cast<const char *>(usrc);
return test_utf82wchar(src, slen, dst, dlen, flags, res, descr);
}
static void test_wchar2utf8(const wchar_t *src, size_t slen, const char *dst, size_t dlen,
int flags, size_t res, const char *descr) {
size_t size;
char *mem = nullptr;
#if WCHAR_MAX == 0xffff
// Hack: if wchar is simulating UCS-2, and the wchar_t input string contains astral characters,
// then tweak the expected size to 0.
if (src) {
const uint32_t astral_mask = 0xFFFF0000U;
for (size_t i = 0; i < slen; i++) {
if ((src[i] & astral_mask) != 0) {
// Astral char. We want this conversion to fail.
res = 0; //!OCLINT(parameter reassignment)
break;
}
}
}
#endif
if (dst) {
// We want to pass a valid pointer to wchar_to_utf8, so allocate at least one byte.
mem = (char *)malloc(dlen + 1);
if (!mem) {
err(L"w2u: %s: MALLOC FAILED", descr);
return;
}
}
size = wchar_to_utf8(src, slen, mem, dlen, flags);
if (res != size) {
err(L"w2u: %s: FAILED (rv: %lu, must be %lu)", descr, size, res);
} else if (dst && std::memcmp(mem, dst, size) != 0) {
err(L"w2u: %s: BROKEN", descr);
}
free(mem);
}
// Annoying variant to handle uchar to avoid narrowing conversion warnings.
static void test_wchar2utf8(const wchar_t *src, size_t slen, const unsigned char *udst, size_t dlen,
int flags, size_t res, const char *descr) {
const char *dst = reinterpret_cast<const char *>(udst);
return test_wchar2utf8(src, slen, dst, dlen, flags, res, descr);
}
// todo!("delete this?");
static void test_utf8() {
say(L"Testing utf8");
wchar_t w1[] = {0x54, 0x65, 0x73, 0x74};
wchar_t w2[] = {0x0422, 0x0435, 0x0441, 0x0442};
wchar_t w3[] = {0x800, 0x1e80, 0x98c4, 0x9910, 0xff00};
wchar_t wm[] = {0x41, 0x0441, 0x3042, 0xff67, 0x9b0d};
wchar_t wb2[] = {0xd800, 0xda00, 0x41, 0xdfff, 0x0a};
wchar_t wbom[] = {0xfeff, 0x41, 0x0a};
wchar_t wbom2[] = {0x41, 0xa};
wchar_t wbom22[] = {0xfeff, 0x41, 0x0a};
unsigned char u1[] = {0x54, 0x65, 0x73, 0x74};
unsigned char u2[] = {0xd0, 0xa2, 0xd0, 0xb5, 0xd1, 0x81, 0xd1, 0x82};
unsigned char u3[] = {0xe0, 0xa0, 0x80, 0xe1, 0xba, 0x80, 0xe9, 0xa3,
0x84, 0xe9, 0xa4, 0x90, 0xef, 0xbc, 0x80};
unsigned char um[] = {0x41, 0xd1, 0x81, 0xe3, 0x81, 0x82, 0xef, 0xbd, 0xa7, 0xe9, 0xac, 0x8d};
unsigned char uc080[] = {0xc0, 0x80};
unsigned char ub2[] = {0xed, 0xa1, 0x8c, 0xed, 0xbe, 0xb4, 0x0a};
unsigned char ubom[] = {0x41, 0xa};
unsigned char ubom2[] = {0xef, 0xbb, 0xbf, 0x41, 0x0a};
#if WCHAR_MAX != 0xffff
wchar_t w4[] = {0x15555, 0xf7777, 0x0a};
wchar_t wb[] = {(wchar_t)-2, 0xa, (wchar_t)0xffffffff, 0x0441};
wchar_t wb1[] = {0x0a, 0x0422};
unsigned char u4[] = {0xf0, 0x95, 0x95, 0x95, 0xf3, 0xb7, 0x9d, 0xb7, 0x0a};
unsigned char ub[] = {0xa, 0xd1, 0x81};
unsigned char ub1[] = {0xa, 0xff, 0xd0, 0xa2, 0xfe, 0x8f, 0xe0, 0x80};
#endif
// UTF-8 -> UCS-4 string.
test_utf82wchar(ubom2, sizeof(ubom2), wbom2, sizeof(wbom2) / sizeof(*wbom2), UTF8_SKIP_BOM,
sizeof(wbom2) / sizeof(*wbom2), "ubom2 skip BOM");
test_utf82wchar(ubom2, sizeof(ubom2), wbom22, sizeof(wbom22) / sizeof(*wbom22), 0,
sizeof(wbom22) / sizeof(*wbom22), "ubom2 BOM");
test_utf82wchar(uc080, sizeof(uc080), nullptr, 0, 0, 0, "uc080 c0 80 - forbitten by rfc3629");
test_utf82wchar(ub2, sizeof(ub2), nullptr, 0, 0, 3, "ub2 resulted in forbitten wchars (len)");
test_utf82wchar(ub2, sizeof(ub2), wb2, sizeof(wb2) / sizeof(*wb2), 0, 0,
"ub2 resulted in forbitten wchars");
test_utf82wchar(ub2, sizeof(ub2), L"\x0a", 1, UTF8_IGNORE_ERROR, 1,
"ub2 resulted in ignored forbitten wchars");
test_utf82wchar(u1, sizeof(u1), w1, sizeof(w1) / sizeof(*w1), 0, sizeof(w1) / sizeof(*w1),
"u1/w1 1 octet chars");
test_utf82wchar(u2, sizeof(u2), w2, sizeof(w2) / sizeof(*w2), 0, sizeof(w2) / sizeof(*w2),
"u2/w2 2 octets chars");
test_utf82wchar(u3, sizeof(u3), w3, sizeof(w3) / sizeof(*w3), 0, sizeof(w3) / sizeof(*w3),
"u3/w3 3 octets chars");
test_utf82wchar("\xff", 1, nullptr, 0, 0, 0, "broken utf-8 0xff symbol");
test_utf82wchar("\xfe", 1, nullptr, 0, 0, 0, "broken utf-8 0xfe symbol");
test_utf82wchar("\x8f", 1, nullptr, 0, 0, 0, "broken utf-8, start from 10 higher bits");
test_utf82wchar((const char *)nullptr, 0, nullptr, 0, 0, 0, "invalid params, all 0");
test_utf82wchar(u1, 0, nullptr, 0, 0, 0, "invalid params, src buf not NULL");
test_utf82wchar((const char *)nullptr, 10, nullptr, 0, 0, 0,
"invalid params, src length is not 0");
// UCS-4 -> UTF-8 string.
const char *const nullc = nullptr;
test_wchar2utf8(wbom, sizeof(wbom) / sizeof(*wbom), ubom, sizeof(ubom), UTF8_SKIP_BOM,
sizeof(ubom), "BOM");
test_wchar2utf8(wb2, sizeof(wb2) / sizeof(*wb2), nullc, 0, 0, 0, "prohibited wchars");
test_wchar2utf8(wb2, sizeof(wb2) / sizeof(*wb2), nullc, 0, UTF8_IGNORE_ERROR, 2,
"ignore prohibited wchars");
test_wchar2utf8(w1, sizeof(w1) / sizeof(*w1), u1, sizeof(u1), 0, sizeof(u1),
"w1/u1 1 octet chars");
test_wchar2utf8(w2, sizeof(w2) / sizeof(*w2), u2, sizeof(u2), 0, sizeof(u2),
"w2/u2 2 octets chars");
test_wchar2utf8(w3, sizeof(w3) / sizeof(*w3), u3, sizeof(u3), 0, sizeof(u3),
"w3/u3 3 octets chars");
test_wchar2utf8(nullptr, 0, nullc, 0, 0, 0, "invalid params, all 0");
test_wchar2utf8(w1, 0, nullc, 0, 0, 0, "invalid params, src buf not NULL");
test_wchar2utf8(w1, sizeof(w1) / sizeof(*w1), u1, 0, 0, 0, "invalid params, dst is not NULL");
test_wchar2utf8(nullptr, 10, nullc, 0, 0, 0, "invalid params, src length is not 0");
test_wchar2utf8(wm, sizeof(wm) / sizeof(*wm), um, sizeof(um), 0, sizeof(um),
"wm/um mixed languages");
test_wchar2utf8(wm, sizeof(wm) / sizeof(*wm), um, sizeof(um) - 1, 0, 0, "wm/um boundaries -1");
test_wchar2utf8(wm, sizeof(wm) / sizeof(*wm), um, sizeof(um) + 1, 0, sizeof(um),
"wm/um boundaries +1");
test_wchar2utf8(wm, sizeof(wm) / sizeof(*wm), nullc, 0, 0, sizeof(um),
"wm/um calculate length");
test_utf82wchar(um, sizeof(um), wm, sizeof(wm) / sizeof(*wm), 0, sizeof(wm) / sizeof(*wm),
"um/wm mixed languages");
test_utf82wchar(um, sizeof(um), wm, sizeof(wm) / sizeof(*wm) + 1, 0, sizeof(wm) / sizeof(*wm),
"um/wm boundaries +1");
test_utf82wchar(um, sizeof(um), nullptr, 0, 0, sizeof(wm) / sizeof(*wm),
"um/wm calculate length");
// The following tests won't pass on systems (e.g., Cygwin) where sizeof wchar_t is 2. That's
// due to several reasons but the primary one is that narrowing conversions of literals assigned
// to the wchar_t arrays above don't result in values that will be treated as errors by the
// conversion functions.
#if WCHAR_MAX != 0xffff
test_utf82wchar(u4, sizeof(u4), w4, sizeof(w4) / sizeof(*w4), 0, sizeof(w4) / sizeof(*w4),
"u4/w4 4 octets chars");
test_wchar2utf8(w4, sizeof(w4) / sizeof(*w4), u4, sizeof(u4), 0, sizeof(u4),
"w4/u4 4 octets chars");
test_wchar2utf8(wb, sizeof(wb) / sizeof(*wb), ub, sizeof(ub), 0, 0, "wb/ub bad chars");
test_wchar2utf8(wb, sizeof(wb) / sizeof(*wb), ub, sizeof(ub), UTF8_IGNORE_ERROR, sizeof(ub),
"wb/ub ignore bad chars");
test_wchar2utf8(wb, sizeof(wb) / sizeof(*wb), nullc, 0, 0, 0,
"wb calculate length of bad chars");
test_wchar2utf8(wb, sizeof(wb) / sizeof(*wb), nullc, 0, UTF8_IGNORE_ERROR, sizeof(ub),
"calculate length, ignore bad chars");
test_utf82wchar(ub1, sizeof(ub1), wb1, sizeof(wb1) / sizeof(*wb1), UTF8_IGNORE_ERROR,
sizeof(wb1) / sizeof(*wb1), "ub1/wb1 ignore bad chars");
test_utf82wchar(ub1, sizeof(ub1), nullptr, 0, 0, 0, "ub1 calculate length of bad chars");
test_utf82wchar(ub1, sizeof(ub1), nullptr, 0, UTF8_IGNORE_ERROR, sizeof(wb1) / sizeof(*wb1),
"ub1 calculate length, ignore bad chars");
#endif
}
// todo!("port this");
static void test_escape_sequences() {
say(L"Testing escape_sequences");
@ -2970,7 +2761,6 @@ static const test_t s_tests[]{
{TEST_GROUP("debounce"), test_debounce},
{TEST_GROUP("debounce"), test_debounce_timeout},
{TEST_GROUP("parser"), test_parser},
{TEST_GROUP("utf8"), test_utf8},
{TEST_GROUP("lru"), test_lru},
{TEST_GROUP("wcstod"), test_wcstod},
{TEST_GROUP("pager_navigation"), test_pager_navigation},

356
src/utf8.cpp
View file

@ -1,356 +0,0 @@
/*
* Copyright (c) 2007 Alexey Vatchenko <av@bsdua.org>
*
* Permission to use, copy, modify, and/or distribute this software for any
* purpose with or without fee is hereby granted, provided that the above
* copyright notice and this permission notice appear in all copies.
*
* THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
* WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
* MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
* ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
* ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
* OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
#include "config.h" // IWYU pragma: keep
#include "utf8.h"
#include <stdint.h> // IWYU pragma: keep
#include <limits>
#include <string>
#include "common.h"
#define _NXT 0x80
#define _SEQ2 0xc0
#define _SEQ3 0xe0
#define _SEQ4 0xf0
#define _BOM 0xfeff
// We can tweak the following typedef to allow us to simulate Windows-style 16 bit wchar's on Unix.
using utf8_wchar_t = wchar_t;
#define UTF8_WCHAR_MAX static_cast<wchar_t>(std::numeric_limits<utf8_wchar_t>::max())
using utf8_wstring_t = std::basic_string<utf8_wchar_t>;
static size_t utf8_to_wchar_internal(const char *in, size_t insize, utf8_wstring_t *out_string,
int flags);
static size_t wchar_to_utf8_internal(const utf8_wchar_t *in, size_t insize, char *out,
size_t outsize, int flags);
static bool safe_copy_wchar_to_utf8_wchar(const wchar_t *in, utf8_wchar_t *out, size_t count) {
bool result = true;
for (size_t i = 0; i < count; i++) {
wchar_t c = in[i];
if (c > UTF8_WCHAR_MAX) {
result = false;
break;
}
out[i] = c;
}
return result;
}
bool wchar_to_utf8_string(const std::wstring &str, std::string *result) {
result->clear();
const size_t inlen = str.size();
if (inlen == 0) {
return true;
}
bool success = false;
const wchar_t *input = str.c_str();
size_t outlen = wchar_to_utf8(input, inlen, nullptr, 0, 0);
if (outlen > 0) {
auto tmp = new char[outlen];
size_t outlen2 = wchar_to_utf8(input, inlen, tmp, outlen, 0);
if (outlen2 > 0) {
result->assign(tmp, outlen2);
success = true;
}
delete[] tmp;
}
return success;
}
size_t utf8_to_wchar(const char *in, size_t insize, std::wstring *out, int flags) {
if (in == nullptr || insize == 0) {
return 0;
}
size_t result;
if (sizeof(wchar_t) == sizeof(utf8_wchar_t)) { //!OCLINT(constant if expression)
result = utf8_to_wchar_internal(in, insize, reinterpret_cast<utf8_wstring_t *>(out), flags);
} else if (out == nullptr) {
result = utf8_to_wchar_internal(in, insize, nullptr, flags);
} else {
// Allocate a temporary buffer to hold the output, invoke the conversion with the temporary,
// and then copy it back.
utf8_wstring_t tmp_output;
result = utf8_to_wchar_internal(in, insize, &tmp_output, flags);
out->insert(out->end(), tmp_output.begin(), tmp_output.end());
}
return result;
}
size_t wchar_to_utf8(const wchar_t *in, size_t insize, char *out, size_t outsize, int flags) {
if (in == nullptr || insize == 0 || (outsize == 0 && out != nullptr)) {
return 0;
}
size_t result;
if (sizeof(wchar_t) == sizeof(utf8_wchar_t)) { //!OCLINT(constant if expression)
result = wchar_to_utf8_internal(reinterpret_cast<const utf8_wchar_t *>(in), insize, out,
outsize, flags);
} else {
// Allocate a temporary buffer to hold the input the std::copy performs the size conversion.
// Note: insize may be 0.
auto tmp_input = new utf8_wchar_t[insize];
if (!safe_copy_wchar_to_utf8_wchar(in, tmp_input, insize)) {
// Our utf8_wchar_t is UCS-16 and there was an astral character.
result = 0;
} else {
// Invoke the conversion with the temporary, then clean up the input.
result = wchar_to_utf8_internal(tmp_input, insize, out, outsize, flags);
}
delete[] tmp_input;
}
return result;
}
static int __wchar_forbitten(utf8_wchar_t sym);
static int __utf8_forbitten(unsigned char octet);
static int __wchar_forbitten(utf8_wchar_t sym) {
// Surrogate pairs.
if (sym >= 0xd800 && sym <= 0xdfff) return -1;
return 0;
}
static int __utf8_forbitten(unsigned char octet) {
switch (octet) {
case 0xc0:
case 0xc1:
case 0xf5:
case 0xff: {
return -1;
}
default: {
return 0;
}
}
}
/// This function translates UTF-8 string into UCS-2 or UCS-4 string (all symbols will be in local
/// machine byte order). It takes the following arguments:
///
/// in - input UTF-8 string. It can be null-terminated.
/// insize - size of input string in bytes.
/// out_string - result buffer for UCS-2/4 string.
///
/// RETURN VALUES
/// The function returns size of result buffer (in wide characters).
/// Zero is returned in case of error.
///
/// CAVEATS
/// 1. If UTF-8 string contains zero symbols, they will be translated
/// as regular symbols.
///
/// 2. If UTF8_IGNORE_ERROR or UTF8_SKIP_BOM flag is set, sizes may vary when `out' is NULL and not
/// NULL. It's because of special UTF-8 sequences which may result in forbitten (by RFC3629) UNICODE
/// characters. So, the caller must check return value every time and not prepare buffer in advance
/// (\0 terminate) but after calling this function.
static size_t utf8_to_wchar_internal(const char *in, size_t insize, utf8_wstring_t *out_string,
int flags) {
unsigned char *p, *lim;
utf8_wchar_t high;
size_t n, total, i, n_bits;
if (in == nullptr || insize == 0) return 0;
if (out_string != nullptr) out_string->clear();
total = 0;
p = reinterpret_cast<unsigned char *>(const_cast<char *>(in));
lim = p + insize;
for (; p < lim; p += n) {
if (__utf8_forbitten(*p) != 0 && (flags & UTF8_IGNORE_ERROR) == 0) return 0;
// Get number of bytes for one wide character.
n = 1; // default: 1 byte. Used when skipping bytes
if ((*p & 0x80) == 0)
high = static_cast<utf8_wchar_t>(*p);
else if ((*p & 0xe0) == _SEQ2) {
n = 2;
high = static_cast<utf8_wchar_t>(*p & 0x1f);
} else if ((*p & 0xf0) == _SEQ3) {
n = 3;
high = static_cast<utf8_wchar_t>(*p & 0x0f);
} else if ((*p & 0xf8) == _SEQ4) {
n = 4;
high = static_cast<utf8_wchar_t>(*p & 0x07);
} else {
if ((flags & UTF8_IGNORE_ERROR) == 0) return 0;
continue;
}
// Does the sequence header tell us truth about length?
if (static_cast<size_t>(lim - p) <= n - 1) {
if ((flags & UTF8_IGNORE_ERROR) == 0) return 0;
n = 1;
continue; // skip
}
// Validate sequence. All symbols must have higher bits set to 10xxxxxx.
if (n > 1) {
for (i = 1; i < n; i++) {
if ((p[i] & 0xc0) != _NXT) break;
}
if (i != n) {
if ((flags & UTF8_IGNORE_ERROR) == 0) return 0;
n = 1;
continue; // skip
}
}
total++;
if (out_string == nullptr) continue;
uint32_t out_val = 0;
n_bits = 0;
for (i = 1; i < n; i++) {
out_val |= static_cast<utf8_wchar_t>(p[n - i] & 0x3f) << n_bits;
n_bits += 6; // 6 low bits in every byte
}
out_val |= high << n_bits;
bool skip = false;
if (__wchar_forbitten(out_val) != 0) {
if ((flags & UTF8_IGNORE_ERROR) == 0) {
return 0; // forbidden character
}
skip = true;
} else if (out_val == _BOM && (flags & UTF8_SKIP_BOM) != 0) {
skip = true;
}
if (skip) {
total--;
} else if (out_val > static_cast<uint32_t>(UTF8_WCHAR_MAX)) {
// wchar_t is UCS-2, but the UTF-8 specified an astral character.
return 0;
} else {
out_string->push_back(out_val);
}
}
return total;
}
/// This function translates UCS-2/4 symbols (given in local machine byte order) into UTF-8 string.
/// It takes the following arguments:
///
/// in - input unicode string. It can be null-terminated.
/// insize - size of input string in wide characters.
/// out - result buffer for utf8 string. If out is NULL, function returns size of result buffer.
/// outsize - size of result buffer.
///
/// RETURN VALUES
/// The function returns size of result buffer (in bytes). Zero is returned in case of error.
///
/// CAVEATS
/// If UCS-4 string contains zero symbols, they will be translated as regular symbols.
static size_t wchar_to_utf8_internal(const utf8_wchar_t *in, size_t insize, char *out,
size_t outsize, int flags) {
const utf8_wchar_t *w, *wlim;
unsigned char *p, *lim;
size_t total, n;
if (in == nullptr || insize == 0 || (outsize == 0 && out != nullptr)) return 0;
w = in;
wlim = w + insize;
p = reinterpret_cast<unsigned char *>(out);
lim = p + outsize;
total = 0;
for (; w < wlim; w++) {
if (__wchar_forbitten(*w) != 0) {
if ((flags & UTF8_IGNORE_ERROR) == 0) return 0;
continue;
}
if (*w == _BOM && (flags & UTF8_SKIP_BOM) != 0) continue;
const int32_t w_wide = *w;
if (w_wide < 0) {
if ((flags & UTF8_IGNORE_ERROR) == 0) return 0;
continue;
}
if (w_wide <= 0x0000007f) {
n = 1;
} else if (w_wide <= 0x000007ff) {
n = 2;
} else if (w_wide <= 0x0000ffff) {
n = 3;
} else if (w_wide <= 0x001fffff) {
n = 4;
} else {
DIE("invalid wide char");
}
total += n;
if (out == nullptr) continue;
if (size_t(lim - p) <= n - 1) return 0; // no space left
// Extract the wchar_t as big-endian. If wchar_t is UCS-16, the first two bytes will be 0.
unsigned char oc[4];
uint32_t w_tmp = *w;
oc[3] = w_tmp & 0xFF;
w_tmp >>= 8;
oc[2] = w_tmp & 0xFF;
w_tmp >>= 8;
oc[1] = w_tmp & 0xFF;
w_tmp >>= 8;
oc[0] = w_tmp & 0xFF;
switch (n) {
case 1: {
p[0] = oc[3];
break;
}
case 2: {
p[1] = _NXT | (oc[3] & 0x3f);
p[0] = _SEQ2 | (oc[3] >> 6) | ((oc[2] & 0x07) << 2);
break;
}
case 3: {
p[2] = _NXT | (oc[3] & 0x3f);
p[1] = _NXT | (oc[3] >> 6) | ((oc[2] & 0x0f) << 2);
p[0] = _SEQ3 | ((oc[2] & 0xf0) >> 4);
break;
}
case 4: {
p[3] = _NXT | (oc[3] & 0x3f);
p[2] = _NXT | (oc[3] >> 6) | ((oc[2] & 0x0f) << 2);
p[1] = _NXT | ((oc[2] & 0xf0) >> 4) | ((oc[1] & 0x03) << 4);
p[0] = _SEQ4 | ((oc[1] & 0x1f) >> 2);
break;
}
default: {
DIE("unexpected utff8 len");
}
}
// NOTE: do not check here for forbidden UTF-8 characters. They cannot appear here because
// we do proper conversion.
p += n;
}
return total;
}

37
src/utf8.h
View file

@ -1,37 +0,0 @@
/*
* Copyright (c) 2007 Alexey Vatchenko <av@bsdua.org>
*
* Permission to use, copy, modify, and/or distribute this software for any
* purpose with or without fee is hereby granted, provided that the above
* copyright notice and this permission notice appear in all copies.
*
* THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
* WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
* MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
* ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
* ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
* OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
// Implementation of UTF-8 charset encoding (RFC3629).
#ifndef _UTF8_H_
#define _UTF8_H_
#include <stddef.h>
#include <string>
#define UTF8_IGNORE_ERROR 0x01
#define UTF8_SKIP_BOM 0x02
/// Convert a string between UTF8 and UCS-2/4 (depending on size of wchar_t). Returns true if
/// successful, storing the result of the conversion in *result*.
bool wchar_to_utf8_string(const std::wstring &str, std::string *result);
/// Convert a string between UTF8 and UCS-2/4 (depending on size of wchar_t). Returns nonzero if
/// successful, storing the result of the conversion in *out*.
size_t utf8_to_wchar(const char *in, size_t insize, std::wstring *out, int flags);
size_t wchar_to_utf8(const wchar_t *in, size_t insize, char *out, size_t outsize, int flags);
#endif /* !_UTF8_H_ */