30#include "Exception.hpp"
42 static void SkipUtf8Bom(FILE* fp);
50 if ((ch & 0xF0) == 0xE0) {
52 }
else if ((ch & 0x80) == 0x00) {
54 }
else if ((ch & 0xE0) == 0xC0) {
56 }
else if ((ch & 0xF8) == 0xF0) {
58 }
else if ((ch & 0xFC) == 0xF8) {
60 }
else if ((ch & 0xFE) == 0xFC) {
70 size_t length = NextCharLengthNoException(str);
82 const size_t length = NextCharLengthNoException(str - 3);
88 const size_t length = NextCharLengthNoException(str - 1);
94 const size_t length = NextCharLengthNoException(str - 2);
99 for (
size_t i = 4; i <= 6; i++) {
100 const size_t length = NextCharLengthNoException(str - i);
112 return str + NextCharLength(str);
119 return str - PrevCharLength(str);
127 while (*str !=
'\0') {
141 while (!IsLineEndingOrFileEnding(*str) && *str != ch) {
151 return ch ==
'\0' || ch ==
'\n' || ch ==
'\r';
157 static std::string
FromSubstr(
const char* str,
size_t length) {
159 newStr.resize(length);
160 strncpy(
const_cast<char*
>(newStr.c_str()), str, length);
169 while (byteLength > 0) {
183 static std::string
TruncateUTF8(
const char* str,
size_t maxByteLength) {
184 std::string wordTrunc;
185 if (NotShorterThan(str, maxByteLength)) {
187 const char* pStr = str;
189 const size_t charLength = NextCharLength(pStr);
190 if (len + charLength > maxByteLength) {
196 wordTrunc = FromSubstr(str, len);
206 static void ReplaceAll(std::string& str,
const char* from,
const char* to) {
207 std::string::size_type pos = 0;
208 std::string::size_type fromLen = strlen(from);
209 std::string::size_type toLen = strlen(to);
210 while ((pos = str.find(from, pos)) != std::string::npos) {
211 str.replace(pos, fromLen, to);
219 static std::string
Join(
const std::vector<std::string>& strings,
220 const std::string& separator) {
221 std::ostringstream buffer;
223 for (
const auto& str : strings) {
236 static std::string
Join(
const std::vector<std::string>& strings) {
237 std::ostringstream buffer;
238 for (
const auto& str : strings) {
244 static void GetByteMap(
const char* str,
const size_t utf8Length,
245 std::vector<size_t>* byteMap) {
246 if (byteMap->size() < utf8Length) {
247 byteMap->resize(utf8Length);
249 const char* pstr = str;
250 for (
size_t i = 0; i < utf8Length; i++) {
251 (*byteMap)[i] = pstr - str;
252 pstr = NextChar(pstr);
257 static std::wstring GetPlatformString(
const std::string& str) {
261 static std::string GetPlatformString(
const std::string& str) {
return str; }
265 static std::string U16ToU8(
const std::wstring& wstr) {
267 int length =
static_cast<int>(wstr.length());
268 int convcnt = WideCharToMultiByte(CP_UTF8, 0, wstr.c_str(), length, NULL, 0,
272 WideCharToMultiByte(CP_UTF8, 0, wstr.c_str(), length, &ret[0], convcnt,
278 static std::wstring U8ToU16(
const std::string& str) {
280 int length =
static_cast<int>(str.length());
281 int convcnt = MultiByteToWideChar(CP_UTF8, 0, str.c_str(), length, NULL, 0);
284 MultiByteToWideChar(CP_UTF8, 0, str.c_str(), length, &ret[0], convcnt);
Definition: Exception.hpp:77
UTF8 std::string utilities.
Definition: UTF8Util.hpp:37
static bool IsLineEndingOrFileEnding(const char ch)
Returns true if the character is a line ending or end of file.
Definition: UTF8Util.hpp:150
static size_t PrevCharLength(const char *str)
Returns the length in byte for the previous UTF8 character.
Definition: UTF8Util.hpp:80
static std::string FromSubstr(const char *str, size_t length)
Copies a substring with given length to a new string.
Definition: UTF8Util.hpp:157
static void ReplaceAll(std::string &str, const char *from, const char *to)
Replaces all patterns in a std::string in place.
Definition: UTF8Util.hpp:206
static size_t NextCharLengthNoException(const char *str)
Returns the length in byte for the next UTF8 character.
Definition: UTF8Util.hpp:48
static bool NotShorterThan(const char *str, size_t byteLength)
Returns true if the given std::string is longer or as long as the given length.
Definition: UTF8Util.hpp:168
static std::string Join(const std::vector< std::string > &strings)
Joins a std::string vector in to a std::string.
Definition: UTF8Util.hpp:236
static std::string TruncateUTF8(const char *str, size_t maxByteLength)
Truncates a std::string with a maximal length in byte.
Definition: UTF8Util.hpp:183
static size_t Length(const char *str)
Returns the UTF8 length of a valid UTF8 std::string.
Definition: UTF8Util.hpp:125
static const char * FindNextInline(const char *str, const char ch)
Finds a character in the same line.
Definition: UTF8Util.hpp:140
static size_t NextCharLength(const char *str)
Returns the length in byte for the next UTF8 character.
Definition: UTF8Util.hpp:69
static std::string Join(const std::vector< std::string > &strings, const std::string &separator)
Joins a std::string vector in to a std::string with a separator.
Definition: UTF8Util.hpp:219
static const char * PrevChar(const char *str)
Move the char* pointer before the previous UTF8 character.
Definition: UTF8Util.hpp:118
static const char * NextChar(const char *str)
Returns the char* pointer over the next UTF8 character.
Definition: UTF8Util.hpp:111