FIX #6 by encoding UTF-8 into UTF-16LE

Author: Boris Filippov

jamspell/lang_model.cpp

@@ -127,7 +127,7 @@ bool TLangModel::Train(const std::string& fileName, const std::string& alphabetF
     std::unordered_map<TGram2Key, TCount, TGram2KeyHash> grams2;
     std::unordered_map<TGram3Key, TCount, TGram3KeyHash> grams3;
 
-    std::cerr << "[info] generating N-grams " << sentences.size() << std::endl;
+    std::cerr << "[info] generating N-grams " << sentenceIds.size() << std::endl;
     uint64_t lastTime = GetCurrentTimeMs();
     size_t total = sentenceIds.size();
     for (size_t i = 0; i < total; ++i) {

jamspell/utils.cpp

@@ -113,7 +113,7 @@ std::wstring UTF8ToWide(const std::string& text) {
     using boost::locale::conv::utf_to_utf;
     return utf_to_utf<wchar_t>(text.c_str(), text.c_str() + text.size());
 #else
-    std::wstring_convert<std::codecvt_utf8_utf16<wchar_t>> converter;
+    std::wstring_convert<std::codecvt_utf8_utf16<wchar_t, 0x10ffff, std::little_endian>> converter;
     return converter.from_bytes(text);
 #endif
 }
@@ -123,7 +123,7 @@ std::string WideToUTF8(const std::wstring& text) {
     using boost::locale::conv::utf_to_utf;
     return utf_to_utf<char>(text.c_str(), text.c_str() + text.size());
 #else
-    using convert_type = std::codecvt_utf8<wchar_t>;
+    using convert_type = std::codecvt_utf8<wchar_t, 0x10ffff, std::little_endian>;
     std::wstring_convert<convert_type, wchar_t> converter;
     return converter.to_bytes(text);
 #endif