utf8: normalize inbound text bytes to UTF-8

Same gap as omnivoice.cpp: the UTF-8 boundary covers argv, console
output, and fopen but not the bytes arriving from stdin or text files.
Windows shells and editors hand those over as UTF-16 with BOM
(PowerShell redirection, Notepad, Out-File) or the ANSI codepage (cmd
pipes), and the raw bytes reach the tokenizer as garbage.

utf8_normalize() closes the gap: UTF-8 BOM stripped on every platform,
UTF-16 BOM losslessly recoded to UTF-8, bytes failing UTF-8 validation
decoded from the ANSI codepage. Valid UTF-8 passes through untouched.
Wired into read_stdin_text (binary mode stdin so CRLF translation
cannot eat UTF-16 0x0D bytes) and read_text_file, which also moves
from raw fopen to utf8_fopen so a non-ASCII --ref-text path opens.

src/utf8.h

@@ -1,8 +1,9 @@
 #pragma once
 // utf8.h: portable UTF-8 boundary for Windows. Inside the project everything
 // is UTF-8; this header bridges to the Windows-native UTF-16 APIs at the
-// three places where the OS forces a recode: argv (CRT decodes from CP_ACP),
+// four places where the OS forces a recode: argv (CRT decodes from CP_ACP),
-// fopen (CRT does too), and any direct Win32 *A call (CreateFileA etc.).
+// fopen (CRT does too), any direct Win32 *A call (CreateFileA etc.), and
+// text bytes arriving from stdin or files in a shell-chosen encoding.
 // POSIX is UTF-8 by convention and every helper degrades to a passthrough.
 
 #include <cstdio>
@@ -86,3 +87,54 @@ static FILE * utf8_fopen(const char * path, const char * mode) {
     return fopen(path, mode);
 #endif
 }
+
+#if !defined(_WIN32)
+#    include <string>
+#endif
+
+// Normalizes a text buffer read from stdin or a file to UTF-8 in place.
+// Windows shells and editors hand bytes over in whatever encoding they
+// default to: UTF-16 with BOM (PowerShell redirection, Notepad) or the
+// ANSI codepage (cmd pipes, legacy editors). Both recode losslessly. A
+// UTF-8 BOM is stripped on every platform, valid UTF-8 passes through
+// untouched, and the ANSI decode only runs on bytes that fail UTF-8
+// validation so it can never alter a well formed prompt.
+static void utf8_normalize(std::string & s) {
+    if (s.size() >= 3 && (unsigned char) s[0] == 0xEF && (unsigned char) s[1] == 0xBB && (unsigned char) s[2] == 0xBF) {
+        s.erase(0, 3);
+        return;
+    }
+#if defined(_WIN32)
+    if (s.size() >= 2) {
+        const unsigned char b0 = (unsigned char) s[0];
+        const unsigned char b1 = (unsigned char) s[1];
+        if ((b0 == 0xFF && b1 == 0xFE) || (b0 == 0xFE && b1 == 0xFF)) {
+            if (b0 == 0xFE) {
+                // UTF-16BE: swap every pair so the buffer reads as UTF-16LE.
+                for (size_t i = 0; i + 1 < s.size(); i += 2) {
+                    char t   = s[i];
+                    s[i]     = s[i + 1];
+                    s[i + 1] = t;
+                }
+            }
+            const wchar_t * w = (const wchar_t *) (s.data() + 2);
+            const int       n = (int) ((s.size() - 2) / sizeof(wchar_t));
+            int             u = WideCharToMultiByte(CP_UTF8, 0, w, n, NULL, 0, NULL, NULL);
+            std::string     out((size_t) u, '\0');
+            WideCharToMultiByte(CP_UTF8, 0, w, n, &out[0], u, NULL, NULL);
+            s.swap(out);
+            return;
+        }
+    }
+    if (s.empty() || MultiByteToWideChar(CP_UTF8, MB_ERR_INVALID_CHARS, s.data(), (int) s.size(), NULL, 0) > 0) {
+        return;
+    }
+    int          n = MultiByteToWideChar(CP_ACP, 0, s.data(), (int) s.size(), NULL, 0);
+    std::wstring w((size_t) n, L'\0');
+    MultiByteToWideChar(CP_ACP, 0, s.data(), (int) s.size(), &w[0], n);
+    int         u = WideCharToMultiByte(CP_UTF8, 0, w.data(), n, NULL, 0, NULL, NULL);
+    std::string out((size_t) u, '\0');
+    WideCharToMultiByte(CP_UTF8, 0, w.data(), n, &out[0], u, NULL, NULL);
+    s.swap(out);
+#endif
+}

tools/qwen-tts.cpp

@@ -14,6 +14,7 @@
 #include "audio-io.h"
 #include "qwen.h"
 #include "rvq-file.h"
+#include "utf8.h"
 
 #include <cstdio>
 #include <cstdlib>
@@ -23,6 +24,11 @@
 #include <sstream>
 #include <string>
 
+#if defined(_WIN32)
+#    include <fcntl.h>
+#    include <io.h>
+#endif
+
 static void print_usage(const char * prog) {
     fprintf(stderr, "qwentts.cpp %s\n\n", qt_version());
     fprintf(stderr,
@@ -98,19 +104,25 @@ struct Args {
     float        codec_left_context_sec;
 };
 
-// Read all of stdin into a string. Trims trailing newlines so a piped
+// Read all of stdin into a string. Binary mode on Windows so UTF-16 input
-// text file behaves like clean utterance input.
+// survives CRLF translation, then normalised to UTF-8. Trims trailing
+// newlines so a piped text file behaves like clean utterance input.
 static std::string read_stdin_text() {
+#if defined(_WIN32)
+    _setmode(_fileno(stdin), _O_BINARY);
+#endif
     std::ostringstream ss;
     ss << std::cin.rdbuf();
     std::string s = ss.str();
+    utf8_normalize(s);
     while (!s.empty() && (s.back() == '\n' || s.back() == '\r')) {
         s.pop_back();
     }
     return s;
 }
 
-// Read a small text file into a string. Trims trailing newlines.
+// Read a small text file into a string, normalised to UTF-8.
+// Trims trailing newlines.
 // 11 bits per code (V <= 2048), matching qwen-codec.
 static const int RVQ_CODE_BITS = 11;
 
@@ -140,7 +152,7 @@ static bool read_spk_file(const char * path, std::vector<float> & emb) {
 }
 
 static bool read_text_file(const char * path, std::string & out) {
-    FILE * f = fopen(path, "rb");
+    FILE * f = utf8_fopen(path, "rb");
     if (!f) {
         fprintf(stderr, "[CLI] FATAL: cannot open '%s'\n", path);
         return false;
@@ -160,6 +172,7 @@ static bool read_text_file(const char * path, std::string & out) {
         return false;
     }
     fclose(f);
+    utf8_normalize(out);
     while (!out.empty() && (out.back() == '\n' || out.back() == '\r')) {
         out.pop_back();
     }