utf8: portable Windows UTF-8 boundary for argv, fopen and CreateFile

CMakeLists.txt

@@ -19,9 +19,12 @@ add_custom_target(version ALL
 # and older x86_64 toolchains need the explicit dependency.
 find_package(Threads REQUIRED)
 
-# Suppress MSVC fopen/sprintf deprecation warnings
+# Suppress MSVC fopen/sprintf deprecation warnings, force UTF-8 source and
+# execution charsets so non-ASCII string literals (CJK in text-chunker.h,
+# prompt-builder.cpp) survive the compile without a BOM.
 if(MSVC)
     add_compile_definitions(_CRT_SECURE_NO_WARNINGS)
+    add_compile_options(/utf-8)
 endif()
 
 # Put executables and backend .so in the same directory (build root).

src/audio-io.h

@@ -18,6 +18,9 @@
 // audio-resample.h: sample rate conversion
 #include "audio-resample.h"
 
+// utf8.h: utf8_fopen, the path-UTF-8-aware fopen used below.
+#include "utf8.h"
+
 // case-insensitive extension check
 static bool audio_io_ends_with(const char * str, const char * suffix) {
     int slen = (int) strlen(str);
@@ -44,7 +47,7 @@ static bool audio_io_ends_with(const char * str, const char * suffix) {
 // Load entire file into memory. Caller frees the returned pointer.
 static uint8_t * audio_io_load_file(const char * path, size_t * size_out) {
     *size_out = 0;
-    FILE * fp = fopen(path, "rb");
+    FILE * fp = utf8_fopen(path, "rb");
     if (!fp) {
         fprintf(stderr, "[Audio] Cannot open %s\n", path);
         return NULL;
@@ -333,7 +336,7 @@ static bool audio_write_wav(const char * path, const float * audio, int T_audio,
     }
 
     const bool to_stdout = (path[0] == '-' && path[1] == '\0');
-    FILE *     fp        = to_stdout ? stdout : fopen(path, "wb");
+    FILE *     fp        = to_stdout ? stdout : utf8_fopen(path, "wb");
     if (!fp) {
         fprintf(stderr, "[WAV] Cannot open %s for writing\n", path);
         return false;

src/debug.h

@@ -4,6 +4,8 @@
 // dump.
 // File format : [int32 ndims] [int32 dim0] [int32 dim1] ... [float data...]
 
+#include "utf8.h"
+
 #include <cmath>
 #include <cstdint>
 #include <cstdio>
@@ -34,7 +36,7 @@ static void debug_dump(const DebugDumper * d, const char * name, const float * d
     for (int i = 0; i < ndims; i++) {
         numel *= shape[i];
     }
-    FILE * f = fopen(path, "wb");
+    FILE * f = utf8_fopen(path, "wb");
     if (!f) {
         fprintf(stderr, "[Debug] cannot write %s\n", path);
         return;

src/gguf-weights.h

@@ -23,8 +23,7 @@
 #include <vector>
 
 #ifdef _WIN32
-#    define NOMINMAX
+#    include "utf8.h"
-#    include <windows.h>
 #else
 #    include <fcntl.h>
 #    include <sys/mman.h>
@@ -79,7 +78,9 @@ static bool gf_load(GGUFModel * gf, const char * path) {
 
     // mmap the file
 #ifdef _WIN32
-    gf->fh = CreateFileA(path, GENERIC_READ, FILE_SHARE_READ, NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL);
+    std::wstring wpath = utf8_to_wide(path);
+    gf->fh =
+        CreateFileW(wpath.c_str(), GENERIC_READ, FILE_SHARE_READ, NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL);
     if (gf->fh == INVALID_HANDLE_VALUE) {
         fprintf(stderr, "[GGUF] Cannot open %s\n", path);
         return false;
@@ -87,7 +88,7 @@ static bool gf_load(GGUFModel * gf, const char * path) {
     LARGE_INTEGER li;
     GetFileSizeEx(gf->fh, &li);
     gf->file_size = (size_t) li.QuadPart;
-    gf->mh        = CreateFileMappingA(gf->fh, NULL, PAGE_READONLY, 0, 0, NULL);
+    gf->mh        = CreateFileMappingW(gf->fh, NULL, PAGE_READONLY, 0, 0, NULL);
     if (!gf->mh) {
         CloseHandle(gf->fh);
         fprintf(stderr, "[GGUF] CreateFileMapping failed %s\n", path);

src/utf8.h

@@ -0,0 +1,88 @@
+#pragma once
+// utf8.h: portable UTF-8 boundary for Windows. Inside the project everything
+// is UTF-8 ; this header bridges to the Windows-native UTF-16 APIs at the
+// three places where the OS forces a recode: argv (CRT decodes from CP_ACP),
+// fopen (CRT does too), and any direct Win32 *A call (CreateFileA etc.).
+// POSIX is UTF-8 by convention and every helper degrades to a passthrough.
+
+#include <cstdio>
+#include <cstdlib>
+
+#if defined(_WIN32)
+#    define WIN32_LEAN_AND_MEAN
+#    define NOMINMAX
+// clang-format off
+// windows.h must precede shellapi.h: shellapi.h pulls in types (HDROP,
+// DECLSPEC_IMPORT, ...) declared by windows.h. SortIncludes would alphabetise
+// the pair and break the build, so the off/on pragmas pin the order.
+#    include <windows.h>
+#    include <shellapi.h>
+// clang-format on
+#    include <string>
+#endif
+
+#if defined(_WIN32)
+// Decode a UTF-8 C string to a wide string (UTF-16). Empty on null or empty
+// input. Used by every Win32 path-taking call site (CreateFileW, MoveFileW
+// and friends) wherever the project still needs the raw wide form.
+static std::wstring utf8_to_wide(const char * s) {
+    if (!s || !*s) {
+        return std::wstring();
+    }
+    int n = MultiByteToWideChar(CP_UTF8, 0, s, -1, NULL, 0);
+    if (n <= 1) {
+        return std::wstring();
+    }
+    std::wstring w((size_t) (n - 1), L'\0');
+    MultiByteToWideChar(CP_UTF8, 0, s, -1, &w[0], n);
+    return w;
+}
+#endif
+
+// First line of main(). Rebuilds argv as UTF-8 from the wide command line
+// and switches console I/O to CP_UTF8 so fprintf with non-ASCII bytes
+// renders correctly without an explicit chcp 65001. POSIX no-op.
+//
+// The replacement argv is allocated once and leaked for the process
+// lifetime: the OS reclaims it on exit, and the top of main is the only
+// site that would ever free it.
+static void utf8_init(int * argc, char *** argv) {
+#if defined(_WIN32)
+    SetConsoleOutputCP(CP_UTF8);
+    SetConsoleCP(CP_UTF8);
+
+    int      wargc = 0;
+    LPWSTR * wargv = CommandLineToArgvW(GetCommandLineW(), &wargc);
+    if (!wargv) {
+        return;
+    }
+
+    char ** new_argv = (char **) malloc(sizeof(char *) * (size_t) (wargc + 1));
+    for (int i = 0; i < wargc; i++) {
+        int n       = WideCharToMultiByte(CP_UTF8, 0, wargv[i], -1, NULL, 0, NULL, NULL);
+        new_argv[i] = (char *) malloc((size_t) n);
+        WideCharToMultiByte(CP_UTF8, 0, wargv[i], -1, new_argv[i], n, NULL, NULL);
+    }
+    new_argv[wargc] = NULL;
+    LocalFree(wargv);
+
+    *argc = wargc;
+    *argv = new_argv;
+#else
+    (void) argc;
+    (void) argv;
+#endif
+}
+
+// UTF-8 aware fopen. POSIX passthrough. Windows decodes the UTF-8 path and
+// mode to UTF-16 and calls _wfopen, the only fopen variant on MSVC that
+// bypasses CP_ACP.
+static FILE * utf8_fopen(const char * path, const char * mode) {
+#if defined(_WIN32)
+    std::wstring wpath = utf8_to_wide(path);
+    std::wstring wmode = utf8_to_wide(mode);
+    return _wfopen(wpath.c_str(), wmode.c_str());
+#else
+    return fopen(path, mode);
+#endif
+}

tools/quantize.cpp

@@ -25,6 +25,7 @@
 
 #include "ggml.h"
 #include "gguf.h"
+#include "utf8.h"
 #include "version.h"
 
 // Quant variant: base type + optional bump rules for important tensors
@@ -227,6 +228,7 @@ static bool to_f32(const void * src, float * dst, int64_t n, enum ggml_type type
 }
 
 int main(int argc, char ** argv) {
+    utf8_init(&argc, &argv);
     if (argc != 4) {
         fprintf(stderr, "qwentts.cpp %s\n\n", QWEN_VERSION);
         fprintf(stderr, "Usage: %s <input.gguf> <output.gguf> <type>\n", argv[0]);
@@ -251,12 +253,14 @@ int main(int argc, char ** argv) {
 
     // Mmap input file
 #ifdef _WIN32
-    HANDLE fh = CreateFileA(inp_path, GENERIC_READ, FILE_SHARE_READ, NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL);
+    std::wstring winp_path = utf8_to_wide(inp_path);
+    HANDLE       fh =
+        CreateFileW(winp_path.c_str(), GENERIC_READ, FILE_SHARE_READ, NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL);
     if (fh == INVALID_HANDLE_VALUE) {
         fprintf(stderr, "[Quantize] Failed to open %s\n", inp_path);
         return 1;
     }
-    HANDLE mh = CreateFileMappingA(fh, NULL, PAGE_READONLY, 0, 0, NULL);
+    HANDLE mh = CreateFileMappingW(fh, NULL, PAGE_READONLY, 0, 0, NULL);
     if (!mh) {
         fprintf(stderr, "[Quantize] CreateFileMapping failed %s\n", inp_path);
         CloseHandle(fh);
@@ -396,7 +400,7 @@ int main(int argc, char ** argv) {
     }
 
     // Stream tensor data one at a time (low memory)
-    FILE * fout = fopen(out_path, "ab");
+    FILE * fout = utf8_fopen(out_path, "ab");
     if (!fout) {
         fprintf(stderr, "[Quantize] Failed to open %s for append\n", out_path);
         return 1;

tools/qwen-codec.cpp

@@ -13,6 +13,7 @@
 #include "audio-io.h"
 #include "backend.h"
 #include "pipeline-codec.h"
+#include "utf8.h"
 #include "version.h"
 
 #include <cstdint>
@@ -82,7 +83,7 @@ static std::vector<uint8_t> pack_codes(const std::vector<int32_t> & codes) {
 // Read a .rvq file and unpack it into K*T codes. T is inferred from the
 // file size: T = (filesize * 8) / (K * QWEN_TOKENIZER_CODE_BITS).
 static bool read_rvq(const char * path, int K, std::vector<int32_t> & codes, int * n_frames) {
-    FILE * f = fopen(path, "rb");
+    FILE * f = utf8_fopen(path, "rb");
     if (!f) {
         fprintf(stderr, "[Codec] FATAL: cannot open %s\n", path);
         return false;
@@ -117,7 +118,7 @@ static bool read_rvq(const char * path, int K, std::vector<int32_t> & codes, int
 // Pack and write a .rvq file.
 static bool write_rvq(const char * path, const std::vector<int32_t> & codes) {
     std::vector<uint8_t> packed = pack_codes(codes);
-    FILE *               f      = fopen(path, "wb");
+    FILE *               f      = utf8_fopen(path, "wb");
     if (!f) {
         fprintf(stderr, "[Codec] FATAL: cannot open %s for write\n", path);
         return false;
@@ -154,6 +155,7 @@ static int infer_mode(const char * path) {
 }
 
 int main(int argc, char ** argv) {
+    utf8_init(&argc, &argv);
     if (argc <= 1) {
         print_usage(argv[0]);
         return 0;

tools/qwen-tts.cpp

@@ -14,6 +14,7 @@
 #include "backend.h"
 #include "bpe.h"
 #include "pipeline-tts.h"
+#include "utf8.h"
 #include "version.h"
 
 #include <cstdio>
@@ -104,7 +105,7 @@ static std::string read_stdin_text() {
 
 // Read a small text file into a string. Trims trailing newlines.
 static bool read_text_file(const char * path, std::string & out) {
-    FILE * f = fopen(path, "rb");
+    FILE * f = utf8_fopen(path, "rb");
     if (!f) {
         fprintf(stderr, "[CLI] FATAL: cannot open '%s'\n", path);
         return false;
@@ -400,6 +401,7 @@ static int run(const Args & a) {
 }
 
 int main(int argc, char ** argv) {
+    utf8_init(&argc, &argv);
     Args a;
     if (!parse_args(argc, argv, a)) {
         print_usage(argv[0]);