enne2/qwentts.cpp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

utf8: portable Windows UTF-8 boundary for argv, fopen and CreateFile

Browse Source
This commit is contained in:
Pascal
2026-05-14 14:10:26 +02:00
parent 57d12ba4e5
commit eb48f33f09
8 changed files with 119 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files
CMakeLists.txt
+4 -1
View File
@@ -19,9 +19,12 @@ add_custom_target(version ALL
# and older x86_64 toolchains need the explicit dependency.
find_package(Threads REQUIRED)
# Suppress MSVC fopen/sprintf deprecation warnings
# Suppress MSVC fopen/sprintf deprecation warnings, force UTF-8 source and
# execution charsets so non-ASCII string literals (CJK in text-chunker.h,
# prompt-builder.cpp) survive the compile without a BOM.
if(MSVC)
add_compile_definitions(_CRT_SECURE_NO_WARNINGS)
add_compile_options(/utf-8)
endif()
# Put executables and backend .so in the same directory (build root).
src/audio-io.h
+5 -2
View File
@@ -18,6 +18,9 @@
// audio-resample.h: sample rate conversion
#include "audio-resample.h"
// utf8.h: utf8_fopen, the path-UTF-8-aware fopen used below.
#include "utf8.h"
// case-insensitive extension check
static bool audio_io_ends_with(const char * str, const char * suffix) {
int slen = (int) strlen(str);
@@ -44,7 +47,7 @@ static bool audio_io_ends_with(const char * str, const char * suffix) {
// Load entire file into memory. Caller frees the returned pointer.
static uint8_t * audio_io_load_file(const char * path, size_t * size_out) {
*size_out = 0;
FILE * fp = fopen(path, "rb");
FILE * fp = utf8_fopen(path, "rb");
if (!fp) {
fprintf(stderr, "[Audio] Cannot open %s\n", path);
return NULL;
@@ -333,7 +336,7 @@ static bool audio_write_wav(const char * path, const float * audio, int T_audio,
}
const bool to_stdout = (path[0] == '-' && path[1] == '\0');
FILE * fp = to_stdout ? stdout : fopen(path, "wb");
FILE * fp = to_stdout ? stdout : utf8_fopen(path, "wb");
if (!fp) {
fprintf(stderr, "[WAV] Cannot open %s for writing\n", path);
return false;
src/debug.h
+3 -1
View File
@@ -4,6 +4,8 @@
// dump.
// File format : [int32 ndims] [int32 dim0] [int32 dim1] ... [float data...]
#include "utf8.h"
#include <cmath>
#include <cstdint>
#include <cstdio>
@@ -34,7 +36,7 @@ static void debug_dump(const DebugDumper * d, const char * name, const float * d
for (int i = 0; i < ndims; i++) {
numel *= shape[i];
}
FILE * f = fopen(path, "wb");
FILE * f = utf8_fopen(path, "wb");
if (!f) {
fprintf(stderr, "[Debug] cannot write %s\n", path);
return;
src/gguf-weights.h
+5 -4
View File
@@ -23,8 +23,7 @@
#include <vector>
#ifdef _WIN32
# define NOMINMAX
# include <windows.h>
# include "utf8.h"
#else
# include <fcntl.h>
# include <sys/mman.h>
@@ -79,7 +78,9 @@ static bool gf_load(GGUFModel * gf, const char * path) {
// mmap the file
#ifdef _WIN32
gf->fh = CreateFileA(path, GENERIC_READ, FILE_SHARE_READ, NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL);
std::wstring wpath = utf8_to_wide(path);
gf->fh =
CreateFileW(wpath.c_str(), GENERIC_READ, FILE_SHARE_READ, NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL);
if (gf->fh == INVALID_HANDLE_VALUE) {
fprintf(stderr, "[GGUF] Cannot open %s\n", path);
return false;
@@ -87,7 +88,7 @@ static bool gf_load(GGUFModel * gf, const char * path) {
LARGE_INTEGER li;
GetFileSizeEx(gf->fh, &li);
gf->file_size = (size_t) li.QuadPart;
gf->mh = CreateFileMappingA(gf->fh, NULL, PAGE_READONLY, 0, 0, NULL);
gf->mh = CreateFileMappingW(gf->fh, NULL, PAGE_READONLY, 0, 0, NULL);
if (!gf->mh) {
CloseHandle(gf->fh);
fprintf(stderr, "[GGUF] CreateFileMapping failed %s\n", path);
src/utf8.h
+88
View File
@@ -0,0 +1,88 @@
#pragma once
// utf8.h: portable UTF-8 boundary for Windows. Inside the project everything
// is UTF-8 ; this header bridges to the Windows-native UTF-16 APIs at the
// three places where the OS forces a recode: argv (CRT decodes from CP_ACP),
// fopen (CRT does too), and any direct Win32 *A call (CreateFileA etc.).
// POSIX is UTF-8 by convention and every helper degrades to a passthrough.
#include <cstdio>
#include <cstdlib>
#if defined(_WIN32)
# define WIN32_LEAN_AND_MEAN
# define NOMINMAX
// clang-format off
// windows.h must precede shellapi.h: shellapi.h pulls in types (HDROP,
// DECLSPEC_IMPORT, ...) declared by windows.h. SortIncludes would alphabetise
// the pair and break the build, so the off/on pragmas pin the order.
# include <windows.h>
# include <shellapi.h>
// clang-format on
# include <string>
#endif
#if defined(_WIN32)
// Decode a UTF-8 C string to a wide string (UTF-16). Empty on null or empty
// input. Used by every Win32 path-taking call site (CreateFileW, MoveFileW
// and friends) wherever the project still needs the raw wide form.
static std::wstring utf8_to_wide(const char * s) {
if (!s || !*s) {
return std::wstring();
}
int n = MultiByteToWideChar(CP_UTF8, 0, s, -1, NULL, 0);
if (n <= 1) {
return std::wstring();
}
std::wstring w((size_t) (n - 1), L'\0');
MultiByteToWideChar(CP_UTF8, 0, s, -1, &w[0], n);
return w;
}
#endif
// First line of main(). Rebuilds argv as UTF-8 from the wide command line
// and switches console I/O to CP_UTF8 so fprintf with non-ASCII bytes
// renders correctly without an explicit chcp 65001. POSIX no-op.
//
// The replacement argv is allocated once and leaked for the process
// lifetime: the OS reclaims it on exit, and the top of main is the only
// site that would ever free it.
static void utf8_init(int * argc, char *** argv) {
#if defined(_WIN32)
SetConsoleOutputCP(CP_UTF8);
SetConsoleCP(CP_UTF8);
int wargc = 0;
LPWSTR * wargv = CommandLineToArgvW(GetCommandLineW(), &wargc);
if (!wargv) {
return;
}
char ** new_argv = (char **) malloc(sizeof(char *) * (size_t) (wargc + 1));
for (int i = 0; i < wargc; i++) {
int n = WideCharToMultiByte(CP_UTF8, 0, wargv[i], -1, NULL, 0, NULL, NULL);
new_argv[i] = (char *) malloc((size_t) n);
WideCharToMultiByte(CP_UTF8, 0, wargv[i], -1, new_argv[i], n, NULL, NULL);
}
new_argv[wargc] = NULL;
LocalFree(wargv);
*argc = wargc;
*argv = new_argv;
#else
(void) argc;
(void) argv;
#endif
}
// UTF-8 aware fopen. POSIX passthrough. Windows decodes the UTF-8 path and
// mode to UTF-16 and calls _wfopen, the only fopen variant on MSVC that
// bypasses CP_ACP.
static FILE * utf8_fopen(const char * path, const char * mode) {
#if defined(_WIN32)
std::wstring wpath = utf8_to_wide(path);
std::wstring wmode = utf8_to_wide(mode);
return _wfopen(wpath.c_str(), wmode.c_str());
#else
return fopen(path, mode);
#endif
}
tools/quantize.cpp
+7 -3
View File
@@ -25,6 +25,7 @@
#include "ggml.h"
#include "gguf.h"
#include "utf8.h"
#include "version.h"
// Quant variant: base type + optional bump rules for important tensors
@@ -227,6 +228,7 @@ static bool to_f32(const void * src, float * dst, int64_t n, enum ggml_type type
}
int main(int argc, char ** argv) {
utf8_init(&argc, &argv);
if (argc != 4) {
fprintf(stderr, "qwentts.cpp %s\n\n", QWEN_VERSION);
fprintf(stderr, "Usage: %s <input.gguf> <output.gguf> <type>\n", argv[0]);
@@ -251,12 +253,14 @@ int main(int argc, char ** argv) {
// Mmap input file
#ifdef _WIN32
HANDLE fh = CreateFileA(inp_path, GENERIC_READ, FILE_SHARE_READ, NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL);
std::wstring winp_path = utf8_to_wide(inp_path);
HANDLE fh =
CreateFileW(winp_path.c_str(), GENERIC_READ, FILE_SHARE_READ, NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL);
if (fh == INVALID_HANDLE_VALUE) {
fprintf(stderr, "[Quantize] Failed to open %s\n", inp_path);
return 1;
}
HANDLE mh = CreateFileMappingA(fh, NULL, PAGE_READONLY, 0, 0, NULL);
HANDLE mh = CreateFileMappingW(fh, NULL, PAGE_READONLY, 0, 0, NULL);
if (!mh) {
fprintf(stderr, "[Quantize] CreateFileMapping failed %s\n", inp_path);
CloseHandle(fh);
@@ -396,7 +400,7 @@ int main(int argc, char ** argv) {
}
// Stream tensor data one at a time (low memory)
FILE * fout = fopen(out_path, "ab");
FILE * fout = utf8_fopen(out_path, "ab");
if (!fout) {
fprintf(stderr, "[Quantize] Failed to open %s for append\n", out_path);
return 1;
tools/qwen-codec.cpp
+4 -2
View File
@@ -13,6 +13,7 @@
#include "audio-io.h"
#include "backend.h"
#include "pipeline-codec.h"
#include "utf8.h"
#include "version.h"
#include <cstdint>
@@ -82,7 +83,7 @@ static std::vector<uint8_t> pack_codes(const std::vector<int32_t> & codes) {
// Read a .rvq file and unpack it into K*T codes. T is inferred from the
// file size: T = (filesize * 8) / (K * QWEN_TOKENIZER_CODE_BITS).
static bool read_rvq(const char * path, int K, std::vector<int32_t> & codes, int * n_frames) {
FILE * f = fopen(path, "rb");
FILE * f = utf8_fopen(path, "rb");
if (!f) {
fprintf(stderr, "[Codec] FATAL: cannot open %s\n", path);
return false;
@@ -117,7 +118,7 @@ static bool read_rvq(const char * path, int K, std::vector<int32_t> & codes, int
// Pack and write a .rvq file.
static bool write_rvq(const char * path, const std::vector<int32_t> & codes) {
std::vector<uint8_t> packed = pack_codes(codes);
FILE * f = fopen(path, "wb");
FILE * f = utf8_fopen(path, "wb");
if (!f) {
fprintf(stderr, "[Codec] FATAL: cannot open %s for write\n", path);
return false;
@@ -154,6 +155,7 @@ static int infer_mode(const char * path) {
}
int main(int argc, char ** argv) {
utf8_init(&argc, &argv);
if (argc <= 1) {
print_usage(argv[0]);
return 0;
tools/qwen-tts.cpp
+3 -1
View File
@@ -14,6 +14,7 @@
#include "backend.h"
#include "bpe.h"
#include "pipeline-tts.h"
#include "utf8.h"
#include "version.h"
#include <cstdio>
@@ -104,7 +105,7 @@ static std::string read_stdin_text() {
// Read a small text file into a string. Trims trailing newlines.
static bool read_text_file(const char * path, std::string & out) {
FILE * f = fopen(path, "rb");
FILE * f = utf8_fopen(path, "rb");
if (!f) {
fprintf(stderr, "[CLI] FATAL: cannot open '%s'\n", path);
return false;
@@ -400,6 +401,7 @@ static int run(const Args & a) {
}
int main(int argc, char ** argv) {
utf8_init(&argc, &argv);
Args a;
if (!parse_args(argc, argv, a)) {
print_usage(argv[0]);