diff --git a/README.md b/README.md index bf9ecc8..6ef0206 100644 --- a/README.md +++ b/README.md @@ -82,6 +82,22 @@ Voice cloning (`clone.sh`, Base, reference WAV plus its transcript) : --lang English -o out.wav < prompt.txt ``` +Pre-encoded reference (`clone.sh`): `qwen-codec --talker` encodes a reference +WAV into two compact latents in one pass, the `.spk` speaker embedding and +the `.rvq` ICL codes, bit-identical to what the `--ref-wav` path computes +internally. Passing them via `--ref-spk` / `--ref-rvq` skips the speaker +encoder and the codec encode on every synthesis: + +``` +build/qwen-codec --model models/qwen-tokenizer-12hz-Q8_0.gguf \ + --talker models/qwen-talker-1.7b-base-Q8_0.gguf -i ref.wav +build/qwen-tts \ + --model models/qwen-talker-1.7b-base-Q8_0.gguf \ + --codec models/qwen-tokenizer-12hz-Q8_0.gguf \ + --ref-spk ref.spk --ref-rvq ref.rvq --ref-text ref.txt \ + --lang English -o out.wav < prompt.txt +``` + Named speaker (`customvoice.sh`, CustomVoice) : ``` diff --git a/examples/clone.cmd b/examples/clone.cmd index 57b591e..57b9754 100644 --- a/examples/clone.cmd +++ b/examples/clone.cmd @@ -5,7 +5,8 @@ set PATH=%~dp0..\build\Release;%PATH% qwen-tts.exe ^ --model ..\models\qwen-talker-1.7b-base-Q8_0.gguf ^ --codec ..\models\qwen-tokenizer-12hz-Q8_0.gguf ^ - --ref-wav freeman.wav ^ + --ref-spk freeman.spk ^ + --ref-rvq freeman.rvq ^ --ref-text freeman.txt ^ --lang English ^ -o clone.wav < prompt.txt diff --git a/examples/clone.sh b/examples/clone.sh index a98cf1c..2b934b2 100755 --- a/examples/clone.sh +++ b/examples/clone.sh @@ -5,7 +5,8 @@ set -eu ../build/qwen-tts \ --model ../models/qwen-talker-1.7b-base-Q8_0.gguf \ --codec ../models/qwen-tokenizer-12hz-Q8_0.gguf \ - --ref-wav freeman.wav \ + --ref-spk freeman.spk \ + --ref-rvq freeman.rvq \ --ref-text freeman.txt \ --lang English \ -o clone.wav < prompt.txt diff --git a/examples/freeman.rvq b/examples/freeman.rvq new file mode 100644 index 0000000..06d122f Binary files /dev/null and b/examples/freeman.rvq differ diff --git a/examples/freeman.spk b/examples/freeman.spk new file mode 100644 index 0000000..e57f209 Binary files /dev/null and b/examples/freeman.spk differ diff --git a/src/pipeline-tts.cpp b/src/pipeline-tts.cpp index e00ab42..4512004 100644 --- a/src/pipeline-tts.cpp +++ b/src/pipeline-tts.cpp @@ -376,14 +376,50 @@ qt_status pipeline_tts_synthesize(PipelineTTS * pt, const std::string speaker = params->speaker ? params->speaker : ""; const std::string ref_text = params->ref_text ? params->ref_text : ""; - // Voice clone mode A: if ref_audio_24k is given, run the speaker - // encoder on the pre-decoded mono buffer and feed the resulting - // embedding straight into the prompt builder. Mutually exclusive - // with --speaker. - const bool has_ref_audio = (params->ref_audio_24k != NULL) && (params->ref_n_samples > 0); + // ABI v2 latent reference fields. Callers compiled against ABI 1 + // never set them; the abi_version gate keeps their uninitialised + // tail bytes out of the read path. + const float * lat_spk_emb = (params->abi_version >= 2) ? params->ref_spk_emb : NULL; + const int lat_spk_dim = (params->abi_version >= 2) ? params->ref_spk_dim : 0; + const int32_t * lat_codes = (params->abi_version >= 2) ? params->ref_codes : NULL; + const int lat_T = (params->abi_version >= 2) ? params->ref_T : 0; + + const bool has_ref_audio = (params->ref_audio_24k != NULL) && (params->ref_n_samples > 0); + const bool has_lat_spk = (lat_spk_emb != NULL) && (lat_spk_dim > 0); + const bool has_lat_codes = (lat_codes != NULL) && (lat_T > 0); + + // Raw waveform and pre-encoded latents are mutually exclusive: the + // caller is told immediately rather than picking a winner silently. + if (has_ref_audio && (has_lat_spk || has_lat_codes)) { + qt_set_error("pipeline_tts_synthesize: ref_audio_24k and ref_spk_emb / ref_codes are mutually exclusive"); + qt_log(QT_LOG_ERROR, "[Pipeline] ref_audio_24k and ref_spk_emb / ref_codes are mutually exclusive"); + return QT_STATUS_INVALID_PARAMS; + } + // Latent ICL codes ride on top of the speaker embedding and need the + // transcript, mirroring the raw path where mode B implies mode A. + if (has_lat_codes && (!has_lat_spk || ref_text.empty())) { + qt_set_error("pipeline_tts_synthesize: ref_codes requires ref_spk_emb and ref_text"); + qt_log(QT_LOG_ERROR, "[Pipeline] ref_codes requires ref_spk_emb and ref_text"); + return QT_STATUS_INVALID_PARAMS; + } + + // Voice clone mode A: a pre-extracted latent embedding feeds the + // prompt builder directly; otherwise, if ref_audio_24k is given, run + // the speaker encoder on the pre-decoded mono buffer. Mutually + // exclusive with --speaker. std::vector ref_spk_emb; const float * ref_spk_emb_ptr = NULL; - if (has_ref_audio) { + if (has_lat_spk) { + if (lat_spk_dim != pt->talker.hidden_size) { + qt_set_error("pipeline_tts_synthesize: ref_spk_dim %d mismatches talker hidden %d", lat_spk_dim, + pt->talker.hidden_size); + qt_log(QT_LOG_ERROR, "[Pipeline] ref_spk_dim %d mismatches talker hidden %d", lat_spk_dim, + pt->talker.hidden_size); + return QT_STATUS_INVALID_PARAMS; + } + ref_spk_emb_ptr = lat_spk_emb; + qt_log(QT_LOG_INFO, "[Pipeline] Latent speaker embedding: %d values", lat_spk_dim); + } else if (has_ref_audio) { if (!pt->has_speaker_encoder) { qt_set_error( "pipeline_tts_synthesize: --ref-wav requires a model with a loaded speaker encoder (Base only)"); @@ -404,17 +440,22 @@ qt_status pipeline_tts_synthesize(PipelineTTS * pt, ref_spk_emb_ptr = ref_spk_emb.data(); } - // Voice clone mode B: if ref_text is also given, encode the - // reference audio into 16 codebook indices via the codec encoder. - // Layout returned by pipeline_codec_encode is [num_codebooks, T_codec] - // row major, matching what the prompt builder expects for the ICL - // sum loop. + // Voice clone mode B: pre-encoded latent codes feed the ICL prompt + // directly; otherwise, if ref_text is given, encode the reference + // audio into 16 codebook indices via the codec encoder. Layout is + // [num_codebooks, T_codec] row major in both cases, matching what + // the prompt builder expects for the ICL sum loop. std::vector ref_codes; - int ref_codes_T = 0; - if (!ref_text.empty()) { + const int32_t * ref_codes_ptr = NULL; + int ref_codes_T = 0; + if (has_lat_codes) { + ref_codes_ptr = lat_codes; + ref_codes_T = lat_T; + qt_log(QT_LOG_INFO, "[Pipeline] Latent ICL ref_codes: %d frames at 12.5 Hz", ref_codes_T); + } else if (!ref_text.empty()) { if (!has_ref_audio) { - qt_set_error("pipeline_tts_synthesize: --ref-text requires --ref-wav"); - qt_log(QT_LOG_ERROR, "[Pipeline] --ref-text requires --ref-wav"); + qt_set_error("pipeline_tts_synthesize: ref_text requires ref_audio_24k or latent ref_codes"); + qt_log(QT_LOG_ERROR, "[Pipeline] ref_text requires ref_audio_24k or latent ref_codes"); return QT_STATUS_INVALID_PARAMS; } // The codec hop is 1920 samples at 24 kHz so n_samples must be @@ -431,7 +472,8 @@ qt_status pipeline_tts_synthesize(PipelineTTS * pt, qt_log(QT_LOG_ERROR, "[Pipeline] pipeline_codec_encode returned empty codes"); return QT_STATUS_GENERATE_FAILED; } - ref_codes_T = (int) ref_codes.size() / pt->num_code_groups; + ref_codes_ptr = ref_codes.data(); + ref_codes_T = (int) ref_codes.size() / pt->num_code_groups; qt_log(QT_LOG_INFO, "[Pipeline] ICL ref_codes: %d frames at 12.5 Hz (%d audio samples)", ref_codes_T, aligned_T); } @@ -444,8 +486,8 @@ qt_status pipeline_tts_synthesize(PipelineTTS * pt, const char * lang = params->lang ? params->lang : "auto"; Timer t_build; - if (!prompt_builder_build(pt, tok, params->text, lang, instruct, speaker, ref_spk_emb_ptr, ref_text, - ref_codes_T > 0 ? ref_codes.data() : NULL, ref_codes_T, &prompt)) { + if (!prompt_builder_build(pt, tok, params->text, lang, instruct, speaker, ref_spk_emb_ptr, ref_text, ref_codes_ptr, + ref_codes_T, &prompt)) { return QT_STATUS_GENERATE_FAILED; } perf.build_ms = t_build.ms(); @@ -469,7 +511,7 @@ qt_status pipeline_tts_synthesize(PipelineTTS * pt, } if (ref_codes_T > 0) { const int shape[2] = { pt->num_code_groups, ref_codes_T }; - debug_dump_i32_as_f32(&d, "ref-codes", ref_codes.data(), shape, 2); + debug_dump_i32_as_f32(&d, "ref-codes", ref_codes_ptr, shape, 2); } } diff --git a/src/qwen.cpp b/src/qwen.cpp index 038622a..0bc3615 100644 --- a/src/qwen.cpp +++ b/src/qwen.cpp @@ -218,6 +218,18 @@ void qt_tts_default_params(struct qt_tts_params * p) { p->on_chunk_user_data = nullptr; p->codec_chunk_sec = 24.0f; p->codec_left_context_sec = 2.0f; + p->ref_spk_emb = nullptr; + p->ref_spk_dim = 0; + p->ref_codes = nullptr; + p->ref_T = 0; +} + +int qt_num_codebooks(const struct qt_context * q) { + if (!q) { + qt_set_error("qt_num_codebooks: q is NULL"); + return 0; + } + return q->pt.num_code_groups; } struct qt_context * qt_init(const struct qt_init_params * params) { @@ -356,22 +368,26 @@ enum qt_status qt_synthesize(struct qt_context * q, const struct qt_tts_params * } return QT_STATUS_MODE_INVALID; } - if (params->ref_audio_24k && mt != "base") { - qt_set_error("--ref-wav is only valid for base models (loaded: %s)", mt.c_str()); + // ABI v2 latent reference fields, same gate as the pipeline. + const bool has_lat_spk = params->abi_version >= 2 && params->ref_spk_emb && params->ref_spk_dim > 0; + const bool has_lat_codes = params->abi_version >= 2 && params->ref_codes && params->ref_T > 0; + + if ((params->ref_audio_24k || has_lat_spk) && mt != "base") { + qt_set_error("--ref-wav / --ref-spk is only valid for base models (loaded: %s)", mt.c_str()); if (out) { qt_audio_free(out); } return QT_STATUS_MODE_INVALID; } - if (params->speaker && params->ref_audio_24k) { - qt_set_error("--speaker and --ref-wav are mutually exclusive"); + if (params->speaker && (params->ref_audio_24k || has_lat_spk)) { + qt_set_error("--speaker and --ref-wav / --ref-spk are mutually exclusive"); if (out) { qt_audio_free(out); } return QT_STATUS_INVALID_PARAMS; } - if (params->ref_text && !params->ref_audio_24k) { - qt_set_error("--ref-text requires --ref-wav"); + if (params->ref_text && !params->ref_audio_24k && !has_lat_codes) { + qt_set_error("--ref-text requires --ref-wav or --ref-rvq"); if (out) { qt_audio_free(out); } diff --git a/src/qwen.h b/src/qwen.h index 88d78ec..80d55d9 100644 --- a/src/qwen.h +++ b/src/qwen.h @@ -57,7 +57,7 @@ extern "C" { // git short hash + commit date string returned by qt_version(); for // binding compat checks, QT_ABI_VERSION is the only number that // matters. -#define QT_ABI_VERSION 1 +#define QT_ABI_VERSION 2 // Returns a static string of the form " ()" identifying // the exact commit this binary was built from. Safe to call from any @@ -269,6 +269,19 @@ struct qt_tts_params { // clamps to >= 0 frames. float codec_chunk_sec; float codec_left_context_sec; + + // ABI v2. Pre-encoded voice reference, the latent counterpart of + // ref_audio_24k. ref_spk_emb is the speaker embedding produced by + // the speaker encoder (ref_spk_dim f32 values, must equal the + // talker hidden size). ref_codes is the ICL code matrix produced + // by the codec encoder, [num_codebooks, ref_T] row-major. + // ref_spk_emb alone selects clone mode A; ref_spk_emb + ref_codes + // + ref_text selects mode B, mirroring the raw constraints. + // Mutually exclusive with ref_audio_24k and speaker. + const float * ref_spk_emb; + int ref_spk_dim; + const int32_t * ref_codes; + int ref_T; }; // Initialise to the standard defaults. Strings NULL, seed -1, @@ -278,6 +291,12 @@ struct qt_tts_params { // codec_left_context_sec 2.0. QT_API void qt_tts_default_params(struct qt_tts_params * p); +// Number of RVQ codebooks (K) of the loaded codec. Pre-encoded ICL +// reference codes passed via ref_codes are laid out [K, ref_T] +// row-major; callers reading a packed .rvq stream need K to derive +// ref_T from the code count. Returns 0 on a NULL handle. +QT_API int qt_num_codebooks(const struct qt_context * q); + // Run the full TTS synthesis. Validates the params against the loaded // model_type (the seven base / custom_voice / voice_design rules), // resolves the seed, hands off to pipeline_tts_synthesize and fills diff --git a/src/rvq-file.h b/src/rvq-file.h new file mode 100644 index 0000000..b152db7 --- /dev/null +++ b/src/rvq-file.h @@ -0,0 +1,111 @@ +#pragma once +// rvq-file.h: packed RVQ code stream file IO (.rvq). +// +// Flat code stream packed at code_bits per code, LSB-first, no header. +// Layout is [K, T] row-major. K and code_bits are fixed by the codec +// config in the GGUF; T is derived from the file size: +// T = (filesize * 8) / (K * code_bits). + +#include "utf8.h" + +#include +#include +#include +#include + +// Pack a flat code stream into code_bits-per-code, LSB-first. Output size +// is ceil(N * code_bits / 8) bytes. +static std::vector rvq_pack_codes(const std::vector & codes, int code_bits) { + const uint32_t mask = (1u << code_bits) - 1u; + const size_t total_bits = codes.size() * (size_t) code_bits; + std::vector out((total_bits + 7) / 8, 0); + + uint64_t acc = 0; + int bits_in_acc = 0; + size_t out_pos = 0; + for (size_t i = 0; i < codes.size(); i++) { + acc |= ((uint64_t) ((uint32_t) codes[i] & mask)) << bits_in_acc; + bits_in_acc += code_bits; + while (bits_in_acc >= 8) { + out[out_pos++] = (uint8_t) (acc & 0xFF); + acc >>= 8; + bits_in_acc -= 8; + } + } + if (bits_in_acc > 0) { + out[out_pos++] = (uint8_t) (acc & 0xFF); + } + return out; +} + +// Symmetric unpack: reads N codes from packed bytes. +static std::vector rvq_unpack_codes(const std::vector & in, size_t n_codes, int code_bits) { + const uint32_t mask = (1u << code_bits) - 1u; + std::vector out(n_codes); + + uint64_t acc = 0; + int bits_in_acc = 0; + size_t in_pos = 0; + for (size_t i = 0; i < n_codes; i++) { + while (bits_in_acc < code_bits && in_pos < in.size()) { + acc |= ((uint64_t) in[in_pos++]) << bits_in_acc; + bits_in_acc += 8; + } + out[i] = (int32_t) (acc & mask); + acc >>= code_bits; + bits_in_acc -= code_bits; + } + return out; +} + +// Read a .rvq file and unpack it into K*T codes. T is inferred from the +// file size. +static bool rvq_read_file(const char * path, int K, int code_bits, std::vector & codes, int * n_frames) { + FILE * f = utf8_fopen(path, "rb"); + if (!f) { + fprintf(stderr, "[RVQ] FATAL: cannot open %s\n", path); + return false; + } + fseek(f, 0, SEEK_END); + long sz = ftell(f); + fseek(f, 0, SEEK_SET); + if (sz <= 0) { + fprintf(stderr, "[RVQ] FATAL: %s is empty\n", path); + fclose(f); + return false; + } + std::vector buf((size_t) sz); + if (fread(buf.data(), 1, buf.size(), f) != buf.size()) { + fprintf(stderr, "[RVQ] FATAL: short read on %s\n", path); + fclose(f); + return false; + } + fclose(f); + + const size_t total_bits = (size_t) sz * 8; + const size_t n_codes = total_bits / (size_t) code_bits; + if (n_codes == 0 || (n_codes % (size_t) K) != 0) { + fprintf(stderr, "[RVQ] FATAL: %s yields %zu codes, not a multiple of K=%d\n", path, n_codes, K); + return false; + } + codes = rvq_unpack_codes(buf, n_codes, code_bits); + *n_frames = (int) (n_codes / (size_t) K); + return true; +} + +// Pack and write a .rvq file. +static bool rvq_write_file(const char * path, const std::vector & codes, int code_bits) { + std::vector packed = rvq_pack_codes(codes, code_bits); + FILE * f = utf8_fopen(path, "wb"); + if (!f) { + fprintf(stderr, "[RVQ] FATAL: cannot open %s for write\n", path); + return false; + } + if (fwrite(packed.data(), 1, packed.size(), f) != packed.size()) { + fprintf(stderr, "[RVQ] FATAL: short write on %s\n", path); + fclose(f); + return false; + } + fclose(f); + return true; +} diff --git a/tools/qwen-codec.cpp b/tools/qwen-codec.cpp index 3aeb15f..1599da7 100644 --- a/tools/qwen-codec.cpp +++ b/tools/qwen-codec.cpp @@ -5,6 +5,13 @@ // file extension: .wav in -> encode, .rvq in -> decode. Output is // auto-named next to the input file by swapping the extension. // +// Encode truncates the input to the hop boundary, strictly conforming +// to the qwen-tts --ref-wav ICL path, so a .rvq produced here feeds +// qwen-tts --ref-rvq directly. Passing --talker additionally runs the +// speaker encoder from the talker GGUF on the full input and writes +// the x-vector embedding next to the .rvq as a .spk file (raw f32, +// enc_dim values), feeding qwen-tts --ref-spk. +// // File format (.rvq): flat code stream packed at 11 bits per code, // LSB-first, no header. Layout is [K, T] row-major. K is fixed by the // codec config in the GGUF (16 codebooks for the 12Hz tokenizer, @@ -12,7 +19,11 @@ #include "audio-io.h" #include "backend.h" +#include "gguf-weights.h" #include "pipeline-codec.h" +#include "rvq-file.h" +#include "speaker-encoder-extract.h" +#include "speaker-encoder-weights.h" #include "utf8.h" #include "version.h" @@ -23,115 +34,24 @@ #include #include -static const uint32_t RVQ_CODE_MASK = (1u << TOKENIZER_CODE_BITS) - 1u; - static void print_usage(const char * prog) { fprintf(stderr, "qwentts.cpp %s\n\n", QWEN_VERSION); fprintf(stderr, - "Usage: %s --model [-i ] [--format ]\n\n" + "Usage: %s --model [-i ] [--talker ] [--format ]\n\n" "Required:\n" " --model Codec GGUF (qwen-tokenizer-12hz-*.gguf)\n\n" "Optional:\n" " -i Input. WAV -> encode, .rvq -> decode\n" + " --talker Talker GGUF (Base only). Encode also extracts the speaker\n" + " embedding and writes it next to the .rvq as a .spk file\n" " --format WAV output format: wav16, wav24, wav32 (default: wav16)\n\n" "Output is auto-named next to input : clip.wav -> clip.rvq, clip.rvq -> clip.wav.\n" + "Encode truncates to the hop boundary, conforming to the qwen-tts --ref-wav path:\n" + "the .rvq feeds qwen-tts --ref-rvq, the .spk feeds qwen-tts --ref-spk.\n" "When -i is omitted, runs a load self-test of the codec GGUF.\n", prog); } -// Symmetric unpack: reads N codes from packed bytes (11 bits LSB-first). -static std::vector unpack_codes(const std::vector & in, size_t n_codes) { - std::vector out(n_codes); - uint64_t acc = 0; - int bits_in_acc = 0; - size_t in_pos = 0; - for (size_t i = 0; i < n_codes; i++) { - while (bits_in_acc < TOKENIZER_CODE_BITS && in_pos < in.size()) { - acc |= ((uint64_t) in[in_pos++]) << bits_in_acc; - bits_in_acc += 8; - } - out[i] = (int32_t) (acc & RVQ_CODE_MASK); - acc >>= TOKENIZER_CODE_BITS; - bits_in_acc -= TOKENIZER_CODE_BITS; - } - return out; -} - -// Pack flat int32 codes into 11-bit LSB-first packed bytes. Output size is -// ceil(N * 11 / 8) bytes. -static std::vector pack_codes(const std::vector & codes) { - const size_t total_bits = codes.size() * (size_t) TOKENIZER_CODE_BITS; - std::vector out((total_bits + 7) / 8, 0); - uint64_t acc = 0; - int bits_in_acc = 0; - size_t out_pos = 0; - for (size_t i = 0; i < codes.size(); i++) { - acc |= ((uint64_t) ((uint32_t) codes[i] & RVQ_CODE_MASK)) << bits_in_acc; - bits_in_acc += TOKENIZER_CODE_BITS; - while (bits_in_acc >= 8) { - out[out_pos++] = (uint8_t) (acc & 0xFF); - acc >>= 8; - bits_in_acc -= 8; - } - } - if (bits_in_acc > 0) { - out[out_pos++] = (uint8_t) (acc & 0xFF); - } - return out; -} - -// Read a .rvq file and unpack it into K*T codes. T is inferred from the -// file size: T = (filesize * 8) / (K * TOKENIZER_CODE_BITS). -static bool read_rvq(const char * path, int K, std::vector & codes, int * n_frames) { - FILE * f = utf8_fopen(path, "rb"); - if (!f) { - fprintf(stderr, "[Codec] FATAL: cannot open %s\n", path); - return false; - } - fseek(f, 0, SEEK_END); - long sz = ftell(f); - fseek(f, 0, SEEK_SET); - if (sz <= 0) { - fprintf(stderr, "[Codec] FATAL: %s is empty\n", path); - fclose(f); - return false; - } - std::vector buf((size_t) sz); - if (fread(buf.data(), 1, buf.size(), f) != buf.size()) { - fprintf(stderr, "[Codec] FATAL: short read on %s\n", path); - fclose(f); - return false; - } - fclose(f); - - const size_t total_bits = (size_t) sz * 8; - const size_t n_codes = total_bits / (size_t) TOKENIZER_CODE_BITS; - if (n_codes == 0 || (n_codes % (size_t) K) != 0) { - fprintf(stderr, "[Codec] FATAL: %s yields %zu codes, not a multiple of K=%d\n", path, n_codes, K); - return false; - } - codes = unpack_codes(buf, n_codes); - *n_frames = (int) (n_codes / (size_t) K); - return true; -} - -// Pack and write a .rvq file. -static bool write_rvq(const char * path, const std::vector & codes) { - std::vector packed = pack_codes(codes); - FILE * f = utf8_fopen(path, "wb"); - if (!f) { - fprintf(stderr, "[Codec] FATAL: cannot open %s for write\n", path); - return false; - } - if (fwrite(packed.data(), 1, packed.size(), f) != packed.size()) { - fprintf(stderr, "[Codec] FATAL: short write on %s\n", path); - fclose(f); - return false; - } - fclose(f); - return true; -} - // Replace or append extension on a path string. static std::string swap_ext(const std::string & path, const char * ext) { size_t dot = path.find_last_of('.'); @@ -154,6 +74,60 @@ static int infer_mode(const char * path) { return 0; } +// Load the speaker encoder from the talker GGUF, run it on the full input +// buffer and write the embedding as a raw f32 .spk file (enc_dim values, +// validated by filesize on the qwen-tts side). Returns 0 on success. +static int extract_spk(const char * talker_path, + BackendPair bp, + const float * audio, + int n_samples, + const char * out_path) { + GGUFModel gf = {}; + if (!gf_load(&gf, talker_path)) { + fprintf(stderr, "[Codec] FATAL: cannot open talker GGUF %s\n", talker_path); + return 1; + } + + SpeakerEncoderWeights sw = {}; + if (!speaker_encoder_weights_load(&sw, gf, bp.backend)) { + fprintf(stderr, "[Codec] FATAL: speaker encoder load failed from %s\n", talker_path); + gf_close(&gf); + return 1; + } + gf_close(&gf); + if (sw.weight_buf == NULL) { + fprintf(stderr, "[Codec] FATAL: %s has no speaker encoder (Base only)\n", talker_path); + return 1; + } + + ggml_backend_sched_t sched = backend_sched_new(bp, 4096); + const int enc_dim = sw.enc_dim; + std::vector emb; + bool ok = speaker_encoder_extract(&sw, sched, audio, n_samples, emb); + ggml_backend_sched_free(sched); + speaker_encoder_weights_free(&sw); + if (!ok || (int) emb.size() != enc_dim) { + fprintf(stderr, "[Codec] FATAL: speaker embedding extraction failed (%zu values, enc_dim %d)\n", emb.size(), + enc_dim); + return 1; + } + + FILE * f = utf8_fopen(out_path, "wb"); + if (!f) { + fprintf(stderr, "[Codec] FATAL: cannot open %s for write\n", out_path); + return 1; + } + if (fwrite(emb.data(), sizeof(float), emb.size(), f) != emb.size()) { + fprintf(stderr, "[Codec] FATAL: short write on %s\n", out_path); + fclose(f); + return 1; + } + fclose(f); + + fprintf(stderr, "[Codec] Wrote %s: %zu f32 values (%zu bytes)\n", out_path, emb.size(), emb.size() * sizeof(float)); + return 0; +} + int main(int argc, char ** argv) { utf8_init(&argc, &argv); if (argc <= 1) { @@ -161,13 +135,16 @@ int main(int argc, char ** argv) { return 0; } - const char * model_path = NULL; - const char * input_path = NULL; - WavFormat wav_fmt = WAV_S16; + const char * model_path = NULL; + const char * input_path = NULL; + const char * talker_path = NULL; + WavFormat wav_fmt = WAV_S16; for (int i = 1; i < argc; i++) { if (strcmp(argv[i], "--model") == 0 && i + 1 < argc) { model_path = argv[++i]; + } else if (strcmp(argv[i], "--talker") == 0 && i + 1 < argc) { + talker_path = argv[++i]; } else if (strcmp(argv[i], "-i") == 0 && i + 1 < argc) { input_path = argv[++i]; } else if (strcmp(argv[i], "--format") == 0 && i + 1 < argc) { @@ -217,40 +194,46 @@ int main(int argc, char ** argv) { if (!input_path) { fprintf(stderr, "[Codec] Load self-test passed\n"); } else if (mode == 1) { - // Encode .wav -> .rvq + // Encode .wav -> .rvq (+ .spk with --talker) const std::string out_str = swap_ext(input_path, ".rvq"); int T_in = 0; float * audio_in = audio_read_mono(input_path, TOKENIZER_SAMPLE_RATE, &T_in); - if (!audio_in || T_in <= 0) { - fprintf(stderr, "[Codec] FATAL: cannot read %s\n", input_path); + if (!audio_in || T_in < TOKENIZER_HOP_LENGTH) { + fprintf(stderr, "[Codec] FATAL: cannot read %s or input shorter than one hop (%d samples)\n", input_path, + TOKENIZER_HOP_LENGTH); free(audio_in); rc = 1; } else { - // Pad to a multiple of HOP_LENGTH so the RVQ frame count is integral. - int hop = TOKENIZER_HOP_LENGTH; - int T_padded = ((T_in + hop - 1) / hop) * hop; - int T_frames = T_padded / hop; + // Truncate to a multiple of HOP_LENGTH, strictly conforming to + // the qwen-tts --ref-wav ICL path. + int hop = TOKENIZER_HOP_LENGTH; + int T_aligned = (T_in / hop) * hop; + int T_frames = T_aligned / hop; - std::vector audio_buf((size_t) T_padded, 0.0f); - memcpy(audio_buf.data(), audio_in, (size_t) T_in * sizeof(float)); - free(audio_in); + fprintf(stderr, "[Codec] Encode: %s, %d samples @ %d Hz, truncated to %d (%d frames @ 12.5 Hz, %.2f s)\n", + input_path, T_in, TOKENIZER_SAMPLE_RATE, T_aligned, T_frames, + (double) T_aligned / (double) TOKENIZER_SAMPLE_RATE); - fprintf(stderr, "[Codec] Encode: %s, %d samples @ %d Hz, padded to %d (%d frames @ 12.5 Hz, %.2f s)\n", - input_path, T_in, TOKENIZER_SAMPLE_RATE, T_padded, T_frames, - (double) T_padded / (double) TOKENIZER_SAMPLE_RATE); - - std::vector codes = pipeline_codec_encode(&pc, audio_buf.data(), T_padded); + std::vector codes = pipeline_codec_encode(&pc, audio_in, T_aligned); if (codes.empty()) { fprintf(stderr, "[Codec] FATAL: encode failed\n"); rc = 1; - } else if (!write_rvq(out_str.c_str(), codes)) { + } else if (!rvq_write_file(out_str.c_str(), codes, TOKENIZER_CODE_BITS)) { rc = 1; } else { fprintf(stderr, "[Codec] Wrote %s: K=%d T=%d, %zu codes -> %zu packed bytes\n", out_str.c_str(), TOKENIZER_NUM_CODEBOOKS, T_frames, codes.size(), (codes.size() * (size_t) TOKENIZER_CODE_BITS + 7) / 8); } + + // Speaker embedding extraction, conforming to the qwen-tts + // --ref-wav mode A path: the encoder consumes the FULL input + // buffer, never the hop-truncated one. + if (rc == 0 && talker_path) { + rc = extract_spk(talker_path, bp, audio_in, T_in, swap_ext(input_path, ".spk").c_str()); + } + free(audio_in); } } else { // Decode .rvq -> .wav @@ -258,7 +241,7 @@ int main(int argc, char ** argv) { std::vector codes; int T = 0; - if (!read_rvq(input_path, TOKENIZER_NUM_CODEBOOKS, codes, &T)) { + if (!rvq_read_file(input_path, TOKENIZER_NUM_CODEBOOKS, TOKENIZER_CODE_BITS, codes, &T)) { rc = 1; } else { fprintf(stderr, "[Codec] Decode: %s, K=%d T=%d (%.2f s)\n", input_path, TOKENIZER_NUM_CODEBOOKS, T, diff --git a/tools/qwen-tts.cpp b/tools/qwen-tts.cpp index 95ec2c2..f07f653 100644 --- a/tools/qwen-tts.cpp +++ b/tools/qwen-tts.cpp @@ -13,6 +13,7 @@ #include "audio-io.h" #include "qwen.h" +#include "rvq-file.h" #include #include @@ -40,6 +41,10 @@ static void print_usage(const char * prog) { " CustomVoice, rejected for Base\n" " --speaker Speaker name (CustomVoice only)\n" " --ref-wav Reference WAV for voice cloning (Base only)\n" + " --ref-spk Pre-extracted speaker embedding from qwen-codec --talker\n" + " (replaces --ref-wav, Base only)\n" + " --ref-rvq Pre-encoded reference codes from qwen-codec (requires\n" + " --ref-spk and --ref-text, enables ICL clone mode)\n" " --ref-text Transcript file for the reference (enables ICL clone mode)\n" " --max-new Max new audio frames (default: 2048)\n" " --codec-chunk-dur Codec decode chunk duration in seconds (default: 24.0)\n" @@ -69,6 +74,8 @@ struct Args { const char * instruct; const char * speaker; const char * ref_wav; + const char * ref_spk; + const char * ref_rvq; const char * ref_text_path; const char * dump_dir; const char * out_wav; @@ -104,6 +111,34 @@ static std::string read_stdin_text() { } // Read a small text file into a string. Trims trailing newlines. +// 11 bits per code (V <= 2048), matching qwen-codec. +static const int RVQ_CODE_BITS = 11; + +// Read a .spk file: raw f32 values, the count IS the embedding dimension. +static bool read_spk_file(const char * path, std::vector & emb) { + FILE * f = utf8_fopen(path, "rb"); + if (!f) { + fprintf(stderr, "[CLI] ERROR: cannot open --ref-spk '%s'\n", path); + return false; + } + fseek(f, 0, SEEK_END); + long sz = ftell(f); + fseek(f, 0, SEEK_SET); + if (sz <= 0 || (sz % (long) sizeof(float)) != 0) { + fprintf(stderr, "[CLI] ERROR: --ref-spk '%s' size %ld is not a positive multiple of 4\n", path, sz); + fclose(f); + return false; + } + emb.resize((size_t) sz / sizeof(float)); + if (fread(emb.data(), sizeof(float), emb.size(), f) != emb.size()) { + fprintf(stderr, "[CLI] ERROR: short read on --ref-spk '%s'\n", path); + fclose(f); + return false; + } + fclose(f); + return true; +} + static bool read_text_file(const char * path, std::string & out) { FILE * f = fopen(path, "rb"); if (!f) { @@ -168,6 +203,10 @@ static bool parse_args(int argc, char ** argv, Args & a) { a.speaker = argv[++i]; } else if (std::strcmp(arg, "--ref-wav") == 0 && i + 1 < argc) { a.ref_wav = argv[++i]; + } else if (std::strcmp(arg, "--ref-spk") == 0 && i + 1 < argc) { + a.ref_spk = argv[++i]; + } else if (std::strcmp(arg, "--ref-rvq") == 0 && i + 1 < argc) { + a.ref_rvq = argv[++i]; } else if (std::strcmp(arg, "--ref-text") == 0 && i + 1 < argc) { a.ref_text_path = argv[++i]; } else if (std::strcmp(arg, "--format") == 0 && i + 1 < argc) { @@ -278,6 +317,29 @@ static int run(const Args & a) { ref_n_samples = T_in; } + // Latent reference files. The .spk holds raw f32 values whose count + // IS the embedding dimension; the .rvq holds the packed ICL code + // matrix. The facade validates the structural constraints (mutual + // exclusions, dim match against the talker hidden size). + std::vector ref_spk_emb; + std::vector ref_codes; + int ref_T = 0; + if (a.ref_spk) { + if (!read_spk_file(a.ref_spk, ref_spk_emb)) { + qt_free(q); + return 1; + } + fprintf(stderr, "[CLI] Reference SPK: %s, %zu f32 values\n", a.ref_spk, ref_spk_emb.size()); + } + if (a.ref_rvq) { + const int K = qt_num_codebooks(q); + if (!rvq_read_file(a.ref_rvq, K, RVQ_CODE_BITS, ref_codes, &ref_T)) { + qt_free(q); + return 1; + } + fprintf(stderr, "[CLI] Reference RVQ: %s, K=%d T=%d\n", a.ref_rvq, K, ref_T); + } + // Resolve output WAV format string: wav16 / wav24 / wav32. Default // wav16 mirrors the omnivoice.cpp default. WavFormat wav_fmt; @@ -323,6 +385,10 @@ static int run(const Args & a) { params.ref_audio_24k = ref_audio_24k; params.ref_n_samples = ref_n_samples; params.ref_text = ref_text; + params.ref_spk_emb = ref_spk_emb.empty() ? NULL : ref_spk_emb.data(); + params.ref_spk_dim = (int) ref_spk_emb.size(); + params.ref_codes = ref_codes.empty() ? NULL : ref_codes.data(); + params.ref_T = ref_T; params.seed = a.seed; params.max_new_tokens = a.max_new_tokens; params.do_sample = a.do_sample;