enne2/qwentts.cpp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

codec: add pre-encoded voice reference (--ref-spk / --ref-rvq)

Browse Source 
qwen-codec --talker extracts the speaker embedding (.spk, raw f32)
and the ICL codes (.rvq) in one pass, encode truncated to the hop
boundary conforming to the --ref-wav path. qwen-tts loads them via
--ref-spk / --ref-rvq and skips the speaker encoder and codec encode
on every synthesis: TTFA 205 ms -> 89 ms. Extends qt_tts_params with
ABI v2 latent fields, adds qt_num_codebooks(), ships freeman.spk +
freeman.rvq and switches clone scripts to the latent path. Output is
bit-identical to the raw path at fixed seed.
This commit is contained in:
Pascal
2026-06-11 21:45:13 +02:00
parent e8e33629c1
commit 0bf4a18b22
11 changed files with 399 additions and 144 deletions
Download Patch File Download Diff File Expand all files Collapse all files
README.md
+16
View File
@@ -82,6 +82,22 @@ Voice cloning (`clone.sh`, Base, reference WAV plus its transcript) :
--lang English -o out.wav < prompt.txt
```
Pre-encoded reference (`clone.sh`): `qwen-codec --talker` encodes a reference
WAV into two compact latents in one pass, the `.spk` speaker embedding and
the `.rvq` ICL codes, bit-identical to what the `--ref-wav` path computes
internally. Passing them via `--ref-spk` / `--ref-rvq` skips the speaker
encoder and the codec encode on every synthesis:
```
build/qwen-codec --model models/qwen-tokenizer-12hz-Q8_0.gguf \
--talker models/qwen-talker-1.7b-base-Q8_0.gguf -i ref.wav
build/qwen-tts \
--model models/qwen-talker-1.7b-base-Q8_0.gguf \
--codec models/qwen-tokenizer-12hz-Q8_0.gguf \
--ref-spk ref.spk --ref-rvq ref.rvq --ref-text ref.txt \
--lang English -o out.wav < prompt.txt
```
Named speaker (`customvoice.sh`, CustomVoice) :
```
examples/clone.cmd
+2 -1
View File
@@ -5,7 +5,8 @@ set PATH=%~dp0..\build\Release;%PATH%
qwen-tts.exe ^
--model ..\models\qwen-talker-1.7b-base-Q8_0.gguf ^
--codec ..\models\qwen-tokenizer-12hz-Q8_0.gguf ^
--ref-wav freeman.wav ^
--ref-spk freeman.spk ^
--ref-rvq freeman.rvq ^
--ref-text freeman.txt ^
--lang English ^
-o clone.wav < prompt.txt
examples/clone.sh
+2 -1
View File
@@ -5,7 +5,8 @@ set -eu
../build/qwen-tts \
--model ../models/qwen-talker-1.7b-base-Q8_0.gguf \
--codec ../models/qwen-tokenizer-12hz-Q8_0.gguf \
--ref-wav freeman.wav \
--ref-spk freeman.spk \
--ref-rvq freeman.rvq \
--ref-text freeman.txt \
--lang English \
-o clone.wav < prompt.txt
examples/freeman.rvq
BIN
View File
Binary file not shown.
examples/freeman.spk
BIN
View File
Binary file not shown.
src/pipeline-tts.cpp
+61 -19
View File
@@ -376,14 +376,50 @@ qt_status pipeline_tts_synthesize(PipelineTTS * pt,
const std::string speaker = params->speaker ? params->speaker : "";
const std::string ref_text = params->ref_text ? params->ref_text : "";
// Voice clone mode A: if ref_audio_24k is given, run the speaker
// encoder on the pre-decoded mono buffer and feed the resulting
// embedding straight into the prompt builder. Mutually exclusive
// with --speaker.
const bool has_ref_audio = (params->ref_audio_24k != NULL) && (params->ref_n_samples > 0);
// ABI v2 latent reference fields. Callers compiled against ABI 1
// never set them; the abi_version gate keeps their uninitialised
// tail bytes out of the read path.
const float * lat_spk_emb = (params->abi_version >= 2) ? params->ref_spk_emb : NULL;
const int lat_spk_dim = (params->abi_version >= 2) ? params->ref_spk_dim : 0;
const int32_t * lat_codes = (params->abi_version >= 2) ? params->ref_codes : NULL;
const int lat_T = (params->abi_version >= 2) ? params->ref_T : 0;
const bool has_ref_audio = (params->ref_audio_24k != NULL) && (params->ref_n_samples > 0);
const bool has_lat_spk = (lat_spk_emb != NULL) && (lat_spk_dim > 0);
const bool has_lat_codes = (lat_codes != NULL) && (lat_T > 0);
// Raw waveform and pre-encoded latents are mutually exclusive: the
// caller is told immediately rather than picking a winner silently.
if (has_ref_audio && (has_lat_spk || has_lat_codes)) {
qt_set_error("pipeline_tts_synthesize: ref_audio_24k and ref_spk_emb / ref_codes are mutually exclusive");
qt_log(QT_LOG_ERROR, "[Pipeline] ref_audio_24k and ref_spk_emb / ref_codes are mutually exclusive");
return QT_STATUS_INVALID_PARAMS;
}
// Latent ICL codes ride on top of the speaker embedding and need the
// transcript, mirroring the raw path where mode B implies mode A.
if (has_lat_codes && (!has_lat_spk || ref_text.empty())) {
qt_set_error("pipeline_tts_synthesize: ref_codes requires ref_spk_emb and ref_text");
qt_log(QT_LOG_ERROR, "[Pipeline] ref_codes requires ref_spk_emb and ref_text");
return QT_STATUS_INVALID_PARAMS;
}
// Voice clone mode A: a pre-extracted latent embedding feeds the
// prompt builder directly; otherwise, if ref_audio_24k is given, run
// the speaker encoder on the pre-decoded mono buffer. Mutually
// exclusive with --speaker.
std::vector<float> ref_spk_emb;
const float * ref_spk_emb_ptr = NULL;
if (has_ref_audio) {
if (has_lat_spk) {
if (lat_spk_dim != pt->talker.hidden_size) {
qt_set_error("pipeline_tts_synthesize: ref_spk_dim %d mismatches talker hidden %d", lat_spk_dim,
pt->talker.hidden_size);
qt_log(QT_LOG_ERROR, "[Pipeline] ref_spk_dim %d mismatches talker hidden %d", lat_spk_dim,
pt->talker.hidden_size);
return QT_STATUS_INVALID_PARAMS;
}
ref_spk_emb_ptr = lat_spk_emb;
qt_log(QT_LOG_INFO, "[Pipeline] Latent speaker embedding: %d values", lat_spk_dim);
} else if (has_ref_audio) {
if (!pt->has_speaker_encoder) {
qt_set_error(
"pipeline_tts_synthesize: --ref-wav requires a model with a loaded speaker encoder (Base only)");
@@ -404,17 +440,22 @@ qt_status pipeline_tts_synthesize(PipelineTTS * pt,
ref_spk_emb_ptr = ref_spk_emb.data();
}
// Voice clone mode B: if ref_text is also given, encode the
// reference audio into 16 codebook indices via the codec encoder.
// Layout returned by pipeline_codec_encode is [num_codebooks, T_codec]
// row major, matching what the prompt builder expects for the ICL
// sum loop.
// Voice clone mode B: pre-encoded latent codes feed the ICL prompt
// directly; otherwise, if ref_text is given, encode the reference
// audio into 16 codebook indices via the codec encoder. Layout is
// [num_codebooks, T_codec] row major in both cases, matching what
// the prompt builder expects for the ICL sum loop.
std::vector<int32_t> ref_codes;
int ref_codes_T = 0;
if (!ref_text.empty()) {
const int32_t * ref_codes_ptr = NULL;
int ref_codes_T = 0;
if (has_lat_codes) {
ref_codes_ptr = lat_codes;
ref_codes_T = lat_T;
qt_log(QT_LOG_INFO, "[Pipeline] Latent ICL ref_codes: %d frames at 12.5 Hz", ref_codes_T);
} else if (!ref_text.empty()) {
if (!has_ref_audio) {
qt_set_error("pipeline_tts_synthesize: --ref-text requires --ref-wav");
qt_log(QT_LOG_ERROR, "[Pipeline] --ref-text requires --ref-wav");
qt_set_error("pipeline_tts_synthesize: ref_text requires ref_audio_24k or latent ref_codes");
qt_log(QT_LOG_ERROR, "[Pipeline] ref_text requires ref_audio_24k or latent ref_codes");
return QT_STATUS_INVALID_PARAMS;
}
// The codec hop is 1920 samples at 24 kHz so n_samples must be
@@ -431,7 +472,8 @@ qt_status pipeline_tts_synthesize(PipelineTTS * pt,
qt_log(QT_LOG_ERROR, "[Pipeline] pipeline_codec_encode returned empty codes");
return QT_STATUS_GENERATE_FAILED;
}
ref_codes_T = (int) ref_codes.size() / pt->num_code_groups;
ref_codes_ptr = ref_codes.data();
ref_codes_T = (int) ref_codes.size() / pt->num_code_groups;
qt_log(QT_LOG_INFO, "[Pipeline] ICL ref_codes: %d frames at 12.5 Hz (%d audio samples)", ref_codes_T,
aligned_T);
}
@@ -444,8 +486,8 @@ qt_status pipeline_tts_synthesize(PipelineTTS * pt,
const char * lang = params->lang ? params->lang : "auto";
Timer t_build;
if (!prompt_builder_build(pt, tok, params->text, lang, instruct, speaker, ref_spk_emb_ptr, ref_text,
ref_codes_T > 0 ? ref_codes.data() : NULL, ref_codes_T, &prompt)) {
if (!prompt_builder_build(pt, tok, params->text, lang, instruct, speaker, ref_spk_emb_ptr, ref_text, ref_codes_ptr,
ref_codes_T, &prompt)) {
return QT_STATUS_GENERATE_FAILED;
}
perf.build_ms = t_build.ms();
@@ -469,7 +511,7 @@ qt_status pipeline_tts_synthesize(PipelineTTS * pt,
}
if (ref_codes_T > 0) {
const int shape[2] = { pt->num_code_groups, ref_codes_T };
debug_dump_i32_as_f32(&d, "ref-codes", ref_codes.data(), shape, 2);
debug_dump_i32_as_f32(&d, "ref-codes", ref_codes_ptr, shape, 2);
}
}
src/qwen.cpp
+22 -6
View File
@@ -218,6 +218,18 @@ void qt_tts_default_params(struct qt_tts_params * p) {
p->on_chunk_user_data = nullptr;
p->codec_chunk_sec = 24.0f;
p->codec_left_context_sec = 2.0f;
p->ref_spk_emb = nullptr;
p->ref_spk_dim = 0;
p->ref_codes = nullptr;
p->ref_T = 0;
}
int qt_num_codebooks(const struct qt_context * q) {
if (!q) {
qt_set_error("qt_num_codebooks: q is NULL");
return 0;
}
return q->pt.num_code_groups;
}
struct qt_context * qt_init(const struct qt_init_params * params) {
@@ -356,22 +368,26 @@ enum qt_status qt_synthesize(struct qt_context * q, const struct qt_tts_params *
}
return QT_STATUS_MODE_INVALID;
}
if (params->ref_audio_24k && mt != "base") {
qt_set_error("--ref-wav is only valid for base models (loaded: %s)", mt.c_str());
// ABI v2 latent reference fields, same gate as the pipeline.
const bool has_lat_spk = params->abi_version >= 2 && params->ref_spk_emb && params->ref_spk_dim > 0;
const bool has_lat_codes = params->abi_version >= 2 && params->ref_codes && params->ref_T > 0;
if ((params->ref_audio_24k || has_lat_spk) && mt != "base") {
qt_set_error("--ref-wav / --ref-spk is only valid for base models (loaded: %s)", mt.c_str());
if (out) {
qt_audio_free(out);
}
return QT_STATUS_MODE_INVALID;
}
if (params->speaker && params->ref_audio_24k) {
qt_set_error("--speaker and --ref-wav are mutually exclusive");
if (params->speaker && (params->ref_audio_24k || has_lat_spk)) {
qt_set_error("--speaker and --ref-wav / --ref-spk are mutually exclusive");
if (out) {
qt_audio_free(out);
}
return QT_STATUS_INVALID_PARAMS;
}
if (params->ref_text && !params->ref_audio_24k) {
qt_set_error("--ref-text requires --ref-wav");
if (params->ref_text && !params->ref_audio_24k && !has_lat_codes) {
qt_set_error("--ref-text requires --ref-wav or --ref-rvq");
if (out) {
qt_audio_free(out);
}
src/qwen.h
+20 -1
View File
@@ -57,7 +57,7 @@ extern "C" {
// git short hash + commit date string returned by qt_version(); for
// binding compat checks, QT_ABI_VERSION is the only number that
// matters.
#define QT_ABI_VERSION 1
#define QT_ABI_VERSION 2
// Returns a static string of the form "<git-hash> (<date>)" identifying
// the exact commit this binary was built from. Safe to call from any
@@ -269,6 +269,19 @@ struct qt_tts_params {
// clamps to >= 0 frames.
float codec_chunk_sec;
float codec_left_context_sec;
// ABI v2. Pre-encoded voice reference, the latent counterpart of
// ref_audio_24k. ref_spk_emb is the speaker embedding produced by
// the speaker encoder (ref_spk_dim f32 values, must equal the
// talker hidden size). ref_codes is the ICL code matrix produced
// by the codec encoder, [num_codebooks, ref_T] row-major.
// ref_spk_emb alone selects clone mode A; ref_spk_emb + ref_codes
// + ref_text selects mode B, mirroring the raw constraints.
// Mutually exclusive with ref_audio_24k and speaker.
const float * ref_spk_emb;
int ref_spk_dim;
const int32_t * ref_codes;
int ref_T;
};
// Initialise to the standard defaults. Strings NULL, seed -1,
@@ -278,6 +291,12 @@ struct qt_tts_params {
// codec_left_context_sec 2.0.
QT_API void qt_tts_default_params(struct qt_tts_params * p);
// Number of RVQ codebooks (K) of the loaded codec. Pre-encoded ICL
// reference codes passed via ref_codes are laid out [K, ref_T]
// row-major; callers reading a packed .rvq stream need K to derive
// ref_T from the code count. Returns 0 on a NULL handle.
QT_API int qt_num_codebooks(const struct qt_context * q);
// Run the full TTS synthesis. Validates the params against the loaded
// model_type (the seven base / custom_voice / voice_design rules),
// resolves the seed, hands off to pipeline_tts_synthesize and fills
src/rvq-file.h
+111
View File
@@ -0,0 +1,111 @@
#pragma once
// rvq-file.h: packed RVQ code stream file IO (.rvq).
//
// Flat code stream packed at code_bits per code, LSB-first, no header.
// Layout is [K, T] row-major. K and code_bits are fixed by the codec
// config in the GGUF; T is derived from the file size:
// T = (filesize * 8) / (K * code_bits).
#include "utf8.h"
#include <cstdint>
#include <cstdio>
#include <string>
#include <vector>
// Pack a flat code stream into code_bits-per-code, LSB-first. Output size
// is ceil(N * code_bits / 8) bytes.
static std::vector<uint8_t> rvq_pack_codes(const std::vector<int32_t> & codes, int code_bits) {
const uint32_t mask = (1u << code_bits) - 1u;
const size_t total_bits = codes.size() * (size_t) code_bits;
std::vector<uint8_t> out((total_bits + 7) / 8, 0);
uint64_t acc = 0;
int bits_in_acc = 0;
size_t out_pos = 0;
for (size_t i = 0; i < codes.size(); i++) {
acc |= ((uint64_t) ((uint32_t) codes[i] & mask)) << bits_in_acc;
bits_in_acc += code_bits;
while (bits_in_acc >= 8) {
out[out_pos++] = (uint8_t) (acc & 0xFF);
acc >>= 8;
bits_in_acc -= 8;
}
}
if (bits_in_acc > 0) {
out[out_pos++] = (uint8_t) (acc & 0xFF);
}
return out;
}
// Symmetric unpack: reads N codes from packed bytes.
static std::vector<int32_t> rvq_unpack_codes(const std::vector<uint8_t> & in, size_t n_codes, int code_bits) {
const uint32_t mask = (1u << code_bits) - 1u;
std::vector<int32_t> out(n_codes);
uint64_t acc = 0;
int bits_in_acc = 0;
size_t in_pos = 0;
for (size_t i = 0; i < n_codes; i++) {
while (bits_in_acc < code_bits && in_pos < in.size()) {
acc |= ((uint64_t) in[in_pos++]) << bits_in_acc;
bits_in_acc += 8;
}
out[i] = (int32_t) (acc & mask);
acc >>= code_bits;
bits_in_acc -= code_bits;
}
return out;
}
// Read a .rvq file and unpack it into K*T codes. T is inferred from the
// file size.
static bool rvq_read_file(const char * path, int K, int code_bits, std::vector<int32_t> & codes, int * n_frames) {
FILE * f = utf8_fopen(path, "rb");
if (!f) {
fprintf(stderr, "[RVQ] FATAL: cannot open %s\n", path);
return false;
}
fseek(f, 0, SEEK_END);
long sz = ftell(f);
fseek(f, 0, SEEK_SET);
if (sz <= 0) {
fprintf(stderr, "[RVQ] FATAL: %s is empty\n", path);
fclose(f);
return false;
}
std::vector<uint8_t> buf((size_t) sz);
if (fread(buf.data(), 1, buf.size(), f) != buf.size()) {
fprintf(stderr, "[RVQ] FATAL: short read on %s\n", path);
fclose(f);
return false;
}
fclose(f);
const size_t total_bits = (size_t) sz * 8;
const size_t n_codes = total_bits / (size_t) code_bits;
if (n_codes == 0 || (n_codes % (size_t) K) != 0) {
fprintf(stderr, "[RVQ] FATAL: %s yields %zu codes, not a multiple of K=%d\n", path, n_codes, K);
return false;
}
codes = rvq_unpack_codes(buf, n_codes, code_bits);
*n_frames = (int) (n_codes / (size_t) K);
return true;
}
// Pack and write a .rvq file.
static bool rvq_write_file(const char * path, const std::vector<int32_t> & codes, int code_bits) {
std::vector<uint8_t> packed = rvq_pack_codes(codes, code_bits);
FILE * f = utf8_fopen(path, "wb");
if (!f) {
fprintf(stderr, "[RVQ] FATAL: cannot open %s for write\n", path);
return false;
}
if (fwrite(packed.data(), 1, packed.size(), f) != packed.size()) {
fprintf(stderr, "[RVQ] FATAL: short write on %s\n", path);
fclose(f);
return false;
}
fclose(f);
return true;
}
tools/qwen-codec.cpp
+99 -116
View File
@@ -5,6 +5,13 @@
// file extension: .wav in -> encode, .rvq in -> decode. Output is
// auto-named next to the input file by swapping the extension.
//
// Encode truncates the input to the hop boundary, strictly conforming
// to the qwen-tts --ref-wav ICL path, so a .rvq produced here feeds
// qwen-tts --ref-rvq directly. Passing --talker additionally runs the
// speaker encoder from the talker GGUF on the full input and writes
// the x-vector embedding next to the .rvq as a .spk file (raw f32,
// enc_dim values), feeding qwen-tts --ref-spk.
//
// File format (.rvq): flat code stream packed at 11 bits per code,
// LSB-first, no header. Layout is [K, T] row-major. K is fixed by the
// codec config in the GGUF (16 codebooks for the 12Hz tokenizer,
@@ -12,7 +19,11 @@
#include "audio-io.h"
#include "backend.h"
#include "gguf-weights.h"
#include "pipeline-codec.h"
#include "rvq-file.h"
#include "speaker-encoder-extract.h"
#include "speaker-encoder-weights.h"
#include "utf8.h"
#include "version.h"
@@ -23,115 +34,24 @@
#include <string>
#include <vector>
static const uint32_t RVQ_CODE_MASK = (1u << TOKENIZER_CODE_BITS) - 1u;
static void print_usage(const char * prog) {
fprintf(stderr, "qwentts.cpp %s\n\n", QWEN_VERSION);
fprintf(stderr,
"Usage: %s --model <gguf> [-i <input>] [--format <fmt>]\n\n"
"Usage: %s --model <gguf> [-i <input>] [--talker <gguf>] [--format <fmt>]\n\n"
"Required:\n"
" --model <gguf> Codec GGUF (qwen-tokenizer-12hz-*.gguf)\n\n"
"Optional:\n"
" -i <path> Input. WAV -> encode, .rvq -> decode\n"
" --talker <gguf> Talker GGUF (Base only). Encode also extracts the speaker\n"
" embedding and writes it next to the .rvq as a .spk file\n"
" --format <fmt> WAV output format: wav16, wav24, wav32 (default: wav16)\n\n"
"Output is auto-named next to input : clip.wav -> clip.rvq, clip.rvq -> clip.wav.\n"
"Encode truncates to the hop boundary, conforming to the qwen-tts --ref-wav path:\n"
"the .rvq feeds qwen-tts --ref-rvq, the .spk feeds qwen-tts --ref-spk.\n"
"When -i is omitted, runs a load self-test of the codec GGUF.\n",
prog);
}
// Symmetric unpack: reads N codes from packed bytes (11 bits LSB-first).
static std::vector<int32_t> unpack_codes(const std::vector<uint8_t> & in, size_t n_codes) {
std::vector<int32_t> out(n_codes);
uint64_t acc = 0;
int bits_in_acc = 0;
size_t in_pos = 0;
for (size_t i = 0; i < n_codes; i++) {
while (bits_in_acc < TOKENIZER_CODE_BITS && in_pos < in.size()) {
acc |= ((uint64_t) in[in_pos++]) << bits_in_acc;
bits_in_acc += 8;
}
out[i] = (int32_t) (acc & RVQ_CODE_MASK);
acc >>= TOKENIZER_CODE_BITS;
bits_in_acc -= TOKENIZER_CODE_BITS;
}
return out;
}
// Pack flat int32 codes into 11-bit LSB-first packed bytes. Output size is
// ceil(N * 11 / 8) bytes.
static std::vector<uint8_t> pack_codes(const std::vector<int32_t> & codes) {
const size_t total_bits = codes.size() * (size_t) TOKENIZER_CODE_BITS;
std::vector<uint8_t> out((total_bits + 7) / 8, 0);
uint64_t acc = 0;
int bits_in_acc = 0;
size_t out_pos = 0;
for (size_t i = 0; i < codes.size(); i++) {
acc |= ((uint64_t) ((uint32_t) codes[i] & RVQ_CODE_MASK)) << bits_in_acc;
bits_in_acc += TOKENIZER_CODE_BITS;
while (bits_in_acc >= 8) {
out[out_pos++] = (uint8_t) (acc & 0xFF);
acc >>= 8;
bits_in_acc -= 8;
}
}
if (bits_in_acc > 0) {
out[out_pos++] = (uint8_t) (acc & 0xFF);
}
return out;
}
// Read a .rvq file and unpack it into K*T codes. T is inferred from the
// file size: T = (filesize * 8) / (K * TOKENIZER_CODE_BITS).
static bool read_rvq(const char * path, int K, std::vector<int32_t> & codes, int * n_frames) {
FILE * f = utf8_fopen(path, "rb");
if (!f) {
fprintf(stderr, "[Codec] FATAL: cannot open %s\n", path);
return false;
}
fseek(f, 0, SEEK_END);
long sz = ftell(f);
fseek(f, 0, SEEK_SET);
if (sz <= 0) {
fprintf(stderr, "[Codec] FATAL: %s is empty\n", path);
fclose(f);
return false;
}
std::vector<uint8_t> buf((size_t) sz);
if (fread(buf.data(), 1, buf.size(), f) != buf.size()) {
fprintf(stderr, "[Codec] FATAL: short read on %s\n", path);
fclose(f);
return false;
}
fclose(f);
const size_t total_bits = (size_t) sz * 8;
const size_t n_codes = total_bits / (size_t) TOKENIZER_CODE_BITS;
if (n_codes == 0 || (n_codes % (size_t) K) != 0) {
fprintf(stderr, "[Codec] FATAL: %s yields %zu codes, not a multiple of K=%d\n", path, n_codes, K);
return false;
}
codes = unpack_codes(buf, n_codes);
*n_frames = (int) (n_codes / (size_t) K);
return true;
}
// Pack and write a .rvq file.
static bool write_rvq(const char * path, const std::vector<int32_t> & codes) {
std::vector<uint8_t> packed = pack_codes(codes);
FILE * f = utf8_fopen(path, "wb");
if (!f) {
fprintf(stderr, "[Codec] FATAL: cannot open %s for write\n", path);
return false;
}
if (fwrite(packed.data(), 1, packed.size(), f) != packed.size()) {
fprintf(stderr, "[Codec] FATAL: short write on %s\n", path);
fclose(f);
return false;
}
fclose(f);
return true;
}
// Replace or append extension on a path string.
static std::string swap_ext(const std::string & path, const char * ext) {
size_t dot = path.find_last_of('.');
@@ -154,6 +74,60 @@ static int infer_mode(const char * path) {
return 0;
}
// Load the speaker encoder from the talker GGUF, run it on the full input
// buffer and write the embedding as a raw f32 .spk file (enc_dim values,
// validated by filesize on the qwen-tts side). Returns 0 on success.
static int extract_spk(const char * talker_path,
BackendPair bp,
const float * audio,
int n_samples,
const char * out_path) {
GGUFModel gf = {};
if (!gf_load(&gf, talker_path)) {
fprintf(stderr, "[Codec] FATAL: cannot open talker GGUF %s\n", talker_path);
return 1;
}
SpeakerEncoderWeights sw = {};
if (!speaker_encoder_weights_load(&sw, gf, bp.backend)) {
fprintf(stderr, "[Codec] FATAL: speaker encoder load failed from %s\n", talker_path);
gf_close(&gf);
return 1;
}
gf_close(&gf);
if (sw.weight_buf == NULL) {
fprintf(stderr, "[Codec] FATAL: %s has no speaker encoder (Base only)\n", talker_path);
return 1;
}
ggml_backend_sched_t sched = backend_sched_new(bp, 4096);
const int enc_dim = sw.enc_dim;
std::vector<float> emb;
bool ok = speaker_encoder_extract(&sw, sched, audio, n_samples, emb);
ggml_backend_sched_free(sched);
speaker_encoder_weights_free(&sw);
if (!ok || (int) emb.size() != enc_dim) {
fprintf(stderr, "[Codec] FATAL: speaker embedding extraction failed (%zu values, enc_dim %d)\n", emb.size(),
enc_dim);
return 1;
}
FILE * f = utf8_fopen(out_path, "wb");
if (!f) {
fprintf(stderr, "[Codec] FATAL: cannot open %s for write\n", out_path);
return 1;
}
if (fwrite(emb.data(), sizeof(float), emb.size(), f) != emb.size()) {
fprintf(stderr, "[Codec] FATAL: short write on %s\n", out_path);
fclose(f);
return 1;
}
fclose(f);
fprintf(stderr, "[Codec] Wrote %s: %zu f32 values (%zu bytes)\n", out_path, emb.size(), emb.size() * sizeof(float));
return 0;
}
int main(int argc, char ** argv) {
utf8_init(&argc, &argv);
if (argc <= 1) {
@@ -161,13 +135,16 @@ int main(int argc, char ** argv) {
return 0;
}
const char * model_path = NULL;
const char * input_path = NULL;
WavFormat wav_fmt = WAV_S16;
const char * model_path = NULL;
const char * input_path = NULL;
const char * talker_path = NULL;
WavFormat wav_fmt = WAV_S16;
for (int i = 1; i < argc; i++) {
if (strcmp(argv[i], "--model") == 0 && i + 1 < argc) {
model_path = argv[++i];
} else if (strcmp(argv[i], "--talker") == 0 && i + 1 < argc) {
talker_path = argv[++i];
} else if (strcmp(argv[i], "-i") == 0 && i + 1 < argc) {
input_path = argv[++i];
} else if (strcmp(argv[i], "--format") == 0 && i + 1 < argc) {
@@ -217,40 +194,46 @@ int main(int argc, char ** argv) {
if (!input_path) {
fprintf(stderr, "[Codec] Load self-test passed\n");
} else if (mode == 1) {
// Encode .wav -> .rvq
// Encode .wav -> .rvq (+ .spk with --talker)
const std::string out_str = swap_ext(input_path, ".rvq");
int T_in = 0;
float * audio_in = audio_read_mono(input_path, TOKENIZER_SAMPLE_RATE, &T_in);
if (!audio_in || T_in <= 0) {
fprintf(stderr, "[Codec] FATAL: cannot read %s\n", input_path);
if (!audio_in || T_in < TOKENIZER_HOP_LENGTH) {
fprintf(stderr, "[Codec] FATAL: cannot read %s or input shorter than one hop (%d samples)\n", input_path,
TOKENIZER_HOP_LENGTH);
free(audio_in);
rc = 1;
} else {
// Pad to a multiple of HOP_LENGTH so the RVQ frame count is integral.
int hop = TOKENIZER_HOP_LENGTH;
int T_padded = ((T_in + hop - 1) / hop) * hop;
int T_frames = T_padded / hop;
// Truncate to a multiple of HOP_LENGTH, strictly conforming to
// the qwen-tts --ref-wav ICL path.
int hop = TOKENIZER_HOP_LENGTH;
int T_aligned = (T_in / hop) * hop;
int T_frames = T_aligned / hop;
std::vector<float> audio_buf((size_t) T_padded, 0.0f);
memcpy(audio_buf.data(), audio_in, (size_t) T_in * sizeof(float));
free(audio_in);
fprintf(stderr, "[Codec] Encode: %s, %d samples @ %d Hz, truncated to %d (%d frames @ 12.5 Hz, %.2f s)\n",
input_path, T_in, TOKENIZER_SAMPLE_RATE, T_aligned, T_frames,
(double) T_aligned / (double) TOKENIZER_SAMPLE_RATE);
fprintf(stderr, "[Codec] Encode: %s, %d samples @ %d Hz, padded to %d (%d frames @ 12.5 Hz, %.2f s)\n",
input_path, T_in, TOKENIZER_SAMPLE_RATE, T_padded, T_frames,
(double) T_padded / (double) TOKENIZER_SAMPLE_RATE);
std::vector<int32_t> codes = pipeline_codec_encode(&pc, audio_buf.data(), T_padded);
std::vector<int32_t> codes = pipeline_codec_encode(&pc, audio_in, T_aligned);
if (codes.empty()) {
fprintf(stderr, "[Codec] FATAL: encode failed\n");
rc = 1;
} else if (!write_rvq(out_str.c_str(), codes)) {
} else if (!rvq_write_file(out_str.c_str(), codes, TOKENIZER_CODE_BITS)) {
rc = 1;
} else {
fprintf(stderr, "[Codec] Wrote %s: K=%d T=%d, %zu codes -> %zu packed bytes\n", out_str.c_str(),
TOKENIZER_NUM_CODEBOOKS, T_frames, codes.size(),
(codes.size() * (size_t) TOKENIZER_CODE_BITS + 7) / 8);
}
// Speaker embedding extraction, conforming to the qwen-tts
// --ref-wav mode A path: the encoder consumes the FULL input
// buffer, never the hop-truncated one.
if (rc == 0 && talker_path) {
rc = extract_spk(talker_path, bp, audio_in, T_in, swap_ext(input_path, ".spk").c_str());
}
free(audio_in);
}
} else {
// Decode .rvq -> .wav
@@ -258,7 +241,7 @@ int main(int argc, char ** argv) {
std::vector<int32_t> codes;
int T = 0;
if (!read_rvq(input_path, TOKENIZER_NUM_CODEBOOKS, codes, &T)) {
if (!rvq_read_file(input_path, TOKENIZER_NUM_CODEBOOKS, TOKENIZER_CODE_BITS, codes, &T)) {
rc = 1;
} else {
fprintf(stderr, "[Codec] Decode: %s, K=%d T=%d (%.2f s)\n", input_path, TOKENIZER_NUM_CODEBOOKS, T,
tools/qwen-tts.cpp
+66
View File
@@ -13,6 +13,7 @@
#include "audio-io.h"
#include "qwen.h"
#include "rvq-file.h"
#include <cstdio>
#include <cstdlib>
@@ -40,6 +41,10 @@ static void print_usage(const char * prog) {
" CustomVoice, rejected for Base\n"
" --speaker <name> Speaker name (CustomVoice only)\n"
" --ref-wav <path> Reference WAV for voice cloning (Base only)\n"
" --ref-spk <path> Pre-extracted speaker embedding from qwen-codec --talker\n"
" (replaces --ref-wav, Base only)\n"
" --ref-rvq <path> Pre-encoded reference codes from qwen-codec (requires\n"
" --ref-spk and --ref-text, enables ICL clone mode)\n"
" --ref-text <path> Transcript file for the reference (enables ICL clone mode)\n"
" --max-new <n> Max new audio frames (default: 2048)\n"
" --codec-chunk-dur <f> Codec decode chunk duration in seconds (default: 24.0)\n"
@@ -69,6 +74,8 @@ struct Args {
const char * instruct;
const char * speaker;
const char * ref_wav;
const char * ref_spk;
const char * ref_rvq;
const char * ref_text_path;
const char * dump_dir;
const char * out_wav;
@@ -104,6 +111,34 @@ static std::string read_stdin_text() {
}
// Read a small text file into a string. Trims trailing newlines.
// 11 bits per code (V <= 2048), matching qwen-codec.
static const int RVQ_CODE_BITS = 11;
// Read a .spk file: raw f32 values, the count IS the embedding dimension.
static bool read_spk_file(const char * path, std::vector<float> & emb) {
FILE * f = utf8_fopen(path, "rb");
if (!f) {
fprintf(stderr, "[CLI] ERROR: cannot open --ref-spk '%s'\n", path);
return false;
}
fseek(f, 0, SEEK_END);
long sz = ftell(f);
fseek(f, 0, SEEK_SET);
if (sz <= 0 || (sz % (long) sizeof(float)) != 0) {
fprintf(stderr, "[CLI] ERROR: --ref-spk '%s' size %ld is not a positive multiple of 4\n", path, sz);
fclose(f);
return false;
}
emb.resize((size_t) sz / sizeof(float));
if (fread(emb.data(), sizeof(float), emb.size(), f) != emb.size()) {
fprintf(stderr, "[CLI] ERROR: short read on --ref-spk '%s'\n", path);
fclose(f);
return false;
}
fclose(f);
return true;
}
static bool read_text_file(const char * path, std::string & out) {
FILE * f = fopen(path, "rb");
if (!f) {
@@ -168,6 +203,10 @@ static bool parse_args(int argc, char ** argv, Args & a) {
a.speaker = argv[++i];
} else if (std::strcmp(arg, "--ref-wav") == 0 && i + 1 < argc) {
a.ref_wav = argv[++i];
} else if (std::strcmp(arg, "--ref-spk") == 0 && i + 1 < argc) {
a.ref_spk = argv[++i];
} else if (std::strcmp(arg, "--ref-rvq") == 0 && i + 1 < argc) {
a.ref_rvq = argv[++i];
} else if (std::strcmp(arg, "--ref-text") == 0 && i + 1 < argc) {
a.ref_text_path = argv[++i];
} else if (std::strcmp(arg, "--format") == 0 && i + 1 < argc) {
@@ -278,6 +317,29 @@ static int run(const Args & a) {
ref_n_samples = T_in;
}
// Latent reference files. The .spk holds raw f32 values whose count
// IS the embedding dimension; the .rvq holds the packed ICL code
// matrix. The facade validates the structural constraints (mutual
// exclusions, dim match against the talker hidden size).
std::vector<float> ref_spk_emb;
std::vector<int32_t> ref_codes;
int ref_T = 0;
if (a.ref_spk) {
if (!read_spk_file(a.ref_spk, ref_spk_emb)) {
qt_free(q);
return 1;
}
fprintf(stderr, "[CLI] Reference SPK: %s, %zu f32 values\n", a.ref_spk, ref_spk_emb.size());
}
if (a.ref_rvq) {
const int K = qt_num_codebooks(q);
if (!rvq_read_file(a.ref_rvq, K, RVQ_CODE_BITS, ref_codes, &ref_T)) {
qt_free(q);
return 1;
}
fprintf(stderr, "[CLI] Reference RVQ: %s, K=%d T=%d\n", a.ref_rvq, K, ref_T);
}
// Resolve output WAV format string: wav16 / wav24 / wav32. Default
// wav16 mirrors the omnivoice.cpp default.
WavFormat wav_fmt;
@@ -323,6 +385,10 @@ static int run(const Args & a) {
params.ref_audio_24k = ref_audio_24k;
params.ref_n_samples = ref_n_samples;
params.ref_text = ref_text;
params.ref_spk_emb = ref_spk_emb.empty() ? NULL : ref_spk_emb.data();
params.ref_spk_dim = (int) ref_spk_emb.size();
params.ref_codes = ref_codes.empty() ? NULL : ref_codes.data();
params.ref_T = ref_T;
params.seed = a.seed;
params.max_new_tokens = a.max_new_tokens;
params.do_sample = a.do_sample;