enne2/qwentts.cpp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

abi, pipeline, cli: conform qwentts on omnivoice convention

Browse Source
This commit is contained in:
Pascal
2026-05-14 21:42:52 +02:00
parent 259e7059f6
commit 38bf6d762a
11 changed files with 602 additions and 278 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/code-predictor-forward.h
+54 -8
View File
@@ -51,10 +51,27 @@ struct CodePredictorOutput {
std::vector<int32_t> codes;
};
// Manual F32 attention chain for the code predictor block. Same shape
// contract as talker_attn_f32: q [hd, T, n_q_heads], k/v [hd, T_full,
// n_kv], output [hd, n_q_heads, T]. Used when use_flash_attn is false.
static struct ggml_tensor * code_predictor_attn_f32(struct ggml_context * ctx,
struct ggml_tensor * q,
struct ggml_tensor * k,
struct ggml_tensor * v,
struct ggml_tensor * mask,
float scale) {
struct ggml_tensor * scores = ggml_mul_mat(ctx, k, q);
scores = ggml_soft_max_ext(ctx, scores, mask, scale, 0.0f);
struct ggml_tensor * vt = ggml_cont(ctx, ggml_transpose(ctx, v));
struct ggml_tensor * out = ggml_mul_mat(ctx, vt, scores);
return ggml_cont(ctx, ggml_permute(ctx, out, 0, 2, 1, 3));
}
// One Qwen3 decoder block, KV cached. K and V for the T fresh positions
// are written into the cache at [n_past, n_past+T) on dim 1; the
// attention reads the contiguous slice [0, n_past+T). Returns the layer
// output [hidden, T].
// output [hidden, T]. use_flash_attn and clamp_fp16 follow the same
// contract as in talker-forward.h.
static struct ggml_tensor * code_predictor_layer_forward(struct ggml_context * ctx,
const CodePredictorWeights * cw,
const TalkerLayer & layer,
@@ -65,6 +82,8 @@ static struct ggml_tensor * code_predictor_layer_forward(struct ggml_context *
struct ggml_tensor * v_cache,
int n_past,
int T,
bool use_flash_attn,
bool clamp_fp16,
struct ggml_cgraph * gf) {
const int n_q_heads = cw->num_attention_heads;
const int n_kv = cw->num_key_value_heads;
@@ -114,16 +133,32 @@ static struct ggml_tensor * code_predictor_layer_forward(struct ggml_context *
// Q permute [hd, n_q_heads, T] -> [hd, T, n_q_heads] for flash_attn_ext.
struct ggml_tensor * q_p = ggml_permute(ctx, q, 0, 2, 1, 3);
// Fused flash attention. Matches the working acestep qw3lm_build_attn
// pattern, fixes the Vulkan autoregressive decode bug.
// Clamp V before attention when clamp_fp16 is set, same rationale
// as the talker block: sub Ampere CUDA tensor cores accumulate in
// FP16 and a V projection overflow corrupts everything downstream.
if (clamp_fp16) {
v_full = ggml_clamp(ctx, v_full, -65504.0f, 65504.0f);
}
// Attention: fused flash kernel or manual F32 chain. Matches the
// working acestep qw3lm_build_attn pattern on the fused branch and
// the omnivoice qwen3_attn_f32 helper on the manual one.
float scale = 1.0f / sqrtf((float) hd);
struct ggml_tensor * attn = ggml_flash_attn_ext(ctx, q_p, k_full, v_full, mask, scale, 0.0f, 0.0f);
ggml_flash_attn_ext_set_prec(attn, GGML_PREC_F32);
struct ggml_tensor * attn;
if (use_flash_attn) {
attn = ggml_flash_attn_ext(ctx, q_p, k_full, v_full, mask, scale, 0.0f, 0.0f);
ggml_flash_attn_ext_set_prec(attn, GGML_PREC_F32);
} else {
attn = code_predictor_attn_f32(ctx, q_p, k_full, v_full, mask, scale);
}
attn = ggml_reshape_2d(ctx, attn, n_q_heads * hd, T);
struct ggml_tensor * o = ggml_mul_mat(ctx, layer.attn.o_proj_w, attn);
x = ggml_add(ctx, x, o);
if (clamp_fp16) {
x = ggml_clamp(ctx, x, -65504.0f, 65504.0f);
}
struct ggml_tensor * h2 = ggml_rms_norm(ctx, x, eps);
h2 = ggml_mul(ctx, h2, layer.post_attn_norm_w);
@@ -135,6 +170,9 @@ static struct ggml_tensor * code_predictor_layer_forward(struct ggml_context *
struct ggml_tensor * mlp = ggml_mul_mat(ctx, layer.mlp.down_proj_w, gu);
x = ggml_add(ctx, x, mlp);
if (clamp_fp16) {
x = ggml_clamp(ctx, x, -65504.0f, 65504.0f);
}
return x;
}
@@ -142,6 +180,7 @@ static struct ggml_tensor * code_predictor_layer_forward(struct ggml_context *
// position `n_past`, run all 5 layers, and pull the logits for the last
// position through lm_head[g_head]. The cache is written as a side
// effect so subsequent decode steps can append a single token.
// use_flash_attn / clamp_fp16 are forwarded as is to every layer.
static bool code_predictor_run(const CodePredictorWeights * cw,
KVCache * kv,
ggml_backend_sched_t sched,
@@ -150,6 +189,8 @@ static bool code_predictor_run(const CodePredictorWeights * cw,
int n_past,
int talker_hidden,
int g_head,
bool use_flash_attn,
bool clamp_fp16,
std::vector<float> * logits_out) {
const int vocab = cw->vocab_size;
const int n_layers = cw->num_hidden_layers;
@@ -188,7 +229,7 @@ static bool code_predictor_run(const CodePredictorWeights * cw,
for (int l = 0; l < n_layers; l++) {
h = code_predictor_layer_forward(gctx, cw, cw->layers[(size_t) l], h, pos_in, mask_in, kv->k[(size_t) l],
kv->v[(size_t) l], n_past, T, gf);
kv->v[(size_t) l], n_past, T, use_flash_attn, clamp_fp16, gf);
}
struct ggml_tensor * h_final = ggml_rms_norm(gctx, h, cw->rms_norm_eps);
@@ -280,6 +321,7 @@ static void embed_row_from_backend(struct ggml_tensor * t, int row_id, int dim,
// is the Philox subsequence of the c0 sample for this step; the 15
// acoustic samples consume subseq_base + 1 .. subseq_base + 15.
// Returns the full vector of 16 codes. dump_dir may be NULL.
// use_flash_attn / clamp_fp16 are forwarded as is to every internal run.
static bool code_predictor_step(const TalkerWeights * tw,
const CodePredictorWeights * cw,
KVCache * kv,
@@ -291,6 +333,8 @@ static bool code_predictor_step(const TalkerWeights * tw,
float top_p,
int64_t seed,
int64_t subseq_base,
bool use_flash_attn,
bool clamp_fp16,
const char * dump_dir,
CodePredictorOutput * out) {
// sub_input slots live at the talker hidden dimension: both the
@@ -316,7 +360,8 @@ static bool code_predictor_step(const TalkerWeights * tw,
embed_row_from_backend(tw->codec_embedding, c0, talker_hidden, prefill_input.data() + (size_t) talker_hidden);
std::vector<float> logits;
if (!code_predictor_run(cw, kv, sched, prefill_input.data(), 2, 0, talker_hidden, 0, &logits)) {
if (!code_predictor_run(cw, kv, sched, prefill_input.data(), 2, 0, talker_hidden, 0, use_flash_attn, clamp_fp16,
&logits)) {
return false;
}
{
@@ -340,7 +385,8 @@ static bool code_predictor_step(const TalkerWeights * tw,
for (int g = 1; g < n_acoustic; g++) {
embed_row_from_backend(cw->codec_embedding[(size_t) (g - 1)], out->codes[(size_t) g], talker_hidden,
step_input.data());
if (!code_predictor_run(cw, kv, sched, step_input.data(), 1, kv->cur_len, talker_hidden, g, &logits)) {
if (!code_predictor_run(cw, kv, sched, step_input.data(), 1, kv->cur_len, talker_hidden, g, use_flash_attn,
clamp_fp16, &logits)) {
return false;
}
float u_g = 0.0f;
src/pipeline-tts.cpp
+203 -81
View File
@@ -103,12 +103,24 @@ static void parse_generation_defaults(const GGUFModel & gf, GenerationDefaults &
g.max_new_tokens = (int) gf_get_u32(gf, "generation.max_new_tokens");
}
bool pipeline_tts_load(PipelineTTS * pt, const char * talker_gguf_path, const char * codec_gguf_path, BackendPair bp) {
bool pipeline_tts_load(PipelineTTS * pt,
const char * talker_gguf_path,
const char * codec_gguf_path,
BackendPair bp,
bool use_fa,
bool clamp_fp16) {
pt->bp = bp;
pt->backend = bp.backend;
pt->sched = NULL;
pt->has_speaker_encoder = false;
// Fused flash attention needs a GPU kernel; CPU only backends fall
// back to the F32 manual chain automatically. clamp_fp16 is forwarded
// verbatim: a no op on backends that already accumulate in F32, an
// FP16 overflow guard on sub Ampere CUDA tensor cores.
pt->use_flash_attn = use_fa && bp.has_gpu;
pt->clamp_fp16 = clamp_fp16;
if (!gf_load(&pt->gguf_talker, talker_gguf_path)) {
qt_log(QT_LOG_ERROR, "[Pipeline] failed to load talker GGUF: %s", talker_gguf_path);
return false;
@@ -217,9 +229,11 @@ bool pipeline_tts_load(PipelineTTS * pt, const char * talker_gguf_path, const ch
}
qt_log(QT_LOG_INFO,
"[Pipeline] Loaded: arch=%s variant=%s tokenizer=%s codebooks=%d speaker_encoder=%s speakers=%zu",
"[Pipeline] Loaded: arch=%s variant=%s tokenizer=%s codebooks=%d speaker_encoder=%s speakers=%zu fa=%s "
"clamp_fp16=%s",
pt->model_size.c_str(), pt->model_type.c_str(), pt->tokenizer_type.c_str(), pt->num_code_groups,
pt->has_speaker_encoder ? "loaded" : "absent", pt->speakers.size());
pt->has_speaker_encoder ? "loaded" : "absent", pt->speakers.size(), pt->use_flash_attn ? "on" : "off",
pt->clamp_fp16 ? "on" : "off");
return true;
}
@@ -267,23 +281,56 @@ static void embed_row_from_gguf(const GGUFModel & gf, const char * tensor_name,
tt->to_float(row, dst, hidden);
}
bool pipeline_tts_synthesize(PipelineTTS * pt,
BPETokenizer * tok,
const PipelineTTSSynthesizeParams & params,
PipelineTTSSynthesizeOutput * out) {
out->audio.clear();
// Helper: malloc a heap copy of a float vector and hand it off into the
// public qt_audio struct. Returns true on success; on OOM sets the
// error string and leaves out untouched. Empty vectors land as an
// allocation of size 0 with a stub malloc to keep the free path simple.
static bool fill_qt_audio(const std::vector<float> & audio, qt_audio * out) {
const size_t n = audio.size();
const size_t bytes = n * sizeof(float);
float * buf = (float *) std::malloc(bytes > 0 ? bytes : 1);
if (!buf) {
qt_set_error("pipeline_tts_synthesize: malloc failed for %zu samples", n);
return false;
}
if (n > 0) {
std::memcpy(buf, audio.data(), bytes);
}
out->samples = buf;
out->n_samples = (int) n;
out->sample_rate = TOKENIZER_SAMPLE_RATE;
out->channels = 1;
return true;
}
int pipeline_tts_duration_sec_to_tokens(const PipelineTTS * /*pt*/, float duration_sec) {
// The 12 Hz Qwen3-TTS tokenizer has a fixed hop of 1920 samples at
// 24 kHz, so the frame rate is 24000 / 1920 = 12.5 Hz regardless of
// the variant loaded. Clamp to a minimum of one frame so a zero or
// negative duration still picks up one decoder step.
const float fps = (float) TOKENIZER_SAMPLE_RATE / (float) TOKENIZER_HOP_LENGTH;
int n_frames = (int) (duration_sec * fps + 0.5f);
if (n_frames < 1) {
n_frames = 1;
}
return n_frames;
}
qt_status pipeline_tts_synthesize(PipelineTTS * pt,
BPETokenizer * tok,
const struct qt_tts_params * params,
int64_t resolved_seed,
struct qt_audio * out) {
PromptBuilderOutput prompt;
const std::string instruct = params.instruct ? params.instruct : "";
const std::string speaker = params.speaker ? params.speaker : "";
const std::string ref_text = params.ref_text ? params.ref_text : "";
const std::string instruct = params->instruct ? params->instruct : "";
const std::string speaker = params->speaker ? params->speaker : "";
const std::string ref_text = params->ref_text ? params->ref_text : "";
// Voice clone mode A: if ref_audio_24k is given, run the speaker
// encoder on the pre-decoded mono buffer and feed the resulting
// embedding straight into the prompt builder. Mutually exclusive
// with --speaker.
const bool has_ref_audio = (params.ref_audio_24k != NULL) && (params.ref_n_samples > 0);
const bool has_ref_audio = (params->ref_audio_24k != NULL) && (params->ref_n_samples > 0);
std::vector<float> ref_spk_emb;
const float * ref_spk_emb_ptr = NULL;
if (has_ref_audio) {
@@ -291,18 +338,18 @@ bool pipeline_tts_synthesize(PipelineTTS * pt,
qt_set_error(
"pipeline_tts_synthesize: --ref-wav requires a model with a loaded speaker encoder (Base only)");
qt_log(QT_LOG_ERROR, "[Pipeline] --ref-wav requires a model with a loaded speaker encoder (Base only)");
return false;
return QT_STATUS_GENERATE_FAILED;
}
if (!speaker_encoder_extract(&pt->speaker_encoder, pt->sched, params.ref_audio_24k, params.ref_n_samples,
ref_spk_emb, params.dump_dir)) {
return false;
if (!speaker_encoder_extract(&pt->speaker_encoder, pt->sched, params->ref_audio_24k, params->ref_n_samples,
ref_spk_emb, params->dump_dir)) {
return QT_STATUS_GENERATE_FAILED;
}
if ((int) ref_spk_emb.size() != pt->talker.hidden_size) {
qt_set_error("pipeline_tts_synthesize: speaker embedding size %zu mismatches talker hidden %d",
ref_spk_emb.size(), pt->talker.hidden_size);
qt_log(QT_LOG_ERROR, "[Pipeline] speaker embedding size %zu mismatches talker hidden %d",
ref_spk_emb.size(), pt->talker.hidden_size);
return false;
return QT_STATUS_GENERATE_FAILED;
}
ref_spk_emb_ptr = ref_spk_emb.data();
}
@@ -318,35 +365,35 @@ bool pipeline_tts_synthesize(PipelineTTS * pt,
if (!has_ref_audio) {
qt_set_error("pipeline_tts_synthesize: --ref-text requires --ref-wav");
qt_log(QT_LOG_ERROR, "[Pipeline] --ref-text requires --ref-wav");
return false;
return QT_STATUS_INVALID_PARAMS;
}
// The codec hop is 1920 samples at 24 kHz so n_samples must be
// a multiple of 1920. Truncate to the nearest hop boundary.
if (params.ref_n_samples < TOKENIZER_HOP_LENGTH) {
qt_set_error("pipeline_tts_synthesize: ref_wav too short for ICL (%d samples)", params.ref_n_samples);
qt_log(QT_LOG_ERROR, "[Pipeline] ref_wav too short for ICL (%d samples)", params.ref_n_samples);
return false;
if (params->ref_n_samples < TOKENIZER_HOP_LENGTH) {
qt_set_error("pipeline_tts_synthesize: ref_wav too short for ICL (%d samples)", params->ref_n_samples);
qt_log(QT_LOG_ERROR, "[Pipeline] ref_wav too short for ICL (%d samples)", params->ref_n_samples);
return QT_STATUS_INVALID_PARAMS;
}
int aligned_T = (params.ref_n_samples / TOKENIZER_HOP_LENGTH) * TOKENIZER_HOP_LENGTH;
ref_codes = pipeline_codec_encode(&pt->codec, params.ref_audio_24k, aligned_T, params.dump_dir);
int aligned_T = (params->ref_n_samples / TOKENIZER_HOP_LENGTH) * TOKENIZER_HOP_LENGTH;
ref_codes = pipeline_codec_encode(&pt->codec, params->ref_audio_24k, aligned_T, params->dump_dir);
if (ref_codes.empty()) {
qt_set_error("pipeline_tts_synthesize: pipeline_codec_encode returned empty codes");
qt_log(QT_LOG_ERROR, "[Pipeline] pipeline_codec_encode returned empty codes");
return false;
return QT_STATUS_GENERATE_FAILED;
}
ref_codes_T = (int) ref_codes.size() / pt->num_code_groups;
qt_log(QT_LOG_INFO, "[Pipeline] ICL ref_codes: %d frames at 12.5 Hz (%d audio samples)", ref_codes_T,
aligned_T);
}
if (!prompt_builder_build(pt, tok, params.text, params.lang, instruct, speaker, ref_spk_emb_ptr, ref_text,
if (!prompt_builder_build(pt, tok, params->text, params->lang, instruct, speaker, ref_spk_emb_ptr, ref_text,
ref_codes_T > 0 ? ref_codes.data() : NULL, ref_codes_T, &prompt)) {
return false;
return QT_STATUS_GENERATE_FAILED;
}
if (params.dump_dir) {
if (params->dump_dir) {
DebugDumper d;
debug_init(&d, params.dump_dir);
debug_init(&d, params->dump_dir);
std::vector<int32_t> ids32(prompt.prompt_ids.begin(), prompt.prompt_ids.end());
int n_ids = (int) ids32.size();
debug_dump_i32_as_f32(&d, "prompt-ids", ids32.data(), &n_ids, 1);
@@ -354,12 +401,12 @@ bool pipeline_tts_synthesize(PipelineTTS * pt,
debug_dump_2d(&d, "trailing-text-hidden", prompt.trailing_text_hidden.data(), prompt.T_trailing, prompt.hidden);
debug_dump_1d(&d, "tts-pad-embed", prompt.tts_pad_embed.data(), prompt.hidden);
// Voice clone dumps: speaker-emb fires when ref_wav is set
// Voice clone dumps: spk-emb fires when ref_wav is set
// (modes A and B), ref-codes fires only when ref_text is also set
// (mode B ICL). Both are no-ops in base / tts / customvoice modes,
// the dump files simply do not appear in those runs.
if (ref_spk_emb_ptr != NULL) {
debug_dump_1d(&d, "speaker-emb", ref_spk_emb_ptr, pt->talker.hidden_size);
debug_dump_1d(&d, "spk-emb", ref_spk_emb_ptr, pt->talker.hidden_size);
}
if (ref_codes_T > 0) {
const int shape[2] = { pt->num_code_groups, ref_codes_T };
@@ -367,34 +414,35 @@ bool pipeline_tts_synthesize(PipelineTTS * pt,
}
}
// Generation loop: at each step we recompute the full Talker prefix
// (no KV cache yet) over the prompt prefix concatenated with all the
// next-token embeddings produced so far, sample c0, run the code
// predictor for the 15 acoustic codes, build the next-token
// embedding by summing the 16 codebook embeddings and the matching
// trailing-text overlay, and append it to the running context. We
// stop on codec_eos or when max_new_tokens is reached.
const int hidden = prompt.hidden;
const int codec_eos_id = pt->codec_specials.eos_id;
const int num_codebooks = pt->num_code_groups;
const int talker_vocab = pt->talker.vocab_size;
// Greedy collapses to temperature <= 0 in sample_top_k_p.
float talker_T = params.do_sample ? params.temperature : 0.0f;
float subtk_T = params.subtalker_do_sample ? params.subtalker_temperature : 0.0f;
float talker_rp = params.repetition_penalty;
// Generation loop: step 0 prefills the talker over the full prompt
// and writes T_ctx positions into the KV cache. Subsequent steps
// feed one next_emb at a time and append one position. The code
// predictor maintains its own per-frame cache that gets reset at
// every step.
const int hidden = prompt.hidden;
const int codec_eos_id = pt->codec_specials.eos_id;
const int num_codebooks = pt->num_code_groups;
const int talker_vocab = pt->talker.vocab_size;
const bool use_fa = pt->use_flash_attn;
const bool clamp_fp16 = pt->clamp_fp16;
const float talker_T = params->do_sample ? params->temperature : 0.0f;
const float subtk_T = params->subtalker_do_sample ? params->subtalker_temperature : 0.0f;
const float talker_rp = params->repetition_penalty;
// Streaming config: when on_chunk is set, decode the codec every
// chunk_frames AR frames and emit through the callback. The buffered
// path keeps every code, decodes once at the very end and copies the
// resulting audio into out. chunk_frames <= 0 falls back to 1 second.
const bool streaming = (params->on_chunk != NULL);
const float chunk_sec = params->chunk_duration_sec > 0.0f ? params->chunk_duration_sec : 1.0f;
const int chunk_frames = pipeline_tts_duration_sec_to_tokens(pt, chunk_sec);
std::vector<std::vector<int32_t>> all_codes;
all_codes.reserve((size_t) params.max_new_tokens);
all_codes.reserve((size_t) params->max_new_tokens);
// c0 codes already emitted, fed to repetition penalty.
std::vector<int32_t> talker_history;
talker_history.reserve((size_t) params.max_new_tokens);
talker_history.reserve((size_t) params->max_new_tokens);
// Global Philox subsequence counter advances once per primitive
// sample (one for c0 of each step, then 15 for the predictor codes).
@@ -402,18 +450,58 @@ bool pipeline_tts_synthesize(PipelineTTS * pt,
std::vector<float> next_emb((size_t) hidden, 0.0f);
for (int step = 0; step < params.max_new_tokens; step++) {
// Streaming bookkeeping: pending_codes accumulates frames since the
// last emit; emit_pending decodes them through the codec and feeds
// on_chunk. Returns false to abort with QT_STATUS_CANCELLED.
std::vector<std::vector<int32_t>> pending_codes;
pending_codes.reserve((size_t) chunk_frames);
auto emit_pending = [&]() -> bool {
if (pending_codes.empty()) {
return true;
}
const int T_frames = (int) pending_codes.size();
std::vector<int32_t> codes_kt((size_t) num_codebooks * (size_t) T_frames);
for (int t = 0; t < T_frames; t++) {
for (int k = 0; k < num_codebooks; k++) {
codes_kt[(size_t) k * (size_t) T_frames + (size_t) t] = pending_codes[(size_t) t][(size_t) k];
}
}
std::vector<float> chunk_audio = pipeline_codec_decode(&pt->codec, codes_kt.data(), num_codebooks, T_frames);
if (chunk_audio.empty()) {
qt_set_error("pipeline_tts_synthesize: streaming codec decode returned no audio");
qt_log(QT_LOG_ERROR, "[Pipeline] streaming codec decode returned no audio");
return false;
}
if (!params->on_chunk(chunk_audio.data(), (int) chunk_audio.size(), params->on_chunk_user_data)) {
qt_log(QT_LOG_INFO, "[Pipeline] on_chunk callback aborted the synthesis");
return false;
}
pending_codes.clear();
return true;
};
for (int step = 0; step < params->max_new_tokens; step++) {
// Cooperative cancellation, polled at every step. Granularity is
// one AR frame = 1 / 12.5 Hz ~ 83 ms of audio, which is well
// below any reasonable UX cancel latency target.
if (params->cancel && params->cancel(params->cancel_user_data)) {
qt_log(QT_LOG_INFO, "[Pipeline] cancelled at step %d", step);
return QT_STATUS_CANCELLED;
}
TalkerForwardOutput fw;
const char * step_dump = (params.dump_dir && step == 0) ? params.dump_dir : NULL;
const char * step_dump = (params->dump_dir && step == 0) ? params->dump_dir : NULL;
bool ok;
if (step == 0) {
ok = talker_forward_prefill(&pt->talker, &pt->talker_kv, pt->sched, prompt.input_embed.data(), prompt.T_ctx,
step_dump, &fw);
use_fa, clamp_fp16, step_dump, &fw);
} else {
ok = talker_forward_decode(&pt->talker, &pt->talker_kv, pt->sched, next_emb.data(), &fw);
ok =
talker_forward_decode(&pt->talker, &pt->talker_kv, pt->sched, next_emb.data(), use_fa, clamp_fp16, &fw);
}
if (!ok) {
return false;
return QT_STATUS_GENERATE_FAILED;
}
// Bisection dump: the talker hidden_last at step 1 is the input
@@ -421,9 +509,9 @@ bool pipeline_tts_synthesize(PipelineTTS * pt,
// step 0. Pairing it byte for byte with the Python hook tells us
// whether the next-emb composition + talker decode round trip
// is bit exact end to end.
if (params.dump_dir && step == 1) {
if (params->dump_dir && step == 1) {
DebugDumper d;
debug_init(&d, params.dump_dir);
debug_init(&d, params->dump_dir);
debug_dump_1d(&d, "talker-hidden-step1", fw.hidden_last.data(), hidden);
}
@@ -431,12 +519,13 @@ bool pipeline_tts_synthesize(PipelineTTS * pt,
// codec_eos. Then run the upstream sampling chain.
apply_suppress(fw.logits_last.data(), talker_vocab, talker_vocab - 1024, talker_vocab, codec_eos_id);
float u_c0 = 0.0f;
int c0 = sample_top_k_p(fw.logits_last.data(), talker_vocab, talker_T, params.top_k, params.top_p, talker_rp,
talker_history.data(), (int) talker_history.size(), params.seed, subseq_counter, &u_c0);
int c0 =
sample_top_k_p(fw.logits_last.data(), talker_vocab, talker_T, params->top_k, params->top_p, talker_rp,
talker_history.data(), (int) talker_history.size(), resolved_seed, subseq_counter, &u_c0);
subseq_counter++;
if (c0 < 0) {
qt_log(QT_LOG_ERROR, "[Pipeline] c0 sample returned no candidate");
return false;
return QT_STATUS_GENERATE_FAILED;
}
// Trace the first 32 samples unconditionally so [Sample] lines
@@ -453,11 +542,11 @@ bool pipeline_tts_synthesize(PipelineTTS * pt,
}
CodePredictorOutput cp;
const char * cp_dump = (params.dump_dir && step == 0) ? params.dump_dir : NULL;
const char * cp_dump = (params->dump_dir && step == 0) ? params->dump_dir : NULL;
if (!code_predictor_step(&pt->talker, &pt->code_predictor, &pt->code_predictor_kv, pt->sched,
fw.hidden_last.data(), c0, subtk_T, params.subtalker_top_k, params.subtalker_top_p,
params.seed, subseq_counter - 1, cp_dump, &cp)) {
return false;
fw.hidden_last.data(), c0, subtk_T, params->subtalker_top_k, params->subtalker_top_p,
resolved_seed, subseq_counter - 1, use_fa, clamp_fp16, cp_dump, &cp)) {
return QT_STATUS_GENERATE_FAILED;
}
// Predictor consumed (num_codebooks - 1) subsequences after the
// c0 one (subseq_base + 1 .. subseq_base + 15).
@@ -465,6 +554,14 @@ bool pipeline_tts_synthesize(PipelineTTS * pt,
all_codes.push_back(cp.codes);
talker_history.push_back(c0);
if (streaming) {
pending_codes.push_back(cp.codes);
if ((int) pending_codes.size() >= chunk_frames) {
if (!emit_pending()) {
return QT_STATUS_CANCELLED;
}
}
}
// Build next-token embedding: sum of 16 codebook embeddings.
// codebook 0 uses talker.codec_embedding, the 15 acoustic
@@ -500,9 +597,9 @@ bool pipeline_tts_synthesize(PipelineTTS * pt,
// is the only thing controlling the talker forward at step 1, so
// matching it bit-exact against Python pinpoints any drift in
// the codebook embedding sums or the trailing text overlay.
if (params.dump_dir && step == 0) {
if (params->dump_dir && step == 0) {
DebugDumper d;
debug_init(&d, params.dump_dir);
debug_init(&d, params->dump_dir);
debug_dump_1d(&d, "next-emb-step0", next_emb.data(), hidden);
}
@@ -513,9 +610,9 @@ bool pipeline_tts_synthesize(PipelineTTS * pt,
qt_log(QT_LOG_INFO, "[Pipeline] Generation done : %zu frames", all_codes.size());
if (params.dump_dir && !all_codes.empty()) {
if (params->dump_dir && !all_codes.empty()) {
DebugDumper d;
debug_init(&d, params.dump_dir);
debug_init(&d, params->dump_dir);
int T_frames = (int) all_codes.size();
std::vector<int32_t> flat((size_t) T_frames * (size_t) num_codebooks);
for (int t = 0; t < T_frames; t++) {
@@ -527,31 +624,56 @@ bool pipeline_tts_synthesize(PipelineTTS * pt,
debug_dump_i32_as_f32(&d, "codes-full", flat.data(), shape, 2);
}
// Codec decode: transpose codes from [T_frames, K] to [K, T_frames]
// because pipeline_codec_decode expects K-major layout (codebooks
// first, frames second), then return the 24 kHz mono audio.
if (all_codes.empty()) {
return true;
// Streaming tail: flush remaining frames through the callback. The
// buffered output stays empty in this branch; the caller already
// received every sample through on_chunk.
if (streaming) {
if (!emit_pending()) {
return QT_STATUS_CANCELLED;
}
out->samples = NULL;
out->n_samples = 0;
out->sample_rate = TOKENIZER_SAMPLE_RATE;
out->channels = 1;
return QT_STATUS_OK;
}
int T_frames = (int) all_codes.size();
// Buffered path: empty all_codes means EOS at step 0 with no audio.
// Return success and an empty qt_audio struct; the facade leaves it
// to the caller to decide what to do with a zero sample synthesis.
if (all_codes.empty()) {
out->samples = NULL;
out->n_samples = 0;
out->sample_rate = TOKENIZER_SAMPLE_RATE;
out->channels = 1;
return QT_STATUS_OK;
}
// Codec decode: transpose codes from [T_frames, K] to [K, T_frames]
// because pipeline_codec_decode expects K-major layout (codebooks
// first, frames second), then materialise the 24 kHz mono audio.
const int T_frames = (int) all_codes.size();
std::vector<int32_t> codes_kt((size_t) num_codebooks * (size_t) T_frames);
for (int t = 0; t < T_frames; t++) {
for (int k = 0; k < num_codebooks; k++) {
codes_kt[(size_t) k * (size_t) T_frames + (size_t) t] = all_codes[(size_t) t][(size_t) k];
}
}
out->audio = pipeline_codec_decode(&pt->codec, codes_kt.data(), num_codebooks, T_frames);
if (out->audio.empty()) {
std::vector<float> audio = pipeline_codec_decode(&pt->codec, codes_kt.data(), num_codebooks, T_frames);
if (audio.empty()) {
qt_set_error("pipeline_tts_synthesize: codec decode returned no audio");
qt_log(QT_LOG_ERROR, "[Pipeline] codec decode returned no audio");
return false;
return QT_STATUS_GENERATE_FAILED;
}
if (params.dump_dir) {
if (params->dump_dir) {
DebugDumper d;
debug_init(&d, params.dump_dir);
debug_dump_1d(&d, "output-audio", out->audio.data(), (int) out->audio.size());
debug_init(&d, params->dump_dir);
debug_dump_1d(&d, "output-audio", audio.data(), (int) audio.size());
}
return true;
if (!fill_qt_audio(audio, out)) {
return QT_STATUS_OOM;
}
return QT_STATUS_OK;
}
src/pipeline-tts.h
+57 -46
View File
@@ -2,10 +2,13 @@
// pipeline-tts.h: full TTS pipeline composition (Talker LM + code
// predictor MTP head + optional speaker encoder + 12Hz codec decoder).
//
// Phase 2.0 covers load-only: parse hyperparameters from both GGUF
// files, load every weight tensor on the configured backend, and
// expose the metadata needed to build forward graphs in later phases.
// No graph construction or sampling is wired here yet.
// pipeline_tts_load opens the talker GGUF and the codec GGUF, parses
// every typed metadata block (specials, languages, speakers,
// generation defaults), loads every weight tensor on the shared
// backend and initialises both KV caches. pipeline_tts_synthesize
// runs the prompt assembly, the autoregressive frame loop and the
// codec decode in one pass; it fills the public qt_audio struct
// directly so the facade in qwen.cpp stays a thin wrapper.
#include "backend.h"
#include "code-predictor-weights.h"
@@ -13,6 +16,7 @@
#include "gguf-weights.h"
#include "kv-cache.h"
#include "pipeline-codec.h"
#include "qwen.h"
#include "speaker-encoder-weights.h"
#include "talker-weights.h"
@@ -43,7 +47,7 @@ struct LanguageEntry {
int id;
};
// Speaker entry for CustomVoice models. id is the codec embedding row id
// Speaker entry for CustomVoice variants. id is the codec embedding row id
// inserted in the talker prefix, dialect is empty unless the speaker
// overrides the user supplied language with a dialect lang_id (eric ->
// sichuan_dialect, dylan -> beijing_dialect on the upstream checkpoint).
@@ -90,6 +94,15 @@ struct PipelineTTS {
ggml_backend_t backend;
ggml_backend_sched_t sched;
// Attention path config, set at load and forwarded to every
// talker / code predictor forward. use_flash_attn collapses to
// false on CPU only backends (fused FA needs a GPU kernel).
// clamp_fp16 inserts ggml_clamp on V before attention and on the
// residual stream between blocks to guard FP16 matmul accumulation
// on sub Ampere CUDA targets.
bool use_flash_attn;
bool clamp_fp16;
// Persistent KV caches: the talker holds the LM context, the
// predictor holds one frame's 16 sub-steps and gets reset every
// frame in code_predictor_step.
@@ -99,50 +112,48 @@ struct PipelineTTS {
// Open the talker GGUF and the codec GGUF, load every module on the
// shared backend. Aborts with a logged error on any missing tensor or
// invalid metadata. Caller frees with pipeline_tts_free.
bool pipeline_tts_load(PipelineTTS * pt, const char * talker_gguf_path, const char * codec_gguf_path, BackendPair bp);
// invalid metadata. use_fa is gated on bp.has_gpu inside the load:
// CPU only runs always use the manual F32 attention chain. clamp_fp16
// is forwarded as is. Caller frees with pipeline_tts_free.
bool pipeline_tts_load(PipelineTTS * pt,
const char * talker_gguf_path,
const char * codec_gguf_path,
BackendPair bp,
bool use_fa,
bool clamp_fp16);
void pipeline_tts_free(PipelineTTS * pt);
struct BPETokenizer;
// Parameters for one synthesis call. Lifetime constraint: text and lang
// are borrowed pointers, must outlive the call. dump_dir, when non-NULL,
// captures step 0 prefill activations plus the codes-full / output-audio
// dumps under the named directory; debug only, slows the run.
struct PipelineTTSSynthesizeParams {
const char * text;
const char * lang;
const char * instruct;
const char * speaker;
const float * ref_audio_24k;
int ref_n_samples;
const char * ref_text;
int64_t seed;
int max_new_tokens;
bool do_sample;
float temperature;
int top_k;
float top_p;
float repetition_penalty;
bool subtalker_do_sample;
float subtalker_temperature;
int subtalker_top_k;
float subtalker_top_p;
const char * dump_dir;
};
// Output of one synthesis call. audio is a 24 kHz mono F32 PCM buffer
// already decoded through the codec; the caller writes it to disk.
struct PipelineTTSSynthesizeOutput {
std::vector<float> audio;
int sample_rate;
};
// Run the full TTS pipeline: prompt assembly, prefill, frame loop with
// sampling, codec decode. Returns false on any failure with a diagnostic
// already routed through qt_log / qt_set_error.
bool pipeline_tts_synthesize(PipelineTTS * pt,
BPETokenizer * tok,
const PipelineTTSSynthesizeParams & params,
PipelineTTSSynthesizeOutput * out);
// sampling, codec decode, fill qt_audio. Reads every knob (text,
// references, sampling, cancel, on_chunk, ...) straight from the
// public qt_tts_params struct so the facade in qwen.cpp can hand it
// off verbatim after the mode validation and seed resolve.
//
// Returns QT_STATUS_OK on success. On any failure returns a negative
// qt_status with a diagnostic already routed through qt_log /
// qt_set_error and leaves `out` empty. QT_STATUS_CANCELLED is returned
// when params->cancel or params->on_chunk returns true / false
// respectively during the AR loop.
//
// In buffered mode (params->on_chunk == NULL) the synthesised waveform
// is malloc allocated into out->samples; the caller releases it with
// qt_audio_free. In streaming mode (params->on_chunk != NULL) audio is
// emitted through the callback as decoded chunks and out->samples
// stays NULL on success.
//
// resolved_seed is the seed actually used for sampling: qt_synthesize
// hands over the same value it logged so dump traces and replays line
// up across runs even when params->seed was -1.
qt_status pipeline_tts_synthesize(PipelineTTS * pt,
BPETokenizer * tok,
const struct qt_tts_params * params,
int64_t resolved_seed,
struct qt_audio * out);
// Convert a duration in seconds to a frame count at the codec frame
// rate (QT_CODEC_SAMPLE_RATE / TOKENIZER_HOP_LENGTH). Clamps to a
// minimum of one frame.
int pipeline_tts_duration_sec_to_tokens(const PipelineTTS * pt, float duration_sec);
src/qwen.cpp
+77 -82
View File
@@ -4,10 +4,10 @@
// so the symbols carry C linkage and are linkable from C, Rust, Go,
// Python ctypes and any other binding generator. The struct
// qt_context opaque handle owns one BackendPair, one PipelineTTS
// (which embeds its PipelineCodec) and one BPETokenizer. qt_init
// walks the load chain in dependency order and unwinds whatever it
// already allocated when any step fails. qt_free mirrors that order
// in reverse.
// (which already embeds the PipelineCodec) and one BPETokenizer.
// qt_init walks the load chain in dependency order and unwinds
// whatever it already allocated when any step fails. qt_free mirrors
// that order in reverse.
//
// This translation unit also absorbs the internal qt_set_error /
// qt_throw / qt_log helpers that the rest of the codebase calls. The
@@ -137,6 +137,19 @@ void qt_log(qt_log_level level, const char * fmt, ...) {
}
}
// Resolve a -1 seed to a hardware random 64-bit value. Anything else is
// forwarded verbatim, so reproducibility is one explicit seed away. The
// resolved value travels into pipeline_tts_synthesize so the dump traces
// log the exact seed that drove the sampler, even when the caller asked
// for non determinism.
static int64_t qt_resolve_seed(int64_t seed) {
if (seed >= 0) {
return seed;
}
std::random_device rd;
return (int64_t) (((uint64_t) rd() << 32) ^ (uint64_t) rd());
}
extern "C" {
const char * qt_version(void) {
@@ -174,12 +187,14 @@ void qt_init_default_params(struct qt_init_params * p) {
p->abi_version = QT_ABI_VERSION;
p->talker_path = nullptr;
p->codec_path = nullptr;
p->use_fa = true;
p->clamp_fp16 = false;
}
void qt_tts_default_params(struct qt_tts_params * p) {
p->abi_version = QT_ABI_VERSION;
p->text = nullptr;
p->lang = "english";
p->lang = nullptr;
p->instruct = nullptr;
p->speaker = nullptr;
p->ref_audio_24k = nullptr;
@@ -197,6 +212,11 @@ void qt_tts_default_params(struct qt_tts_params * p) {
p->subtalker_top_k = 50;
p->subtalker_top_p = 1.0f;
p->dump_dir = nullptr;
p->cancel = nullptr;
p->cancel_user_data = nullptr;
p->on_chunk = nullptr;
p->on_chunk_user_data = nullptr;
p->chunk_duration_sec = 1.0f;
}
struct qt_context * qt_init(const struct qt_init_params * params) {
@@ -231,7 +251,8 @@ struct qt_context * qt_init(const struct qt_init_params * params) {
qt_throw("qt_init: backend_init failed (no GGML backend available)");
}
if (!pipeline_tts_load(&q->pt, params->talker_path, params->codec_path, q->bp)) {
if (!pipeline_tts_load(&q->pt, params->talker_path, params->codec_path, q->bp, params->use_fa,
params->clamp_fp16)) {
qt_throw("qt_init: pipeline_tts_load failed for '%s' / '%s'", params->talker_path, params->codec_path);
}
@@ -266,29 +287,28 @@ void qt_free(struct qt_context * q) {
delete q;
}
// Resolve a -1 seed to a hardware random 64-bit value. Anything else is
// forwarded verbatim, so reproducibility is one explicit seed away.
static int64_t qt_resolve_seed(int64_t seed) {
if (seed >= 0) {
return seed;
}
std::random_device rd;
return (int64_t) (((uint64_t) rd() << 32) ^ (uint64_t) rd());
}
enum qt_status qt_synthesize(struct qt_context * q, const struct qt_tts_params * params, struct qt_audio * out) {
if (!q || !params || !out) {
qt_set_error("qt_synthesize: q, params or out is NULL");
if (!q || !params) {
qt_set_error("qt_synthesize: q or params is NULL");
if (out) {
qt_audio_free(out);
}
return QT_STATUS_INVALID_PARAMS;
}
// Streaming mode (on_chunk non NULL) emits through the callback and
// leaves out unused, so out=NULL is valid there. Buffered mode
// requires out to receive the synthesised waveform.
if (!params->on_chunk && !out) {
qt_set_error("qt_synthesize: out is NULL in buffered mode");
return QT_STATUS_INVALID_PARAMS;
}
if (params->abi_version > QT_ABI_VERSION) {
qt_set_error(
"qt_synthesize: params->abi_version %d > QT_ABI_VERSION %d (binding compiled against a newer header)",
params->abi_version, QT_ABI_VERSION);
qt_audio_free(out);
if (out) {
qt_audio_free(out);
}
return QT_STATUS_INVALID_PARAMS;
}
@@ -301,104 +321,79 @@ enum qt_status qt_synthesize(struct qt_context * q, const struct qt_tts_params *
const std::string & mt = q->pt.model_type;
if (params->speaker && mt != "custom_voice") {
qt_set_error("--speaker is only valid for custom_voice models (loaded: %s)", mt.c_str());
qt_audio_free(out);
if (out) {
qt_audio_free(out);
}
return QT_STATUS_MODE_INVALID;
}
if (params->instruct && mt == "base") {
qt_set_error("--instruct is not supported for base models");
qt_audio_free(out);
if (out) {
qt_audio_free(out);
}
return QT_STATUS_MODE_INVALID;
}
if (mt == "custom_voice" && !params->speaker) {
qt_set_error("custom_voice models require --speaker");
qt_audio_free(out);
if (out) {
qt_audio_free(out);
}
return QT_STATUS_MODE_INVALID;
}
if (mt == "voice_design" && (!params->instruct || params->instruct[0] == '\0')) {
qt_set_error("voice_design models require --instruct");
qt_audio_free(out);
if (out) {
qt_audio_free(out);
}
return QT_STATUS_MODE_INVALID;
}
if (params->ref_audio_24k && mt != "base") {
qt_set_error("--ref-wav is only valid for base models (loaded: %s)", mt.c_str());
qt_audio_free(out);
if (out) {
qt_audio_free(out);
}
return QT_STATUS_MODE_INVALID;
}
if (params->speaker && params->ref_audio_24k) {
qt_set_error("--speaker and --ref-wav are mutually exclusive");
qt_audio_free(out);
if (out) {
qt_audio_free(out);
}
return QT_STATUS_INVALID_PARAMS;
}
if (params->ref_text && !params->ref_audio_24k) {
qt_set_error("--ref-text requires --ref-wav");
qt_audio_free(out);
if (out) {
qt_audio_free(out);
}
return QT_STATUS_INVALID_PARAMS;
}
// Translate the public POD params into the internal C++ struct
// expected by pipeline_tts_synthesize. Borrowed pointers are
// forwarded verbatim; the lifetime contract on the public side
// (caller keeps strings alive for the duration of the call)
// matches what the pipeline already requires.
PipelineTTSSynthesizeParams p = {};
p.text = params->text;
p.lang = params->lang;
p.instruct = params->instruct;
p.speaker = params->speaker;
p.ref_audio_24k = params->ref_audio_24k;
p.ref_n_samples = params->ref_n_samples;
p.ref_text = params->ref_text;
p.seed = qt_resolve_seed(params->seed);
p.max_new_tokens = params->max_new_tokens;
p.do_sample = params->do_sample;
p.temperature = params->temperature;
p.top_k = params->top_k;
p.top_p = params->top_p;
p.repetition_penalty = params->repetition_penalty;
p.subtalker_do_sample = params->subtalker_do_sample;
p.subtalker_temperature = params->subtalker_temperature;
p.subtalker_top_k = params->subtalker_top_k;
p.subtalker_top_p = params->subtalker_top_p;
p.dump_dir = params->dump_dir;
// Defense in depth: the synthesis path normally reports failures
// via bool return + qt_set_error. A future load-style throw or any
// std::bad_alloc deep inside the GGML backend is caught here and
// converted to QT_STATUS_GENERATE_FAILED so an exception never
// via qt_status return + qt_set_error. A future load-style throw or
// any std::bad_alloc deep inside the GGML backend is caught here
// and converted to QT_STATUS_GENERATE_FAILED so an exception never
// crosses the extern "C" boundary.
try {
PipelineTTSSynthesizeOutput pout;
if (!pipeline_tts_synthesize(&q->pt, &q->tok, p, &pout)) {
qt_audio_free(out);
return QT_STATUS_GENERATE_FAILED;
}
// Copy the std::vector<float> into a malloc-backed buffer the
// caller can free with std::free via qt_audio_free. The
// vector itself goes out of scope at function exit, releasing
// its own storage independently.
const size_t n = pout.audio.size();
const size_t bytes = n * sizeof(float);
float * buf = (float *) std::malloc(bytes > 0 ? bytes : 1);
if (!buf) {
qt_set_error("qt_synthesize: malloc failed for %zu samples", n);
qt_audio_free(out);
return QT_STATUS_OOM;
}
if (n > 0) {
std::memcpy(buf, pout.audio.data(), bytes);
}
out->samples = buf;
out->n_samples = (int) n;
out->sample_rate = pout.sample_rate;
out->channels = 1;
return QT_STATUS_OK;
const int64_t resolved_seed = qt_resolve_seed(params->seed);
return pipeline_tts_synthesize(&q->pt, &q->tok, params, resolved_seed, out);
} catch (const std::exception & e) {
qt_set_error("%s", e.what());
qt_log(QT_LOG_ERROR, "[Qwen] %s", e.what());
qt_audio_free(out);
if (out) {
qt_audio_free(out);
}
return QT_STATUS_GENERATE_FAILED;
}
}
int qt_duration_sec_to_tokens(const struct qt_context * q, float duration_sec) {
if (!q) {
qt_set_error("qt_duration_sec_to_tokens: q is NULL");
qt_log(QT_LOG_ERROR, "[Qwen] qt_duration_sec_to_tokens requires a valid handle");
return 1;
}
return pipeline_tts_duration_sec_to_tokens(&q->pt, duration_sec);
}
} // extern "C"
src/qwen.h
+69 -12
View File
@@ -56,8 +56,15 @@ extern "C" {
// There is no separate semver triple. The runtime build identity is the
// git short hash + commit date string returned by qt_version(); for
// binding compat checks, QT_ABI_VERSION is the only number that
// matters. Aligned on OV_ABI_VERSION = 2 for the omnivoice ABI cousin.
#define QT_ABI_VERSION 2
// matters.
#define QT_ABI_VERSION 1
// Codec sample rate. The 12 Hz Qwen3-TTS audio tokenizer always produces
// 24 kHz mono PCM through its DAC v2 decoder. Exposed at the ABI so a
// caller that needs to resample a reference WAV before passing it via
// qt_tts_params.ref_audio_24k can do so without pulling in any internal
// header. The constant is immutable for this model family.
#define QT_CODEC_SAMPLE_RATE 24000
// Returns a static string of the form "<git-hash> (<date>)" identifying
// the exact commit this binary was built from. Safe to call from any
@@ -72,6 +79,7 @@ enum qt_status {
QT_STATUS_MODE_INVALID = -2,
QT_STATUS_GENERATE_FAILED = -3,
QT_STATUS_OOM = -4,
QT_STATUS_CANCELLED = -5,
};
// Returns the last error message produced on the calling thread by any
@@ -93,7 +101,7 @@ QT_API const char * qt_last_error(void);
struct qt_audio {
float * samples; // mono PCM, malloc allocated
int n_samples; // length in samples
int sample_rate; // 24000 for the 12 Hz Qwen3-TTS tokenizer
int sample_rate; // QT_CODEC_SAMPLE_RATE (24000)
int channels; // 1 (mono)
};
@@ -109,16 +117,21 @@ struct qt_context;
// custom_voice / voice_design checkpoints) the speaker encoder; the
// codec GGUF holds the 12 Hz audio tokenizer. abi_version stays first
// so a future struct growth keeps reading the version field at offset
// 0. No use_fa / clamp_fp16 yet: the current pipeline_tts_load picks
// flash attention from backend capability without a user knob.
// 0. use_fa enables fused flash attention in the Talker and Code
// Predictor forwards when a GPU backend is present (CPU always uses the
// F32 manual chain); clamp_fp16 inserts ggml_clamp(-65504, 65504) on V
// before attention and on the residual stream between blocks to guard
// FP16 matmul accumulation on sub Ampere CUDA targets.
struct qt_init_params {
int abi_version;
const char * talker_path;
const char * codec_path;
bool use_fa;
bool clamp_fp16;
};
// Initialise to the standard defaults: both paths NULL (caller must set
// them before calling qt_init), abi_version set to QT_ABI_VERSION.
// them before calling qt_init), use_fa true, clamp_fp16 false.
QT_API void qt_init_default_params(struct qt_init_params * p);
// Allocate every module described by params. Returns NULL on any
@@ -131,6 +144,27 @@ QT_API struct qt_context * qt_init(const struct qt_init_params * params);
// Safe on NULL.
QT_API void qt_free(struct qt_context * q);
// Cooperative cancellation callback. Returns true to request the
// synthesis to abort. Polled at the top of every Talker decode step in
// the autoregressive loop, so the cancel granularity is roughly one
// audio frame, i.e. 1 / 12 Hz ~ 83 ms.
typedef bool (*qt_cancel_cb)(void * user_data);
// Streaming output callback. When set on qt_tts_params, the synth
// pipeline runs in streaming mode: audio is decoded chunk by chunk from
// the AR codec frames and emitted through this callback rather than
// accumulated into the `out` buffer of qt_synthesize. Returning false
// aborts the synthesis with QT_STATUS_CANCELLED, identical to the
// qt_cancel_cb behaviour. The samples pointer is mono float PCM at
// QT_CODEC_SAMPLE_RATE; valid only for the duration of the call.
// user_data is forwarded verbatim from on_chunk_user_data.
//
// The chunk granularity is driven by chunk_duration_sec in qt_tts_params:
// once the AR loop has produced enough frames to cover that duration,
// the codec decodes that bundle and emits it. The last chunk on EOS /
// max_new flushes whatever frames remain.
typedef bool (*qt_audio_chunk_cb)(const float * samples, int n_samples, void * user_data);
// Log severity. Numerically ordered so a callback can filter with a
// simple `if (level < threshold) return;`. ERROR is reserved for
// failure reports that the lib also surfaces via qt_status /
@@ -184,8 +218,8 @@ struct qt_tts_params {
// Optional voice reference for base mode voice cloning. Mode A
// (x_vector_only) sets ref_audio_24k only; mode B (ICL) sets
// both ref_audio_24k and ref_text. ref_audio_24k is a mono float
// PCM buffer sampled at the codec sample rate (24 kHz). Mutually
// exclusive with speaker. Rejected for custom_voice / voice_design.
// PCM buffer sampled at QT_CODEC_SAMPLE_RATE. Mutually exclusive
// with speaker. Rejected for custom_voice / voice_design.
const float * ref_audio_24k;
int ref_n_samples;
const char * ref_text;
@@ -211,22 +245,45 @@ struct qt_tts_params {
// Intermediate tensor dump directory. NULL disables dumps. Debug
// only, slows the run.
const char * dump_dir;
// Cooperative cancellation. cancel NULL disables the feature.
// cancel_user_data is forwarded to the callback verbatim. Polled
// at the top of every Talker decode step (~83 ms granularity).
qt_cancel_cb cancel;
void * cancel_user_data;
// Streaming output. When on_chunk is non NULL, qt_synthesize runs
// the streaming pipeline: audio chunks emit through on_chunk and
// `out` stays empty on success. on_chunk NULL keeps the buffered
// path. chunk_duration_sec drives the chunk size at codec sample
// rate; values <= 0 fall back to 1.0 second. The last chunk on
// EOS or max_new flushes whatever frames remain.
qt_audio_chunk_cb on_chunk;
void * on_chunk_user_data;
float chunk_duration_sec;
};
// Initialise to the standard defaults. Strings NULL, seed -1,
// max_new_tokens 2048, do_sample true, temperature 0.9, top_k 50,
// top_p 1.0, repetition_penalty 1.05, subtalker mirrors talker,
// dump_dir NULL.
// dump_dir NULL, cancel NULL, on_chunk NULL, chunk_duration_sec 1.0.
QT_API void qt_tts_default_params(struct qt_tts_params * p);
// Run the full TTS synthesis. Validates the params against the loaded
// model_type (the seven base / custom_voice / voice_design rules),
// resolves the seed, hands off to pipeline_tts_synthesize and fills
// `out` with mono float PCM at the codec sample rate. Returns
// QT_STATUS_OK on success; on any failure returns a negative
// qt_status describing the cause and leaves `out` empty.
// `out` with mono float PCM at QT_CODEC_SAMPLE_RATE in buffered mode.
// In streaming mode (params->on_chunk != NULL), audio is emitted
// through the callback and `out` stays empty. Returns QT_STATUS_OK on
// success; on any failure returns a negative qt_status describing the
// cause and leaves `out` empty.
QT_API enum qt_status qt_synthesize(struct qt_context * q, const struct qt_tts_params * params, struct qt_audio * out);
// Convert a duration in seconds to a frame count using the codec
// frame rate (QT_CODEC_SAMPLE_RATE / TOKENIZER_HOP_LENGTH = 12.5 Hz).
// Clamps to a minimum of one frame.
QT_API int qt_duration_sec_to_tokens(const struct qt_context * q, float duration_sec);
#ifdef __cplusplus
}
#endif
src/talker-forward.h
+70 -12
View File
@@ -75,11 +75,38 @@ static bool talker_is_bisect_layer(int l) {
return false;
}
// Manual F32 attention chain. Used when use_flash_attn is false: GQA
// scaled dot product with explicit mul_mat / soft_max_ext / mul_mat,
// FP32 accumulators end to end. Mirrors the qwen3_attn_f32 helper in
// omnivoice.cpp/src/qwen3-enc.h. Inputs and output stay in the same
// layout flash_attn_ext expects: q [hd, T, n_q_heads], k/v [hd, T_full,
// n_kv], output [hd, n_q_heads, T]; the caller reshapes to
// [n_q_heads * hd, T] before o_proj exactly as on the FA path.
static struct ggml_tensor * talker_attn_f32(struct ggml_context * ctx,
struct ggml_tensor * q,
struct ggml_tensor * k,
struct ggml_tensor * v,
struct ggml_tensor * mask,
float scale) {
struct ggml_tensor * scores = ggml_mul_mat(ctx, k, q);
scores = ggml_soft_max_ext(ctx, scores, mask, scale, 0.0f);
struct ggml_tensor * vt = ggml_cont(ctx, ggml_transpose(ctx, v));
struct ggml_tensor * out = ggml_mul_mat(ctx, vt, scores);
return ggml_cont(ctx, ggml_permute(ctx, out, 0, 2, 1, 3));
}
// Build the per-layer block, KV cached. K and V for the T fresh
// positions get computed normally, then written into the cache at
// [n_past, n_past+T) on dim 1. The attention reads the contiguous slice
// [0, n_past+T) on the same dim, which covers the full causal context
// in one tensor view. Returns the layer output [hidden, T].
//
// use_flash_attn picks between the fused ggml_flash_attn_ext kernel
// (GPU only, FP16 accumulation guarded with set_prec(F32)) and the
// manual F32 chain. clamp_fp16 inserts ggml_clamp(-65504, 65504) on V
// before attention and on the residual stream after the attention and
// MLP adds, protecting sub Ampere CUDA tensor cores that accumulate in
// FP16 from overflow.
static struct ggml_tensor * talker_layer_forward(struct ggml_context * ctx,
const TalkerWeights * tw,
const TalkerLayer & layer,
@@ -90,6 +117,8 @@ static struct ggml_tensor * talker_layer_forward(struct ggml_context * ctx,
struct ggml_tensor * v_cache,
int n_past,
int T,
bool use_flash_attn,
bool clamp_fp16,
struct ggml_cgraph * gf) {
const int n_q_heads = tw->num_attention_heads;
const int n_kv = tw->num_key_value_heads;
@@ -156,16 +185,32 @@ static struct ggml_tensor * talker_layer_forward(struct ggml_context * ctx,
// No cont: flash_attn_ext takes the view directly, like acestep does.
struct ggml_tensor * q_p = ggml_permute(ctx, q, 0, 2, 1, 3);
// Fused flash attention. The manual mul_mat + soft_max_ext + mul_mat
// path it replaces had a Vulkan bug in autoregressive decode (T=1)
// where the KV cache view stride on dim 2 (= max_T * hd, non
// Clamp V before attention. Sub Ampere tensor cores accumulate in
// FP16 and the V projection can overflow to inf, which corrupts
// every subsequent attention. ggml_clamp is a no op on the F32
// manual path but stays here to keep both branches numerically
// aligned when the user opts into clamp_fp16.
if (clamp_fp16) {
v_full = ggml_clamp(ctx, v_full, -65504.0f, 65504.0f);
}
// Attention: fused flash kernel (FP16 accumulation, set_prec(F32)
// promotes the softmax / V matmul accumulator back to F32) or
// manual F32 chain. The fused path replaces a mul_mat + soft_max_ext
// + mul_mat sequence that had a Vulkan bug in autoregressive decode
// (T=1) where the KV cache view stride on dim 2 (= max_T * hd, non
// contiguous with dim 1 of length T_full) caused the second mul_mat
// to diverge silently. Flash attention has a backend-tested kernel
// on every target (CPU, CUDA, Vulkan), matches the working acestep
// qw3lm_build_attn pattern.
// to diverge silently. The manual path is the F32 reference, used
// when use_flash_attn is false (CPU only backends, or explicit
// request from the user).
float scale = 1.0f / sqrtf((float) hd);
struct ggml_tensor * attn = ggml_flash_attn_ext(ctx, q_p, k_full, v_full, mask, scale, 0.0f, 0.0f);
ggml_flash_attn_ext_set_prec(attn, GGML_PREC_F32);
struct ggml_tensor * attn;
if (use_flash_attn) {
attn = ggml_flash_attn_ext(ctx, q_p, k_full, v_full, mask, scale, 0.0f, 0.0f);
ggml_flash_attn_ext_set_prec(attn, GGML_PREC_F32);
} else {
attn = talker_attn_f32(ctx, q_p, k_full, v_full, mask, scale);
}
// Flash attention output is [hd, n_q_heads, T], flatten heads for o_proj.
attn = ggml_reshape_2d(ctx, attn, n_q_heads * hd, T);
@@ -173,6 +218,9 @@ static struct ggml_tensor * talker_layer_forward(struct ggml_context * ctx,
struct ggml_tensor * o = ggml_mul_mat(ctx, layer.attn.o_proj_w, attn);
x = ggml_add(ctx, x, o);
if (clamp_fp16) {
x = ggml_clamp(ctx, x, -65504.0f, 65504.0f);
}
// MLP block: pre-norm + SwiGLU + residual
struct ggml_tensor * h2 = ggml_rms_norm(ctx, x, eps);
@@ -185,19 +233,25 @@ static struct ggml_tensor * talker_layer_forward(struct ggml_context * ctx,
struct ggml_tensor * mlp = ggml_mul_mat(ctx, layer.mlp.down_proj_w, gu);
x = ggml_add(ctx, x, mlp);
if (clamp_fp16) {
x = ggml_clamp(ctx, x, -65504.0f, 65504.0f);
}
return x;
}
// Shared core that builds the graph, allocates, uploads inputs, runs
// it and pulls out the last position hidden + logits. T tokens are
// appended to the cache starting at n_past. When n_past == 0 and
// dump_dir is set, the bisect taps fire on the prefill path.
// dump_dir is set, the bisect taps fire on the prefill path. use_fa /
// clamp_fp16 are forwarded as is to every layer.
static bool talker_forward_core(const TalkerWeights * tw,
KVCache * kv,
ggml_backend_sched_t sched,
const float * input_embed,
int T,
int n_past,
bool use_flash_attn,
bool clamp_fp16,
const char * dump_dir,
TalkerForwardOutput * out) {
const int hidden = tw->hidden_size;
@@ -242,7 +296,7 @@ static bool talker_forward_core(const TalkerWeights * tw,
std::vector<struct ggml_tensor *> taps(TALKER_N_BISECT_LAYERS, NULL);
for (int l = 0; l < n_layers; l++) {
h = talker_layer_forward(gctx, tw, tw->layers[(size_t) l], h, pos_in, mask_in, kv->k[(size_t) l],
kv->v[(size_t) l], n_past, T, gf);
kv->v[(size_t) l], n_past, T, use_flash_attn, clamp_fp16, gf);
if (record_taps && talker_is_bisect_layer(l)) {
for (int i = 0; i < TALKER_N_BISECT_LAYERS; i++) {
if (TALKER_BISECT_LAYERS[i] == l) {
@@ -374,6 +428,8 @@ static bool talker_forward_prefill(const TalkerWeights * tw,
ggml_backend_sched_t sched,
const float * input_embed,
int T,
bool use_flash_attn,
bool clamp_fp16,
const char * dump_dir,
TalkerForwardOutput * out) {
kv_cache_reset(kv);
@@ -381,7 +437,7 @@ static bool talker_forward_prefill(const TalkerWeights * tw,
fprintf(stderr, "[TalkerForward] FATAL: prefill T=%d exceeds cache max_seq_len=%d\n", T, kv->max_seq_len);
return false;
}
return talker_forward_core(tw, kv, sched, input_embed, T, 0, dump_dir, out);
return talker_forward_core(tw, kv, sched, input_embed, T, 0, use_flash_attn, clamp_fp16, dump_dir, out);
}
// Decode: feed exactly one embedding and append one position to the
@@ -391,11 +447,13 @@ static bool talker_forward_decode(const TalkerWeights * tw,
KVCache * kv,
ggml_backend_sched_t sched,
const float * input_embed_1,
bool use_flash_attn,
bool clamp_fp16,
TalkerForwardOutput * out) {
if (kv->cur_len + 1 > kv->max_seq_len) {
fprintf(stderr, "[TalkerForward] FATAL: decode would overflow cache (%d + 1 > %d)\n", kv->cur_len,
kv->max_seq_len);
return false;
}
return talker_forward_core(tw, kv, sched, input_embed_1, 1, kv->cur_len, NULL, out);
return talker_forward_core(tw, kv, sched, input_embed_1, 1, kv->cur_len, use_flash_attn, clamp_fp16, NULL, out);
}
tests/abi-c.c
+49 -19
View File
@@ -1,4 +1,4 @@
/* tests/abi-c.c : link-only ABI smoke test for qwen.h.
/* tests/abi-c.c: link-only ABI smoke test for qwen.h.
*
* Compiled in pure C99 with -Wall -Werror -pedantic. The purpose of this
* test is NOT to run a full synthesis (no GGUF loaded, no model required);
@@ -22,11 +22,23 @@
#include <stdlib.h>
#include <string.h>
static bool stub_cancel(void * ud) {
(void) ud;
return false;
}
static bool stub_on_chunk(const float * samples, int n_samples, void * ud) {
(void) samples;
(void) n_samples;
(void) ud;
return true;
}
/* Counter incremented by the stub log callback. The probe checks that at
* least one log line was routed through the callback by triggering a
* qt_init failure (which emits a [qwen] ERROR line via qt_log). */
static int g_log_lines = 0;
static enum qt_log_level g_last_log_level = QT_LOG_DEBUG;
* qt_init failure (which emits a [Qwen] ERROR line via qt_log). */
static int g_log_lines = 0;
static enum qt_log_level g_last_log_level = QT_LOG_DEBUG;
static char g_last_log_msg[512] = { 0 };
static void stub_log(enum qt_log_level level, const char * msg, void * user_data) {
@@ -55,29 +67,44 @@ int main(void) {
struct qt_tts_params params;
qt_tts_default_params(&params);
/* Sanity-check a few default values, including the abi_version. */
if (params.max_new_tokens != 2048 || params.temperature != 0.9f) {
/* Sanity-check a few default values, including the abi_version and
* the new use_fa / clamp_fp16 / on_chunk / chunk_duration_sec slots. */
if (params.max_new_tokens != 2048 || params.chunk_duration_sec <= 0.0f) {
fprintf(stderr, "[Probe] default values do not match\n");
return 1;
}
if (iparams.abi_version != QT_ABI_VERSION || params.abi_version != QT_ABI_VERSION) {
fprintf(stderr, "[Probe] abi_version not set by qwen_*_default_params\n");
fprintf(stderr, "[Probe] abi_version not set by qt_*_default_params\n");
return 1;
}
if (!iparams.use_fa || iparams.clamp_fp16) {
fprintf(stderr, "[Probe] init_params defaults do not match (use_fa=true, clamp_fp16=false)\n");
return 1;
}
if (QT_CODEC_SAMPLE_RATE != 24000) {
fprintf(stderr, "[Probe] QT_CODEC_SAMPLE_RATE is not 24000\n");
return 1;
}
/* Touch every output struct field so the compiler validates the
* layout end-to-end without ever needing a model. */
/* Touch every reference-pointer field, every callback typedef and
* every output struct field so the compiler validates the layout
* end-to-end without ever needing a model. */
params.cancel = stub_cancel;
params.cancel_user_data = NULL;
params.on_chunk = stub_on_chunk;
params.on_chunk_user_data = NULL;
struct qt_audio audio = { 0 };
qt_audio_free(&audio);
/* Install the log callback before the failing init so the [qwen]
/* Install the log callback before the failing init so the [Qwen]
* ERROR line lands on stub_log instead of stderr. */
qt_log_set(stub_log, NULL);
/* Call every entry through its early-return path. qt_init returns
* NULL on missing talker_path / codec_path, qt_synthesize fails on
* NULL handle, qt_free is safe on NULL. None of these load a model,
* but the linker must resolve every name to satisfy the call. */
* NULL on missing paths, qt_synthesize / qt_duration_sec_to_tokens
* fail on NULL handle, qt_free is safe on NULL. None of these load a
* model, but the linker must resolve every name to satisfy the call. */
struct qt_context * dummy = qt_init(NULL);
if (dummy != NULL) {
fprintf(stderr, "[Probe] qt_init(NULL) was supposed to return NULL\n");
@@ -102,17 +129,14 @@ int main(void) {
return 6;
}
if (g_last_log_level != QT_LOG_ERROR) {
fprintf(stderr, "[Probe] last log level was %d, expected %d\n", (int) g_last_log_level,
(int) QT_LOG_ERROR);
fprintf(stderr, "[Probe] last log level was %d, expected %d\n", (int) g_last_log_level, (int) QT_LOG_ERROR);
return 7;
}
printf("[Probe] qt_log_set routed %d line(s), last: '%s'\n", g_log_lines, g_last_log_msg);
printf("[Probe] qt_last_error reads '%s'\n", err);
/* abi_version validation : a struct claiming a future ABI must be
* rejected up front, before any allocation. Both paths are filled
* with placeholders so the NULL guard does not short-circuit the
* abi_version branch. */
* rejected up front, before any allocation. */
struct qt_init_params future_iparams;
qt_init_default_params(&future_iparams);
future_iparams.talker_path = "irrelevant.gguf";
@@ -132,8 +156,14 @@ int main(void) {
return 3;
}
int frames = qt_duration_sec_to_tokens(NULL, 1.0f);
if (frames < 1) {
fprintf(stderr, "[Probe] qt_duration_sec_to_tokens returned %d, expected >= 1\n", frames);
return 4;
}
/* Restore the default stderr fallback before exit so the trailing
* [qwen] log lines from the cleanup paths land where the user
* [Qwen] log lines from the cleanup paths land where the user
* expects them. */
qt_log_set(NULL, NULL);
tests/debug-clone-cossim.py
+7 -10
View File
@@ -13,7 +13,7 @@ length, the shorter one padded with tts_pad / truncated as needed.
Cote Python the speaker embedding is captured directly via
model.extract_speaker_embedding, and the reference codec frames via
model.speech_tokenizer.encode. Both intermediates land as speaker-emb.bin
model.speech_tokenizer.encode. Both intermediates land as spk-emb.bin
and ref-codes.bin and are compared against the C++ side dumps emitted
by pipeline-tts.cpp when --ref-wav and --ref-text are set.
@@ -38,9 +38,6 @@ CKPT = "../checkpoints/Qwen3-TTS-12Hz-1.7B-Base"
DUMP_CPP = "cpp/clone"
DUMP_PT = "python/clone"
DEFAULT_REF_AUDIO = "../examples/freeman.wav"
DEFAULT_REF_TEXT = "../examples/freeman.txt"
# Mode B adds two pre-talker stages to the standard list : the speaker
# embedding extracted from the reference audio (ECAPA forward, projected to
# talker hidden), and the reference codec frames at 12.5 Hz. Plus three
@@ -65,7 +62,7 @@ STAGES_CLONE = cc.STAGES_STANDARD + [
("SpkBlock3", "spk-block3.bin"),
("SpkMFA", "spk-mfa.bin"),
("SpkASP", "spk-asp.bin"),
("SpeakerEmb", "speaker-emb.bin"),
("SpeakerEmb", "spk-emb.bin"),
]
def install_clone_hooks(model, dump_dir):
@@ -271,9 +268,9 @@ def dump_mel_mag_python(ref_wav, dump_dir):
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--prompt", default="../examples/prompt.txt")
ap.add_argument("--ref-wav", default=DEFAULT_REF_AUDIO,
ap.add_argument("--ref-wav", default="../examples/freeman.wav",
help="reference WAV path for voice cloning")
ap.add_argument("--ref-text-file", default=DEFAULT_REF_TEXT,
ap.add_argument("--ref-text", default="../examples/freeman.txt",
help="path to a UTF-8 file with the transcript of ref-wav")
ap.add_argument("--seed", type=int, default=42)
ap.add_argument("--lang", default="english")
@@ -297,7 +294,7 @@ def main():
with open(args.prompt, "r", encoding="utf-8") as f:
text = f.read().strip()
with open(args.ref_text_file, "r", encoding="utf-8") as f:
with open(args.ref_text, "r", encoding="utf-8") as f:
ref_text = f.read().strip()
print(f"[Input] Prompt: {len(text)} chars: {text[:60]}{'...' if len(text) > 60 else ''}")
print(f"[Input] RefAudio: {args.ref_wav}")
@@ -356,7 +353,7 @@ def main():
# Extract speaker embedding via ECAPA forward, projected to talker hidden.
spk_emb = model.extract_speaker_embedding(audio=ref_wav, sr=ref_sr)
print(f"[Python] SpeakerEmb shape: {tuple(spk_emb.shape)} dtype: {spk_emb.dtype}")
cc.save_dump(os.path.join(DUMP_PT, "speaker-emb.bin"), spk_emb)
cc.save_dump(os.path.join(DUMP_PT, "spk-emb.bin"), spk_emb)
# Encode the reference audio to 16 codebook codes at 12.5 Hz. The encode
# call returns shape [T_codec, K=16] after the internal transpose, while
@@ -460,7 +457,7 @@ def main():
"--seed", str(args.seed),
"--text", text,
"--ref-wav", args.ref_wav,
"--ref-text", ref_text,
"--ref-text", args.ref_text,
"--lang", args.lang,
"--max-new", str(args.max_new_tokens),
"--dump", DUMP_CPP,
tests/debug-customvoice-cossim.py
+1 -3
View File
@@ -35,12 +35,10 @@ CKPT = "../checkpoints/Qwen3-TTS-12Hz-1.7B-CustomVoice"
DUMP_CPP = "cpp/customvoice"
DUMP_PT = "python/customvoice"
DEFAULT_SPEAKER = "vivian"
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--prompt", default="../examples/prompt.txt")
ap.add_argument("--speaker", default=DEFAULT_SPEAKER,
ap.add_argument("--speaker", default="vivian",
help="speaker preset key (lowercase), validated by the model")
ap.add_argument("--instruct", default="",
help="optional style instruction, empty disables the instruct prefix")
tests/debug-tts-cossim.py
+1 -3
View File
@@ -32,12 +32,10 @@ CKPT = "../checkpoints/Qwen3-TTS-12Hz-1.7B-VoiceDesign"
DUMP_CPP = "cpp/tts"
DUMP_PT = "python/tts"
DEFAULT_INSTRUCT = "male, young adult, moderate pitch"
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--prompt", default="../examples/prompt.txt")
ap.add_argument("--instruct", default=DEFAULT_INSTRUCT,
ap.add_argument("--instruct", default="male, young adult, moderate pitch",
help="natural language style instruction")
ap.add_argument("--seed", type=int, default=42)
ap.add_argument("--lang", default="english")
tools/qwen-tts.cpp
+14 -2
View File
@@ -13,7 +13,6 @@
// --text or stdin if --text is absent.
#include "audio-io.h"
#include "pipeline-codec.h"
#include "qwen.h"
#include <cstdio>
@@ -58,6 +57,9 @@ static void print_usage(const char * prog) {
" --sub-temp <f> Sub-talker temperature (default: 0.9)\n"
" --sub-top-k <n> Sub-talker top-k (default: 50)\n"
" --sub-top-p <f> Sub-talker top-p (default: 1.0)\n\n"
"Backend options:\n"
" --no-fa Disable flash attention (manual F32 attention chain)\n"
" --clamp-fp16 Clamp hidden states + V to FP16 range (sub Ampere CUDA)\n\n"
"Debug:\n"
" --dump <dir> Dump intermediate tensors for cossim debug\n",
prog);
@@ -86,6 +88,8 @@ struct Args {
float subtalker_top_p;
float subtalker_temperature;
bool subtalker_do_sample;
bool use_fa;
bool clamp_fp16;
};
// Read all of stdin into a string. Trims trailing newlines so a piped
@@ -143,6 +147,8 @@ static bool parse_args(int argc, char ** argv, Args & a) {
a.subtalker_top_k = 50;
a.subtalker_top_p = 1.0f;
a.subtalker_temperature = 0.9f;
a.use_fa = true;
a.clamp_fp16 = false;
for (int i = 1; i < argc; i++) {
const char * arg = argv[i];
if (std::strcmp(arg, "-h") == 0 || std::strcmp(arg, "--help") == 0) {
@@ -194,6 +200,10 @@ static bool parse_args(int argc, char ** argv, Args & a) {
a.subtalker_top_k = std::atoi(argv[++i]);
} else if (std::strcmp(arg, "--sub-top-p") == 0 && i + 1 < argc) {
a.subtalker_top_p = (float) std::atof(argv[++i]);
} else if (std::strcmp(arg, "--no-fa") == 0) {
a.use_fa = false;
} else if (std::strcmp(arg, "--clamp-fp16") == 0) {
a.clamp_fp16 = true;
} else if (std::strcmp(arg, "-o") == 0 && i + 1 < argc) {
a.out_wav = argv[++i];
} else {
@@ -213,6 +223,8 @@ static int run(const Args & a) {
qt_init_default_params(&iparams);
iparams.talker_path = a.model;
iparams.codec_path = a.codec;
iparams.use_fa = a.use_fa;
iparams.clamp_fp16 = a.clamp_fp16;
qt_context * q = qt_init(&iparams);
if (!q) {
@@ -246,7 +258,7 @@ static int run(const Args & a) {
int ref_n_samples = 0;
if (a.ref_wav) {
int T_in = 0;
float * raw = audio_read_mono(a.ref_wav, TOKENIZER_SAMPLE_RATE, &T_in);
float * raw = audio_read_mono(a.ref_wav, QT_CODEC_SAMPLE_RATE, &T_in);
if (!raw || T_in <= 0) {
fprintf(stderr, "[CLI] ERROR: cannot read --ref-wav '%s'\n", a.ref_wav);
if (raw) {