From 38bf6d762a0acc1cbe5d5e1a91276f63d971984d Mon Sep 17 00:00:00 2001 From: Pascal Date: Thu, 14 May 2026 21:42:52 +0200 Subject: [PATCH] abi, pipeline, cli: conform qwentts on omnivoice convention --- src/code-predictor-forward.h | 62 ++++++- src/pipeline-tts.cpp | 284 +++++++++++++++++++++--------- src/pipeline-tts.h | 103 ++++++----- src/qwen.cpp | 159 ++++++++--------- src/qwen.h | 81 +++++++-- src/talker-forward.h | 82 +++++++-- tests/abi-c.c | 68 +++++-- tests/debug-clone-cossim.py | 17 +- tests/debug-customvoice-cossim.py | 4 +- tests/debug-tts-cossim.py | 4 +- tools/qwen-tts.cpp | 16 +- 11 files changed, 602 insertions(+), 278 deletions(-) diff --git a/src/code-predictor-forward.h b/src/code-predictor-forward.h index 487aa61..8f39822 100644 --- a/src/code-predictor-forward.h +++ b/src/code-predictor-forward.h @@ -51,10 +51,27 @@ struct CodePredictorOutput { std::vector codes; }; +// Manual F32 attention chain for the code predictor block. Same shape +// contract as talker_attn_f32: q [hd, T, n_q_heads], k/v [hd, T_full, +// n_kv], output [hd, n_q_heads, T]. Used when use_flash_attn is false. +static struct ggml_tensor * code_predictor_attn_f32(struct ggml_context * ctx, + struct ggml_tensor * q, + struct ggml_tensor * k, + struct ggml_tensor * v, + struct ggml_tensor * mask, + float scale) { + struct ggml_tensor * scores = ggml_mul_mat(ctx, k, q); + scores = ggml_soft_max_ext(ctx, scores, mask, scale, 0.0f); + struct ggml_tensor * vt = ggml_cont(ctx, ggml_transpose(ctx, v)); + struct ggml_tensor * out = ggml_mul_mat(ctx, vt, scores); + return ggml_cont(ctx, ggml_permute(ctx, out, 0, 2, 1, 3)); +} + // One Qwen3 decoder block, KV cached. K and V for the T fresh positions // are written into the cache at [n_past, n_past+T) on dim 1; the // attention reads the contiguous slice [0, n_past+T). Returns the layer -// output [hidden, T]. +// output [hidden, T]. use_flash_attn and clamp_fp16 follow the same +// contract as in talker-forward.h. static struct ggml_tensor * code_predictor_layer_forward(struct ggml_context * ctx, const CodePredictorWeights * cw, const TalkerLayer & layer, @@ -65,6 +82,8 @@ static struct ggml_tensor * code_predictor_layer_forward(struct ggml_context * struct ggml_tensor * v_cache, int n_past, int T, + bool use_flash_attn, + bool clamp_fp16, struct ggml_cgraph * gf) { const int n_q_heads = cw->num_attention_heads; const int n_kv = cw->num_key_value_heads; @@ -114,16 +133,32 @@ static struct ggml_tensor * code_predictor_layer_forward(struct ggml_context * // Q permute [hd, n_q_heads, T] -> [hd, T, n_q_heads] for flash_attn_ext. struct ggml_tensor * q_p = ggml_permute(ctx, q, 0, 2, 1, 3); - // Fused flash attention. Matches the working acestep qw3lm_build_attn - // pattern, fixes the Vulkan autoregressive decode bug. + // Clamp V before attention when clamp_fp16 is set, same rationale + // as the talker block: sub Ampere CUDA tensor cores accumulate in + // FP16 and a V projection overflow corrupts everything downstream. + if (clamp_fp16) { + v_full = ggml_clamp(ctx, v_full, -65504.0f, 65504.0f); + } + + // Attention: fused flash kernel or manual F32 chain. Matches the + // working acestep qw3lm_build_attn pattern on the fused branch and + // the omnivoice qwen3_attn_f32 helper on the manual one. float scale = 1.0f / sqrtf((float) hd); - struct ggml_tensor * attn = ggml_flash_attn_ext(ctx, q_p, k_full, v_full, mask, scale, 0.0f, 0.0f); - ggml_flash_attn_ext_set_prec(attn, GGML_PREC_F32); + struct ggml_tensor * attn; + if (use_flash_attn) { + attn = ggml_flash_attn_ext(ctx, q_p, k_full, v_full, mask, scale, 0.0f, 0.0f); + ggml_flash_attn_ext_set_prec(attn, GGML_PREC_F32); + } else { + attn = code_predictor_attn_f32(ctx, q_p, k_full, v_full, mask, scale); + } attn = ggml_reshape_2d(ctx, attn, n_q_heads * hd, T); struct ggml_tensor * o = ggml_mul_mat(ctx, layer.attn.o_proj_w, attn); x = ggml_add(ctx, x, o); + if (clamp_fp16) { + x = ggml_clamp(ctx, x, -65504.0f, 65504.0f); + } struct ggml_tensor * h2 = ggml_rms_norm(ctx, x, eps); h2 = ggml_mul(ctx, h2, layer.post_attn_norm_w); @@ -135,6 +170,9 @@ static struct ggml_tensor * code_predictor_layer_forward(struct ggml_context * struct ggml_tensor * mlp = ggml_mul_mat(ctx, layer.mlp.down_proj_w, gu); x = ggml_add(ctx, x, mlp); + if (clamp_fp16) { + x = ggml_clamp(ctx, x, -65504.0f, 65504.0f); + } return x; } @@ -142,6 +180,7 @@ static struct ggml_tensor * code_predictor_layer_forward(struct ggml_context * // position `n_past`, run all 5 layers, and pull the logits for the last // position through lm_head[g_head]. The cache is written as a side // effect so subsequent decode steps can append a single token. +// use_flash_attn / clamp_fp16 are forwarded as is to every layer. static bool code_predictor_run(const CodePredictorWeights * cw, KVCache * kv, ggml_backend_sched_t sched, @@ -150,6 +189,8 @@ static bool code_predictor_run(const CodePredictorWeights * cw, int n_past, int talker_hidden, int g_head, + bool use_flash_attn, + bool clamp_fp16, std::vector * logits_out) { const int vocab = cw->vocab_size; const int n_layers = cw->num_hidden_layers; @@ -188,7 +229,7 @@ static bool code_predictor_run(const CodePredictorWeights * cw, for (int l = 0; l < n_layers; l++) { h = code_predictor_layer_forward(gctx, cw, cw->layers[(size_t) l], h, pos_in, mask_in, kv->k[(size_t) l], - kv->v[(size_t) l], n_past, T, gf); + kv->v[(size_t) l], n_past, T, use_flash_attn, clamp_fp16, gf); } struct ggml_tensor * h_final = ggml_rms_norm(gctx, h, cw->rms_norm_eps); @@ -280,6 +321,7 @@ static void embed_row_from_backend(struct ggml_tensor * t, int row_id, int dim, // is the Philox subsequence of the c0 sample for this step; the 15 // acoustic samples consume subseq_base + 1 .. subseq_base + 15. // Returns the full vector of 16 codes. dump_dir may be NULL. +// use_flash_attn / clamp_fp16 are forwarded as is to every internal run. static bool code_predictor_step(const TalkerWeights * tw, const CodePredictorWeights * cw, KVCache * kv, @@ -291,6 +333,8 @@ static bool code_predictor_step(const TalkerWeights * tw, float top_p, int64_t seed, int64_t subseq_base, + bool use_flash_attn, + bool clamp_fp16, const char * dump_dir, CodePredictorOutput * out) { // sub_input slots live at the talker hidden dimension: both the @@ -316,7 +360,8 @@ static bool code_predictor_step(const TalkerWeights * tw, embed_row_from_backend(tw->codec_embedding, c0, talker_hidden, prefill_input.data() + (size_t) talker_hidden); std::vector logits; - if (!code_predictor_run(cw, kv, sched, prefill_input.data(), 2, 0, talker_hidden, 0, &logits)) { + if (!code_predictor_run(cw, kv, sched, prefill_input.data(), 2, 0, talker_hidden, 0, use_flash_attn, clamp_fp16, + &logits)) { return false; } { @@ -340,7 +385,8 @@ static bool code_predictor_step(const TalkerWeights * tw, for (int g = 1; g < n_acoustic; g++) { embed_row_from_backend(cw->codec_embedding[(size_t) (g - 1)], out->codes[(size_t) g], talker_hidden, step_input.data()); - if (!code_predictor_run(cw, kv, sched, step_input.data(), 1, kv->cur_len, talker_hidden, g, &logits)) { + if (!code_predictor_run(cw, kv, sched, step_input.data(), 1, kv->cur_len, talker_hidden, g, use_flash_attn, + clamp_fp16, &logits)) { return false; } float u_g = 0.0f; diff --git a/src/pipeline-tts.cpp b/src/pipeline-tts.cpp index b84e400..466d91b 100644 --- a/src/pipeline-tts.cpp +++ b/src/pipeline-tts.cpp @@ -103,12 +103,24 @@ static void parse_generation_defaults(const GGUFModel & gf, GenerationDefaults & g.max_new_tokens = (int) gf_get_u32(gf, "generation.max_new_tokens"); } -bool pipeline_tts_load(PipelineTTS * pt, const char * talker_gguf_path, const char * codec_gguf_path, BackendPair bp) { +bool pipeline_tts_load(PipelineTTS * pt, + const char * talker_gguf_path, + const char * codec_gguf_path, + BackendPair bp, + bool use_fa, + bool clamp_fp16) { pt->bp = bp; pt->backend = bp.backend; pt->sched = NULL; pt->has_speaker_encoder = false; + // Fused flash attention needs a GPU kernel; CPU only backends fall + // back to the F32 manual chain automatically. clamp_fp16 is forwarded + // verbatim: a no op on backends that already accumulate in F32, an + // FP16 overflow guard on sub Ampere CUDA tensor cores. + pt->use_flash_attn = use_fa && bp.has_gpu; + pt->clamp_fp16 = clamp_fp16; + if (!gf_load(&pt->gguf_talker, talker_gguf_path)) { qt_log(QT_LOG_ERROR, "[Pipeline] failed to load talker GGUF: %s", talker_gguf_path); return false; @@ -217,9 +229,11 @@ bool pipeline_tts_load(PipelineTTS * pt, const char * talker_gguf_path, const ch } qt_log(QT_LOG_INFO, - "[Pipeline] Loaded: arch=%s variant=%s tokenizer=%s codebooks=%d speaker_encoder=%s speakers=%zu", + "[Pipeline] Loaded: arch=%s variant=%s tokenizer=%s codebooks=%d speaker_encoder=%s speakers=%zu fa=%s " + "clamp_fp16=%s", pt->model_size.c_str(), pt->model_type.c_str(), pt->tokenizer_type.c_str(), pt->num_code_groups, - pt->has_speaker_encoder ? "loaded" : "absent", pt->speakers.size()); + pt->has_speaker_encoder ? "loaded" : "absent", pt->speakers.size(), pt->use_flash_attn ? "on" : "off", + pt->clamp_fp16 ? "on" : "off"); return true; } @@ -267,23 +281,56 @@ static void embed_row_from_gguf(const GGUFModel & gf, const char * tensor_name, tt->to_float(row, dst, hidden); } -bool pipeline_tts_synthesize(PipelineTTS * pt, - BPETokenizer * tok, - const PipelineTTSSynthesizeParams & params, - PipelineTTSSynthesizeOutput * out) { - out->audio.clear(); +// Helper: malloc a heap copy of a float vector and hand it off into the +// public qt_audio struct. Returns true on success; on OOM sets the +// error string and leaves out untouched. Empty vectors land as an +// allocation of size 0 with a stub malloc to keep the free path simple. +static bool fill_qt_audio(const std::vector & audio, qt_audio * out) { + const size_t n = audio.size(); + const size_t bytes = n * sizeof(float); + float * buf = (float *) std::malloc(bytes > 0 ? bytes : 1); + if (!buf) { + qt_set_error("pipeline_tts_synthesize: malloc failed for %zu samples", n); + return false; + } + if (n > 0) { + std::memcpy(buf, audio.data(), bytes); + } + out->samples = buf; + out->n_samples = (int) n; out->sample_rate = TOKENIZER_SAMPLE_RATE; + out->channels = 1; + return true; +} +int pipeline_tts_duration_sec_to_tokens(const PipelineTTS * /*pt*/, float duration_sec) { + // The 12 Hz Qwen3-TTS tokenizer has a fixed hop of 1920 samples at + // 24 kHz, so the frame rate is 24000 / 1920 = 12.5 Hz regardless of + // the variant loaded. Clamp to a minimum of one frame so a zero or + // negative duration still picks up one decoder step. + const float fps = (float) TOKENIZER_SAMPLE_RATE / (float) TOKENIZER_HOP_LENGTH; + int n_frames = (int) (duration_sec * fps + 0.5f); + if (n_frames < 1) { + n_frames = 1; + } + return n_frames; +} + +qt_status pipeline_tts_synthesize(PipelineTTS * pt, + BPETokenizer * tok, + const struct qt_tts_params * params, + int64_t resolved_seed, + struct qt_audio * out) { PromptBuilderOutput prompt; - const std::string instruct = params.instruct ? params.instruct : ""; - const std::string speaker = params.speaker ? params.speaker : ""; - const std::string ref_text = params.ref_text ? params.ref_text : ""; + const std::string instruct = params->instruct ? params->instruct : ""; + const std::string speaker = params->speaker ? params->speaker : ""; + const std::string ref_text = params->ref_text ? params->ref_text : ""; // Voice clone mode A: if ref_audio_24k is given, run the speaker // encoder on the pre-decoded mono buffer and feed the resulting // embedding straight into the prompt builder. Mutually exclusive // with --speaker. - const bool has_ref_audio = (params.ref_audio_24k != NULL) && (params.ref_n_samples > 0); + const bool has_ref_audio = (params->ref_audio_24k != NULL) && (params->ref_n_samples > 0); std::vector ref_spk_emb; const float * ref_spk_emb_ptr = NULL; if (has_ref_audio) { @@ -291,18 +338,18 @@ bool pipeline_tts_synthesize(PipelineTTS * pt, qt_set_error( "pipeline_tts_synthesize: --ref-wav requires a model with a loaded speaker encoder (Base only)"); qt_log(QT_LOG_ERROR, "[Pipeline] --ref-wav requires a model with a loaded speaker encoder (Base only)"); - return false; + return QT_STATUS_GENERATE_FAILED; } - if (!speaker_encoder_extract(&pt->speaker_encoder, pt->sched, params.ref_audio_24k, params.ref_n_samples, - ref_spk_emb, params.dump_dir)) { - return false; + if (!speaker_encoder_extract(&pt->speaker_encoder, pt->sched, params->ref_audio_24k, params->ref_n_samples, + ref_spk_emb, params->dump_dir)) { + return QT_STATUS_GENERATE_FAILED; } if ((int) ref_spk_emb.size() != pt->talker.hidden_size) { qt_set_error("pipeline_tts_synthesize: speaker embedding size %zu mismatches talker hidden %d", ref_spk_emb.size(), pt->talker.hidden_size); qt_log(QT_LOG_ERROR, "[Pipeline] speaker embedding size %zu mismatches talker hidden %d", ref_spk_emb.size(), pt->talker.hidden_size); - return false; + return QT_STATUS_GENERATE_FAILED; } ref_spk_emb_ptr = ref_spk_emb.data(); } @@ -318,35 +365,35 @@ bool pipeline_tts_synthesize(PipelineTTS * pt, if (!has_ref_audio) { qt_set_error("pipeline_tts_synthesize: --ref-text requires --ref-wav"); qt_log(QT_LOG_ERROR, "[Pipeline] --ref-text requires --ref-wav"); - return false; + return QT_STATUS_INVALID_PARAMS; } // The codec hop is 1920 samples at 24 kHz so n_samples must be // a multiple of 1920. Truncate to the nearest hop boundary. - if (params.ref_n_samples < TOKENIZER_HOP_LENGTH) { - qt_set_error("pipeline_tts_synthesize: ref_wav too short for ICL (%d samples)", params.ref_n_samples); - qt_log(QT_LOG_ERROR, "[Pipeline] ref_wav too short for ICL (%d samples)", params.ref_n_samples); - return false; + if (params->ref_n_samples < TOKENIZER_HOP_LENGTH) { + qt_set_error("pipeline_tts_synthesize: ref_wav too short for ICL (%d samples)", params->ref_n_samples); + qt_log(QT_LOG_ERROR, "[Pipeline] ref_wav too short for ICL (%d samples)", params->ref_n_samples); + return QT_STATUS_INVALID_PARAMS; } - int aligned_T = (params.ref_n_samples / TOKENIZER_HOP_LENGTH) * TOKENIZER_HOP_LENGTH; - ref_codes = pipeline_codec_encode(&pt->codec, params.ref_audio_24k, aligned_T, params.dump_dir); + int aligned_T = (params->ref_n_samples / TOKENIZER_HOP_LENGTH) * TOKENIZER_HOP_LENGTH; + ref_codes = pipeline_codec_encode(&pt->codec, params->ref_audio_24k, aligned_T, params->dump_dir); if (ref_codes.empty()) { qt_set_error("pipeline_tts_synthesize: pipeline_codec_encode returned empty codes"); qt_log(QT_LOG_ERROR, "[Pipeline] pipeline_codec_encode returned empty codes"); - return false; + return QT_STATUS_GENERATE_FAILED; } ref_codes_T = (int) ref_codes.size() / pt->num_code_groups; qt_log(QT_LOG_INFO, "[Pipeline] ICL ref_codes: %d frames at 12.5 Hz (%d audio samples)", ref_codes_T, aligned_T); } - if (!prompt_builder_build(pt, tok, params.text, params.lang, instruct, speaker, ref_spk_emb_ptr, ref_text, + if (!prompt_builder_build(pt, tok, params->text, params->lang, instruct, speaker, ref_spk_emb_ptr, ref_text, ref_codes_T > 0 ? ref_codes.data() : NULL, ref_codes_T, &prompt)) { - return false; + return QT_STATUS_GENERATE_FAILED; } - if (params.dump_dir) { + if (params->dump_dir) { DebugDumper d; - debug_init(&d, params.dump_dir); + debug_init(&d, params->dump_dir); std::vector ids32(prompt.prompt_ids.begin(), prompt.prompt_ids.end()); int n_ids = (int) ids32.size(); debug_dump_i32_as_f32(&d, "prompt-ids", ids32.data(), &n_ids, 1); @@ -354,12 +401,12 @@ bool pipeline_tts_synthesize(PipelineTTS * pt, debug_dump_2d(&d, "trailing-text-hidden", prompt.trailing_text_hidden.data(), prompt.T_trailing, prompt.hidden); debug_dump_1d(&d, "tts-pad-embed", prompt.tts_pad_embed.data(), prompt.hidden); - // Voice clone dumps: speaker-emb fires when ref_wav is set + // Voice clone dumps: spk-emb fires when ref_wav is set // (modes A and B), ref-codes fires only when ref_text is also set // (mode B ICL). Both are no-ops in base / tts / customvoice modes, // the dump files simply do not appear in those runs. if (ref_spk_emb_ptr != NULL) { - debug_dump_1d(&d, "speaker-emb", ref_spk_emb_ptr, pt->talker.hidden_size); + debug_dump_1d(&d, "spk-emb", ref_spk_emb_ptr, pt->talker.hidden_size); } if (ref_codes_T > 0) { const int shape[2] = { pt->num_code_groups, ref_codes_T }; @@ -367,34 +414,35 @@ bool pipeline_tts_synthesize(PipelineTTS * pt, } } - // Generation loop: at each step we recompute the full Talker prefix - // (no KV cache yet) over the prompt prefix concatenated with all the - // next-token embeddings produced so far, sample c0, run the code - // predictor for the 15 acoustic codes, build the next-token - // embedding by summing the 16 codebook embeddings and the matching - // trailing-text overlay, and append it to the running context. We - // stop on codec_eos or when max_new_tokens is reached. - const int hidden = prompt.hidden; - const int codec_eos_id = pt->codec_specials.eos_id; - const int num_codebooks = pt->num_code_groups; - const int talker_vocab = pt->talker.vocab_size; - - // Greedy collapses to temperature <= 0 in sample_top_k_p. - float talker_T = params.do_sample ? params.temperature : 0.0f; - float subtk_T = params.subtalker_do_sample ? params.subtalker_temperature : 0.0f; - float talker_rp = params.repetition_penalty; - // Generation loop: step 0 prefills the talker over the full prompt // and writes T_ctx positions into the KV cache. Subsequent steps // feed one next_emb at a time and append one position. The code // predictor maintains its own per-frame cache that gets reset at // every step. + const int hidden = prompt.hidden; + const int codec_eos_id = pt->codec_specials.eos_id; + const int num_codebooks = pt->num_code_groups; + const int talker_vocab = pt->talker.vocab_size; + const bool use_fa = pt->use_flash_attn; + const bool clamp_fp16 = pt->clamp_fp16; + const float talker_T = params->do_sample ? params->temperature : 0.0f; + const float subtk_T = params->subtalker_do_sample ? params->subtalker_temperature : 0.0f; + const float talker_rp = params->repetition_penalty; + + // Streaming config: when on_chunk is set, decode the codec every + // chunk_frames AR frames and emit through the callback. The buffered + // path keeps every code, decodes once at the very end and copies the + // resulting audio into out. chunk_frames <= 0 falls back to 1 second. + const bool streaming = (params->on_chunk != NULL); + const float chunk_sec = params->chunk_duration_sec > 0.0f ? params->chunk_duration_sec : 1.0f; + const int chunk_frames = pipeline_tts_duration_sec_to_tokens(pt, chunk_sec); + std::vector> all_codes; - all_codes.reserve((size_t) params.max_new_tokens); + all_codes.reserve((size_t) params->max_new_tokens); // c0 codes already emitted, fed to repetition penalty. std::vector talker_history; - talker_history.reserve((size_t) params.max_new_tokens); + talker_history.reserve((size_t) params->max_new_tokens); // Global Philox subsequence counter advances once per primitive // sample (one for c0 of each step, then 15 for the predictor codes). @@ -402,18 +450,58 @@ bool pipeline_tts_synthesize(PipelineTTS * pt, std::vector next_emb((size_t) hidden, 0.0f); - for (int step = 0; step < params.max_new_tokens; step++) { + // Streaming bookkeeping: pending_codes accumulates frames since the + // last emit; emit_pending decodes them through the codec and feeds + // on_chunk. Returns false to abort with QT_STATUS_CANCELLED. + std::vector> pending_codes; + pending_codes.reserve((size_t) chunk_frames); + + auto emit_pending = [&]() -> bool { + if (pending_codes.empty()) { + return true; + } + const int T_frames = (int) pending_codes.size(); + std::vector codes_kt((size_t) num_codebooks * (size_t) T_frames); + for (int t = 0; t < T_frames; t++) { + for (int k = 0; k < num_codebooks; k++) { + codes_kt[(size_t) k * (size_t) T_frames + (size_t) t] = pending_codes[(size_t) t][(size_t) k]; + } + } + std::vector chunk_audio = pipeline_codec_decode(&pt->codec, codes_kt.data(), num_codebooks, T_frames); + if (chunk_audio.empty()) { + qt_set_error("pipeline_tts_synthesize: streaming codec decode returned no audio"); + qt_log(QT_LOG_ERROR, "[Pipeline] streaming codec decode returned no audio"); + return false; + } + if (!params->on_chunk(chunk_audio.data(), (int) chunk_audio.size(), params->on_chunk_user_data)) { + qt_log(QT_LOG_INFO, "[Pipeline] on_chunk callback aborted the synthesis"); + return false; + } + pending_codes.clear(); + return true; + }; + + for (int step = 0; step < params->max_new_tokens; step++) { + // Cooperative cancellation, polled at every step. Granularity is + // one AR frame = 1 / 12.5 Hz ~ 83 ms of audio, which is well + // below any reasonable UX cancel latency target. + if (params->cancel && params->cancel(params->cancel_user_data)) { + qt_log(QT_LOG_INFO, "[Pipeline] cancelled at step %d", step); + return QT_STATUS_CANCELLED; + } + TalkerForwardOutput fw; - const char * step_dump = (params.dump_dir && step == 0) ? params.dump_dir : NULL; + const char * step_dump = (params->dump_dir && step == 0) ? params->dump_dir : NULL; bool ok; if (step == 0) { ok = talker_forward_prefill(&pt->talker, &pt->talker_kv, pt->sched, prompt.input_embed.data(), prompt.T_ctx, - step_dump, &fw); + use_fa, clamp_fp16, step_dump, &fw); } else { - ok = talker_forward_decode(&pt->talker, &pt->talker_kv, pt->sched, next_emb.data(), &fw); + ok = + talker_forward_decode(&pt->talker, &pt->talker_kv, pt->sched, next_emb.data(), use_fa, clamp_fp16, &fw); } if (!ok) { - return false; + return QT_STATUS_GENERATE_FAILED; } // Bisection dump: the talker hidden_last at step 1 is the input @@ -421,9 +509,9 @@ bool pipeline_tts_synthesize(PipelineTTS * pt, // step 0. Pairing it byte for byte with the Python hook tells us // whether the next-emb composition + talker decode round trip // is bit exact end to end. - if (params.dump_dir && step == 1) { + if (params->dump_dir && step == 1) { DebugDumper d; - debug_init(&d, params.dump_dir); + debug_init(&d, params->dump_dir); debug_dump_1d(&d, "talker-hidden-step1", fw.hidden_last.data(), hidden); } @@ -431,12 +519,13 @@ bool pipeline_tts_synthesize(PipelineTTS * pt, // codec_eos. Then run the upstream sampling chain. apply_suppress(fw.logits_last.data(), talker_vocab, talker_vocab - 1024, talker_vocab, codec_eos_id); float u_c0 = 0.0f; - int c0 = sample_top_k_p(fw.logits_last.data(), talker_vocab, talker_T, params.top_k, params.top_p, talker_rp, - talker_history.data(), (int) talker_history.size(), params.seed, subseq_counter, &u_c0); + int c0 = + sample_top_k_p(fw.logits_last.data(), talker_vocab, talker_T, params->top_k, params->top_p, talker_rp, + talker_history.data(), (int) talker_history.size(), resolved_seed, subseq_counter, &u_c0); subseq_counter++; if (c0 < 0) { qt_log(QT_LOG_ERROR, "[Pipeline] c0 sample returned no candidate"); - return false; + return QT_STATUS_GENERATE_FAILED; } // Trace the first 32 samples unconditionally so [Sample] lines @@ -453,11 +542,11 @@ bool pipeline_tts_synthesize(PipelineTTS * pt, } CodePredictorOutput cp; - const char * cp_dump = (params.dump_dir && step == 0) ? params.dump_dir : NULL; + const char * cp_dump = (params->dump_dir && step == 0) ? params->dump_dir : NULL; if (!code_predictor_step(&pt->talker, &pt->code_predictor, &pt->code_predictor_kv, pt->sched, - fw.hidden_last.data(), c0, subtk_T, params.subtalker_top_k, params.subtalker_top_p, - params.seed, subseq_counter - 1, cp_dump, &cp)) { - return false; + fw.hidden_last.data(), c0, subtk_T, params->subtalker_top_k, params->subtalker_top_p, + resolved_seed, subseq_counter - 1, use_fa, clamp_fp16, cp_dump, &cp)) { + return QT_STATUS_GENERATE_FAILED; } // Predictor consumed (num_codebooks - 1) subsequences after the // c0 one (subseq_base + 1 .. subseq_base + 15). @@ -465,6 +554,14 @@ bool pipeline_tts_synthesize(PipelineTTS * pt, all_codes.push_back(cp.codes); talker_history.push_back(c0); + if (streaming) { + pending_codes.push_back(cp.codes); + if ((int) pending_codes.size() >= chunk_frames) { + if (!emit_pending()) { + return QT_STATUS_CANCELLED; + } + } + } // Build next-token embedding: sum of 16 codebook embeddings. // codebook 0 uses talker.codec_embedding, the 15 acoustic @@ -500,9 +597,9 @@ bool pipeline_tts_synthesize(PipelineTTS * pt, // is the only thing controlling the talker forward at step 1, so // matching it bit-exact against Python pinpoints any drift in // the codebook embedding sums or the trailing text overlay. - if (params.dump_dir && step == 0) { + if (params->dump_dir && step == 0) { DebugDumper d; - debug_init(&d, params.dump_dir); + debug_init(&d, params->dump_dir); debug_dump_1d(&d, "next-emb-step0", next_emb.data(), hidden); } @@ -513,9 +610,9 @@ bool pipeline_tts_synthesize(PipelineTTS * pt, qt_log(QT_LOG_INFO, "[Pipeline] Generation done : %zu frames", all_codes.size()); - if (params.dump_dir && !all_codes.empty()) { + if (params->dump_dir && !all_codes.empty()) { DebugDumper d; - debug_init(&d, params.dump_dir); + debug_init(&d, params->dump_dir); int T_frames = (int) all_codes.size(); std::vector flat((size_t) T_frames * (size_t) num_codebooks); for (int t = 0; t < T_frames; t++) { @@ -527,31 +624,56 @@ bool pipeline_tts_synthesize(PipelineTTS * pt, debug_dump_i32_as_f32(&d, "codes-full", flat.data(), shape, 2); } - // Codec decode: transpose codes from [T_frames, K] to [K, T_frames] - // because pipeline_codec_decode expects K-major layout (codebooks - // first, frames second), then return the 24 kHz mono audio. - if (all_codes.empty()) { - return true; + // Streaming tail: flush remaining frames through the callback. The + // buffered output stays empty in this branch; the caller already + // received every sample through on_chunk. + if (streaming) { + if (!emit_pending()) { + return QT_STATUS_CANCELLED; + } + out->samples = NULL; + out->n_samples = 0; + out->sample_rate = TOKENIZER_SAMPLE_RATE; + out->channels = 1; + return QT_STATUS_OK; } - int T_frames = (int) all_codes.size(); + // Buffered path: empty all_codes means EOS at step 0 with no audio. + // Return success and an empty qt_audio struct; the facade leaves it + // to the caller to decide what to do with a zero sample synthesis. + if (all_codes.empty()) { + out->samples = NULL; + out->n_samples = 0; + out->sample_rate = TOKENIZER_SAMPLE_RATE; + out->channels = 1; + return QT_STATUS_OK; + } + + // Codec decode: transpose codes from [T_frames, K] to [K, T_frames] + // because pipeline_codec_decode expects K-major layout (codebooks + // first, frames second), then materialise the 24 kHz mono audio. + const int T_frames = (int) all_codes.size(); std::vector codes_kt((size_t) num_codebooks * (size_t) T_frames); for (int t = 0; t < T_frames; t++) { for (int k = 0; k < num_codebooks; k++) { codes_kt[(size_t) k * (size_t) T_frames + (size_t) t] = all_codes[(size_t) t][(size_t) k]; } } - out->audio = pipeline_codec_decode(&pt->codec, codes_kt.data(), num_codebooks, T_frames); - if (out->audio.empty()) { + std::vector audio = pipeline_codec_decode(&pt->codec, codes_kt.data(), num_codebooks, T_frames); + if (audio.empty()) { + qt_set_error("pipeline_tts_synthesize: codec decode returned no audio"); qt_log(QT_LOG_ERROR, "[Pipeline] codec decode returned no audio"); - return false; + return QT_STATUS_GENERATE_FAILED; } - if (params.dump_dir) { + if (params->dump_dir) { DebugDumper d; - debug_init(&d, params.dump_dir); - debug_dump_1d(&d, "output-audio", out->audio.data(), (int) out->audio.size()); + debug_init(&d, params->dump_dir); + debug_dump_1d(&d, "output-audio", audio.data(), (int) audio.size()); } - return true; + if (!fill_qt_audio(audio, out)) { + return QT_STATUS_OOM; + } + return QT_STATUS_OK; } diff --git a/src/pipeline-tts.h b/src/pipeline-tts.h index 9467c0b..1d9c03c 100644 --- a/src/pipeline-tts.h +++ b/src/pipeline-tts.h @@ -2,10 +2,13 @@ // pipeline-tts.h: full TTS pipeline composition (Talker LM + code // predictor MTP head + optional speaker encoder + 12Hz codec decoder). // -// Phase 2.0 covers load-only: parse hyperparameters from both GGUF -// files, load every weight tensor on the configured backend, and -// expose the metadata needed to build forward graphs in later phases. -// No graph construction or sampling is wired here yet. +// pipeline_tts_load opens the talker GGUF and the codec GGUF, parses +// every typed metadata block (specials, languages, speakers, +// generation defaults), loads every weight tensor on the shared +// backend and initialises both KV caches. pipeline_tts_synthesize +// runs the prompt assembly, the autoregressive frame loop and the +// codec decode in one pass; it fills the public qt_audio struct +// directly so the facade in qwen.cpp stays a thin wrapper. #include "backend.h" #include "code-predictor-weights.h" @@ -13,6 +16,7 @@ #include "gguf-weights.h" #include "kv-cache.h" #include "pipeline-codec.h" +#include "qwen.h" #include "speaker-encoder-weights.h" #include "talker-weights.h" @@ -43,7 +47,7 @@ struct LanguageEntry { int id; }; -// Speaker entry for CustomVoice models. id is the codec embedding row id +// Speaker entry for CustomVoice variants. id is the codec embedding row id // inserted in the talker prefix, dialect is empty unless the speaker // overrides the user supplied language with a dialect lang_id (eric -> // sichuan_dialect, dylan -> beijing_dialect on the upstream checkpoint). @@ -90,6 +94,15 @@ struct PipelineTTS { ggml_backend_t backend; ggml_backend_sched_t sched; + // Attention path config, set at load and forwarded to every + // talker / code predictor forward. use_flash_attn collapses to + // false on CPU only backends (fused FA needs a GPU kernel). + // clamp_fp16 inserts ggml_clamp on V before attention and on the + // residual stream between blocks to guard FP16 matmul accumulation + // on sub Ampere CUDA targets. + bool use_flash_attn; + bool clamp_fp16; + // Persistent KV caches: the talker holds the LM context, the // predictor holds one frame's 16 sub-steps and gets reset every // frame in code_predictor_step. @@ -99,50 +112,48 @@ struct PipelineTTS { // Open the talker GGUF and the codec GGUF, load every module on the // shared backend. Aborts with a logged error on any missing tensor or -// invalid metadata. Caller frees with pipeline_tts_free. -bool pipeline_tts_load(PipelineTTS * pt, const char * talker_gguf_path, const char * codec_gguf_path, BackendPair bp); +// invalid metadata. use_fa is gated on bp.has_gpu inside the load: +// CPU only runs always use the manual F32 attention chain. clamp_fp16 +// is forwarded as is. Caller frees with pipeline_tts_free. +bool pipeline_tts_load(PipelineTTS * pt, + const char * talker_gguf_path, + const char * codec_gguf_path, + BackendPair bp, + bool use_fa, + bool clamp_fp16); void pipeline_tts_free(PipelineTTS * pt); struct BPETokenizer; -// Parameters for one synthesis call. Lifetime constraint: text and lang -// are borrowed pointers, must outlive the call. dump_dir, when non-NULL, -// captures step 0 prefill activations plus the codes-full / output-audio -// dumps under the named directory; debug only, slows the run. -struct PipelineTTSSynthesizeParams { - const char * text; - const char * lang; - const char * instruct; - const char * speaker; - const float * ref_audio_24k; - int ref_n_samples; - const char * ref_text; - int64_t seed; - int max_new_tokens; - bool do_sample; - float temperature; - int top_k; - float top_p; - float repetition_penalty; - bool subtalker_do_sample; - float subtalker_temperature; - int subtalker_top_k; - float subtalker_top_p; - const char * dump_dir; -}; - -// Output of one synthesis call. audio is a 24 kHz mono F32 PCM buffer -// already decoded through the codec; the caller writes it to disk. -struct PipelineTTSSynthesizeOutput { - std::vector audio; - int sample_rate; -}; - // Run the full TTS pipeline: prompt assembly, prefill, frame loop with -// sampling, codec decode. Returns false on any failure with a diagnostic -// already routed through qt_log / qt_set_error. -bool pipeline_tts_synthesize(PipelineTTS * pt, - BPETokenizer * tok, - const PipelineTTSSynthesizeParams & params, - PipelineTTSSynthesizeOutput * out); +// sampling, codec decode, fill qt_audio. Reads every knob (text, +// references, sampling, cancel, on_chunk, ...) straight from the +// public qt_tts_params struct so the facade in qwen.cpp can hand it +// off verbatim after the mode validation and seed resolve. +// +// Returns QT_STATUS_OK on success. On any failure returns a negative +// qt_status with a diagnostic already routed through qt_log / +// qt_set_error and leaves `out` empty. QT_STATUS_CANCELLED is returned +// when params->cancel or params->on_chunk returns true / false +// respectively during the AR loop. +// +// In buffered mode (params->on_chunk == NULL) the synthesised waveform +// is malloc allocated into out->samples; the caller releases it with +// qt_audio_free. In streaming mode (params->on_chunk != NULL) audio is +// emitted through the callback as decoded chunks and out->samples +// stays NULL on success. +// +// resolved_seed is the seed actually used for sampling: qt_synthesize +// hands over the same value it logged so dump traces and replays line +// up across runs even when params->seed was -1. +qt_status pipeline_tts_synthesize(PipelineTTS * pt, + BPETokenizer * tok, + const struct qt_tts_params * params, + int64_t resolved_seed, + struct qt_audio * out); + +// Convert a duration in seconds to a frame count at the codec frame +// rate (QT_CODEC_SAMPLE_RATE / TOKENIZER_HOP_LENGTH). Clamps to a +// minimum of one frame. +int pipeline_tts_duration_sec_to_tokens(const PipelineTTS * pt, float duration_sec); diff --git a/src/qwen.cpp b/src/qwen.cpp index 60d6f6c..90b7431 100644 --- a/src/qwen.cpp +++ b/src/qwen.cpp @@ -4,10 +4,10 @@ // so the symbols carry C linkage and are linkable from C, Rust, Go, // Python ctypes and any other binding generator. The struct // qt_context opaque handle owns one BackendPair, one PipelineTTS -// (which embeds its PipelineCodec) and one BPETokenizer. qt_init -// walks the load chain in dependency order and unwinds whatever it -// already allocated when any step fails. qt_free mirrors that order -// in reverse. +// (which already embeds the PipelineCodec) and one BPETokenizer. +// qt_init walks the load chain in dependency order and unwinds +// whatever it already allocated when any step fails. qt_free mirrors +// that order in reverse. // // This translation unit also absorbs the internal qt_set_error / // qt_throw / qt_log helpers that the rest of the codebase calls. The @@ -137,6 +137,19 @@ void qt_log(qt_log_level level, const char * fmt, ...) { } } +// Resolve a -1 seed to a hardware random 64-bit value. Anything else is +// forwarded verbatim, so reproducibility is one explicit seed away. The +// resolved value travels into pipeline_tts_synthesize so the dump traces +// log the exact seed that drove the sampler, even when the caller asked +// for non determinism. +static int64_t qt_resolve_seed(int64_t seed) { + if (seed >= 0) { + return seed; + } + std::random_device rd; + return (int64_t) (((uint64_t) rd() << 32) ^ (uint64_t) rd()); +} + extern "C" { const char * qt_version(void) { @@ -174,12 +187,14 @@ void qt_init_default_params(struct qt_init_params * p) { p->abi_version = QT_ABI_VERSION; p->talker_path = nullptr; p->codec_path = nullptr; + p->use_fa = true; + p->clamp_fp16 = false; } void qt_tts_default_params(struct qt_tts_params * p) { p->abi_version = QT_ABI_VERSION; p->text = nullptr; - p->lang = "english"; + p->lang = nullptr; p->instruct = nullptr; p->speaker = nullptr; p->ref_audio_24k = nullptr; @@ -197,6 +212,11 @@ void qt_tts_default_params(struct qt_tts_params * p) { p->subtalker_top_k = 50; p->subtalker_top_p = 1.0f; p->dump_dir = nullptr; + p->cancel = nullptr; + p->cancel_user_data = nullptr; + p->on_chunk = nullptr; + p->on_chunk_user_data = nullptr; + p->chunk_duration_sec = 1.0f; } struct qt_context * qt_init(const struct qt_init_params * params) { @@ -231,7 +251,8 @@ struct qt_context * qt_init(const struct qt_init_params * params) { qt_throw("qt_init: backend_init failed (no GGML backend available)"); } - if (!pipeline_tts_load(&q->pt, params->talker_path, params->codec_path, q->bp)) { + if (!pipeline_tts_load(&q->pt, params->talker_path, params->codec_path, q->bp, params->use_fa, + params->clamp_fp16)) { qt_throw("qt_init: pipeline_tts_load failed for '%s' / '%s'", params->talker_path, params->codec_path); } @@ -266,29 +287,28 @@ void qt_free(struct qt_context * q) { delete q; } -// Resolve a -1 seed to a hardware random 64-bit value. Anything else is -// forwarded verbatim, so reproducibility is one explicit seed away. -static int64_t qt_resolve_seed(int64_t seed) { - if (seed >= 0) { - return seed; - } - std::random_device rd; - return (int64_t) (((uint64_t) rd() << 32) ^ (uint64_t) rd()); -} - enum qt_status qt_synthesize(struct qt_context * q, const struct qt_tts_params * params, struct qt_audio * out) { - if (!q || !params || !out) { - qt_set_error("qt_synthesize: q, params or out is NULL"); + if (!q || !params) { + qt_set_error("qt_synthesize: q or params is NULL"); if (out) { qt_audio_free(out); } return QT_STATUS_INVALID_PARAMS; } + // Streaming mode (on_chunk non NULL) emits through the callback and + // leaves out unused, so out=NULL is valid there. Buffered mode + // requires out to receive the synthesised waveform. + if (!params->on_chunk && !out) { + qt_set_error("qt_synthesize: out is NULL in buffered mode"); + return QT_STATUS_INVALID_PARAMS; + } if (params->abi_version > QT_ABI_VERSION) { qt_set_error( "qt_synthesize: params->abi_version %d > QT_ABI_VERSION %d (binding compiled against a newer header)", params->abi_version, QT_ABI_VERSION); - qt_audio_free(out); + if (out) { + qt_audio_free(out); + } return QT_STATUS_INVALID_PARAMS; } @@ -301,104 +321,79 @@ enum qt_status qt_synthesize(struct qt_context * q, const struct qt_tts_params * const std::string & mt = q->pt.model_type; if (params->speaker && mt != "custom_voice") { qt_set_error("--speaker is only valid for custom_voice models (loaded: %s)", mt.c_str()); - qt_audio_free(out); + if (out) { + qt_audio_free(out); + } return QT_STATUS_MODE_INVALID; } if (params->instruct && mt == "base") { qt_set_error("--instruct is not supported for base models"); - qt_audio_free(out); + if (out) { + qt_audio_free(out); + } return QT_STATUS_MODE_INVALID; } if (mt == "custom_voice" && !params->speaker) { qt_set_error("custom_voice models require --speaker"); - qt_audio_free(out); + if (out) { + qt_audio_free(out); + } return QT_STATUS_MODE_INVALID; } if (mt == "voice_design" && (!params->instruct || params->instruct[0] == '\0')) { qt_set_error("voice_design models require --instruct"); - qt_audio_free(out); + if (out) { + qt_audio_free(out); + } return QT_STATUS_MODE_INVALID; } if (params->ref_audio_24k && mt != "base") { qt_set_error("--ref-wav is only valid for base models (loaded: %s)", mt.c_str()); - qt_audio_free(out); + if (out) { + qt_audio_free(out); + } return QT_STATUS_MODE_INVALID; } if (params->speaker && params->ref_audio_24k) { qt_set_error("--speaker and --ref-wav are mutually exclusive"); - qt_audio_free(out); + if (out) { + qt_audio_free(out); + } return QT_STATUS_INVALID_PARAMS; } if (params->ref_text && !params->ref_audio_24k) { qt_set_error("--ref-text requires --ref-wav"); - qt_audio_free(out); + if (out) { + qt_audio_free(out); + } return QT_STATUS_INVALID_PARAMS; } - // Translate the public POD params into the internal C++ struct - // expected by pipeline_tts_synthesize. Borrowed pointers are - // forwarded verbatim; the lifetime contract on the public side - // (caller keeps strings alive for the duration of the call) - // matches what the pipeline already requires. - PipelineTTSSynthesizeParams p = {}; - p.text = params->text; - p.lang = params->lang; - p.instruct = params->instruct; - p.speaker = params->speaker; - p.ref_audio_24k = params->ref_audio_24k; - p.ref_n_samples = params->ref_n_samples; - p.ref_text = params->ref_text; - p.seed = qt_resolve_seed(params->seed); - p.max_new_tokens = params->max_new_tokens; - p.do_sample = params->do_sample; - p.temperature = params->temperature; - p.top_k = params->top_k; - p.top_p = params->top_p; - p.repetition_penalty = params->repetition_penalty; - p.subtalker_do_sample = params->subtalker_do_sample; - p.subtalker_temperature = params->subtalker_temperature; - p.subtalker_top_k = params->subtalker_top_k; - p.subtalker_top_p = params->subtalker_top_p; - p.dump_dir = params->dump_dir; - // Defense in depth: the synthesis path normally reports failures - // via bool return + qt_set_error. A future load-style throw or any - // std::bad_alloc deep inside the GGML backend is caught here and - // converted to QT_STATUS_GENERATE_FAILED so an exception never + // via qt_status return + qt_set_error. A future load-style throw or + // any std::bad_alloc deep inside the GGML backend is caught here + // and converted to QT_STATUS_GENERATE_FAILED so an exception never // crosses the extern "C" boundary. try { - PipelineTTSSynthesizeOutput pout; - if (!pipeline_tts_synthesize(&q->pt, &q->tok, p, &pout)) { - qt_audio_free(out); - return QT_STATUS_GENERATE_FAILED; - } - - // Copy the std::vector into a malloc-backed buffer the - // caller can free with std::free via qt_audio_free. The - // vector itself goes out of scope at function exit, releasing - // its own storage independently. - const size_t n = pout.audio.size(); - const size_t bytes = n * sizeof(float); - float * buf = (float *) std::malloc(bytes > 0 ? bytes : 1); - if (!buf) { - qt_set_error("qt_synthesize: malloc failed for %zu samples", n); - qt_audio_free(out); - return QT_STATUS_OOM; - } - if (n > 0) { - std::memcpy(buf, pout.audio.data(), bytes); - } - out->samples = buf; - out->n_samples = (int) n; - out->sample_rate = pout.sample_rate; - out->channels = 1; - return QT_STATUS_OK; + const int64_t resolved_seed = qt_resolve_seed(params->seed); + return pipeline_tts_synthesize(&q->pt, &q->tok, params, resolved_seed, out); } catch (const std::exception & e) { qt_set_error("%s", e.what()); qt_log(QT_LOG_ERROR, "[Qwen] %s", e.what()); - qt_audio_free(out); + if (out) { + qt_audio_free(out); + } return QT_STATUS_GENERATE_FAILED; } } +int qt_duration_sec_to_tokens(const struct qt_context * q, float duration_sec) { + if (!q) { + qt_set_error("qt_duration_sec_to_tokens: q is NULL"); + qt_log(QT_LOG_ERROR, "[Qwen] qt_duration_sec_to_tokens requires a valid handle"); + return 1; + } + return pipeline_tts_duration_sec_to_tokens(&q->pt, duration_sec); +} + } // extern "C" diff --git a/src/qwen.h b/src/qwen.h index bebf1c6..f311aaa 100644 --- a/src/qwen.h +++ b/src/qwen.h @@ -56,8 +56,15 @@ extern "C" { // There is no separate semver triple. The runtime build identity is the // git short hash + commit date string returned by qt_version(); for // binding compat checks, QT_ABI_VERSION is the only number that -// matters. Aligned on OV_ABI_VERSION = 2 for the omnivoice ABI cousin. -#define QT_ABI_VERSION 2 +// matters. +#define QT_ABI_VERSION 1 + +// Codec sample rate. The 12 Hz Qwen3-TTS audio tokenizer always produces +// 24 kHz mono PCM through its DAC v2 decoder. Exposed at the ABI so a +// caller that needs to resample a reference WAV before passing it via +// qt_tts_params.ref_audio_24k can do so without pulling in any internal +// header. The constant is immutable for this model family. +#define QT_CODEC_SAMPLE_RATE 24000 // Returns a static string of the form " ()" identifying // the exact commit this binary was built from. Safe to call from any @@ -72,6 +79,7 @@ enum qt_status { QT_STATUS_MODE_INVALID = -2, QT_STATUS_GENERATE_FAILED = -3, QT_STATUS_OOM = -4, + QT_STATUS_CANCELLED = -5, }; // Returns the last error message produced on the calling thread by any @@ -93,7 +101,7 @@ QT_API const char * qt_last_error(void); struct qt_audio { float * samples; // mono PCM, malloc allocated int n_samples; // length in samples - int sample_rate; // 24000 for the 12 Hz Qwen3-TTS tokenizer + int sample_rate; // QT_CODEC_SAMPLE_RATE (24000) int channels; // 1 (mono) }; @@ -109,16 +117,21 @@ struct qt_context; // custom_voice / voice_design checkpoints) the speaker encoder; the // codec GGUF holds the 12 Hz audio tokenizer. abi_version stays first // so a future struct growth keeps reading the version field at offset -// 0. No use_fa / clamp_fp16 yet: the current pipeline_tts_load picks -// flash attention from backend capability without a user knob. +// 0. use_fa enables fused flash attention in the Talker and Code +// Predictor forwards when a GPU backend is present (CPU always uses the +// F32 manual chain); clamp_fp16 inserts ggml_clamp(-65504, 65504) on V +// before attention and on the residual stream between blocks to guard +// FP16 matmul accumulation on sub Ampere CUDA targets. struct qt_init_params { int abi_version; const char * talker_path; const char * codec_path; + bool use_fa; + bool clamp_fp16; }; // Initialise to the standard defaults: both paths NULL (caller must set -// them before calling qt_init), abi_version set to QT_ABI_VERSION. +// them before calling qt_init), use_fa true, clamp_fp16 false. QT_API void qt_init_default_params(struct qt_init_params * p); // Allocate every module described by params. Returns NULL on any @@ -131,6 +144,27 @@ QT_API struct qt_context * qt_init(const struct qt_init_params * params); // Safe on NULL. QT_API void qt_free(struct qt_context * q); +// Cooperative cancellation callback. Returns true to request the +// synthesis to abort. Polled at the top of every Talker decode step in +// the autoregressive loop, so the cancel granularity is roughly one +// audio frame, i.e. 1 / 12 Hz ~ 83 ms. +typedef bool (*qt_cancel_cb)(void * user_data); + +// Streaming output callback. When set on qt_tts_params, the synth +// pipeline runs in streaming mode: audio is decoded chunk by chunk from +// the AR codec frames and emitted through this callback rather than +// accumulated into the `out` buffer of qt_synthesize. Returning false +// aborts the synthesis with QT_STATUS_CANCELLED, identical to the +// qt_cancel_cb behaviour. The samples pointer is mono float PCM at +// QT_CODEC_SAMPLE_RATE; valid only for the duration of the call. +// user_data is forwarded verbatim from on_chunk_user_data. +// +// The chunk granularity is driven by chunk_duration_sec in qt_tts_params: +// once the AR loop has produced enough frames to cover that duration, +// the codec decodes that bundle and emits it. The last chunk on EOS / +// max_new flushes whatever frames remain. +typedef bool (*qt_audio_chunk_cb)(const float * samples, int n_samples, void * user_data); + // Log severity. Numerically ordered so a callback can filter with a // simple `if (level < threshold) return;`. ERROR is reserved for // failure reports that the lib also surfaces via qt_status / @@ -184,8 +218,8 @@ struct qt_tts_params { // Optional voice reference for base mode voice cloning. Mode A // (x_vector_only) sets ref_audio_24k only; mode B (ICL) sets // both ref_audio_24k and ref_text. ref_audio_24k is a mono float - // PCM buffer sampled at the codec sample rate (24 kHz). Mutually - // exclusive with speaker. Rejected for custom_voice / voice_design. + // PCM buffer sampled at QT_CODEC_SAMPLE_RATE. Mutually exclusive + // with speaker. Rejected for custom_voice / voice_design. const float * ref_audio_24k; int ref_n_samples; const char * ref_text; @@ -211,22 +245,45 @@ struct qt_tts_params { // Intermediate tensor dump directory. NULL disables dumps. Debug // only, slows the run. const char * dump_dir; + + // Cooperative cancellation. cancel NULL disables the feature. + // cancel_user_data is forwarded to the callback verbatim. Polled + // at the top of every Talker decode step (~83 ms granularity). + qt_cancel_cb cancel; + void * cancel_user_data; + + // Streaming output. When on_chunk is non NULL, qt_synthesize runs + // the streaming pipeline: audio chunks emit through on_chunk and + // `out` stays empty on success. on_chunk NULL keeps the buffered + // path. chunk_duration_sec drives the chunk size at codec sample + // rate; values <= 0 fall back to 1.0 second. The last chunk on + // EOS or max_new flushes whatever frames remain. + qt_audio_chunk_cb on_chunk; + void * on_chunk_user_data; + float chunk_duration_sec; }; // Initialise to the standard defaults. Strings NULL, seed -1, // max_new_tokens 2048, do_sample true, temperature 0.9, top_k 50, // top_p 1.0, repetition_penalty 1.05, subtalker mirrors talker, -// dump_dir NULL. +// dump_dir NULL, cancel NULL, on_chunk NULL, chunk_duration_sec 1.0. QT_API void qt_tts_default_params(struct qt_tts_params * p); // Run the full TTS synthesis. Validates the params against the loaded // model_type (the seven base / custom_voice / voice_design rules), // resolves the seed, hands off to pipeline_tts_synthesize and fills -// `out` with mono float PCM at the codec sample rate. Returns -// QT_STATUS_OK on success; on any failure returns a negative -// qt_status describing the cause and leaves `out` empty. +// `out` with mono float PCM at QT_CODEC_SAMPLE_RATE in buffered mode. +// In streaming mode (params->on_chunk != NULL), audio is emitted +// through the callback and `out` stays empty. Returns QT_STATUS_OK on +// success; on any failure returns a negative qt_status describing the +// cause and leaves `out` empty. QT_API enum qt_status qt_synthesize(struct qt_context * q, const struct qt_tts_params * params, struct qt_audio * out); +// Convert a duration in seconds to a frame count using the codec +// frame rate (QT_CODEC_SAMPLE_RATE / TOKENIZER_HOP_LENGTH = 12.5 Hz). +// Clamps to a minimum of one frame. +QT_API int qt_duration_sec_to_tokens(const struct qt_context * q, float duration_sec); + #ifdef __cplusplus } #endif diff --git a/src/talker-forward.h b/src/talker-forward.h index f85aebf..bb44281 100644 --- a/src/talker-forward.h +++ b/src/talker-forward.h @@ -75,11 +75,38 @@ static bool talker_is_bisect_layer(int l) { return false; } +// Manual F32 attention chain. Used when use_flash_attn is false: GQA +// scaled dot product with explicit mul_mat / soft_max_ext / mul_mat, +// FP32 accumulators end to end. Mirrors the qwen3_attn_f32 helper in +// omnivoice.cpp/src/qwen3-enc.h. Inputs and output stay in the same +// layout flash_attn_ext expects: q [hd, T, n_q_heads], k/v [hd, T_full, +// n_kv], output [hd, n_q_heads, T]; the caller reshapes to +// [n_q_heads * hd, T] before o_proj exactly as on the FA path. +static struct ggml_tensor * talker_attn_f32(struct ggml_context * ctx, + struct ggml_tensor * q, + struct ggml_tensor * k, + struct ggml_tensor * v, + struct ggml_tensor * mask, + float scale) { + struct ggml_tensor * scores = ggml_mul_mat(ctx, k, q); + scores = ggml_soft_max_ext(ctx, scores, mask, scale, 0.0f); + struct ggml_tensor * vt = ggml_cont(ctx, ggml_transpose(ctx, v)); + struct ggml_tensor * out = ggml_mul_mat(ctx, vt, scores); + return ggml_cont(ctx, ggml_permute(ctx, out, 0, 2, 1, 3)); +} + // Build the per-layer block, KV cached. K and V for the T fresh // positions get computed normally, then written into the cache at // [n_past, n_past+T) on dim 1. The attention reads the contiguous slice // [0, n_past+T) on the same dim, which covers the full causal context // in one tensor view. Returns the layer output [hidden, T]. +// +// use_flash_attn picks between the fused ggml_flash_attn_ext kernel +// (GPU only, FP16 accumulation guarded with set_prec(F32)) and the +// manual F32 chain. clamp_fp16 inserts ggml_clamp(-65504, 65504) on V +// before attention and on the residual stream after the attention and +// MLP adds, protecting sub Ampere CUDA tensor cores that accumulate in +// FP16 from overflow. static struct ggml_tensor * talker_layer_forward(struct ggml_context * ctx, const TalkerWeights * tw, const TalkerLayer & layer, @@ -90,6 +117,8 @@ static struct ggml_tensor * talker_layer_forward(struct ggml_context * ctx, struct ggml_tensor * v_cache, int n_past, int T, + bool use_flash_attn, + bool clamp_fp16, struct ggml_cgraph * gf) { const int n_q_heads = tw->num_attention_heads; const int n_kv = tw->num_key_value_heads; @@ -156,16 +185,32 @@ static struct ggml_tensor * talker_layer_forward(struct ggml_context * ctx, // No cont: flash_attn_ext takes the view directly, like acestep does. struct ggml_tensor * q_p = ggml_permute(ctx, q, 0, 2, 1, 3); - // Fused flash attention. The manual mul_mat + soft_max_ext + mul_mat - // path it replaces had a Vulkan bug in autoregressive decode (T=1) - // where the KV cache view stride on dim 2 (= max_T * hd, non + // Clamp V before attention. Sub Ampere tensor cores accumulate in + // FP16 and the V projection can overflow to inf, which corrupts + // every subsequent attention. ggml_clamp is a no op on the F32 + // manual path but stays here to keep both branches numerically + // aligned when the user opts into clamp_fp16. + if (clamp_fp16) { + v_full = ggml_clamp(ctx, v_full, -65504.0f, 65504.0f); + } + + // Attention: fused flash kernel (FP16 accumulation, set_prec(F32) + // promotes the softmax / V matmul accumulator back to F32) or + // manual F32 chain. The fused path replaces a mul_mat + soft_max_ext + // + mul_mat sequence that had a Vulkan bug in autoregressive decode + // (T=1) where the KV cache view stride on dim 2 (= max_T * hd, non // contiguous with dim 1 of length T_full) caused the second mul_mat - // to diverge silently. Flash attention has a backend-tested kernel - // on every target (CPU, CUDA, Vulkan), matches the working acestep - // qw3lm_build_attn pattern. + // to diverge silently. The manual path is the F32 reference, used + // when use_flash_attn is false (CPU only backends, or explicit + // request from the user). float scale = 1.0f / sqrtf((float) hd); - struct ggml_tensor * attn = ggml_flash_attn_ext(ctx, q_p, k_full, v_full, mask, scale, 0.0f, 0.0f); - ggml_flash_attn_ext_set_prec(attn, GGML_PREC_F32); + struct ggml_tensor * attn; + if (use_flash_attn) { + attn = ggml_flash_attn_ext(ctx, q_p, k_full, v_full, mask, scale, 0.0f, 0.0f); + ggml_flash_attn_ext_set_prec(attn, GGML_PREC_F32); + } else { + attn = talker_attn_f32(ctx, q_p, k_full, v_full, mask, scale); + } // Flash attention output is [hd, n_q_heads, T], flatten heads for o_proj. attn = ggml_reshape_2d(ctx, attn, n_q_heads * hd, T); @@ -173,6 +218,9 @@ static struct ggml_tensor * talker_layer_forward(struct ggml_context * ctx, struct ggml_tensor * o = ggml_mul_mat(ctx, layer.attn.o_proj_w, attn); x = ggml_add(ctx, x, o); + if (clamp_fp16) { + x = ggml_clamp(ctx, x, -65504.0f, 65504.0f); + } // MLP block: pre-norm + SwiGLU + residual struct ggml_tensor * h2 = ggml_rms_norm(ctx, x, eps); @@ -185,19 +233,25 @@ static struct ggml_tensor * talker_layer_forward(struct ggml_context * ctx, struct ggml_tensor * mlp = ggml_mul_mat(ctx, layer.mlp.down_proj_w, gu); x = ggml_add(ctx, x, mlp); + if (clamp_fp16) { + x = ggml_clamp(ctx, x, -65504.0f, 65504.0f); + } return x; } // Shared core that builds the graph, allocates, uploads inputs, runs // it and pulls out the last position hidden + logits. T tokens are // appended to the cache starting at n_past. When n_past == 0 and -// dump_dir is set, the bisect taps fire on the prefill path. +// dump_dir is set, the bisect taps fire on the prefill path. use_fa / +// clamp_fp16 are forwarded as is to every layer. static bool talker_forward_core(const TalkerWeights * tw, KVCache * kv, ggml_backend_sched_t sched, const float * input_embed, int T, int n_past, + bool use_flash_attn, + bool clamp_fp16, const char * dump_dir, TalkerForwardOutput * out) { const int hidden = tw->hidden_size; @@ -242,7 +296,7 @@ static bool talker_forward_core(const TalkerWeights * tw, std::vector taps(TALKER_N_BISECT_LAYERS, NULL); for (int l = 0; l < n_layers; l++) { h = talker_layer_forward(gctx, tw, tw->layers[(size_t) l], h, pos_in, mask_in, kv->k[(size_t) l], - kv->v[(size_t) l], n_past, T, gf); + kv->v[(size_t) l], n_past, T, use_flash_attn, clamp_fp16, gf); if (record_taps && talker_is_bisect_layer(l)) { for (int i = 0; i < TALKER_N_BISECT_LAYERS; i++) { if (TALKER_BISECT_LAYERS[i] == l) { @@ -374,6 +428,8 @@ static bool talker_forward_prefill(const TalkerWeights * tw, ggml_backend_sched_t sched, const float * input_embed, int T, + bool use_flash_attn, + bool clamp_fp16, const char * dump_dir, TalkerForwardOutput * out) { kv_cache_reset(kv); @@ -381,7 +437,7 @@ static bool talker_forward_prefill(const TalkerWeights * tw, fprintf(stderr, "[TalkerForward] FATAL: prefill T=%d exceeds cache max_seq_len=%d\n", T, kv->max_seq_len); return false; } - return talker_forward_core(tw, kv, sched, input_embed, T, 0, dump_dir, out); + return talker_forward_core(tw, kv, sched, input_embed, T, 0, use_flash_attn, clamp_fp16, dump_dir, out); } // Decode: feed exactly one embedding and append one position to the @@ -391,11 +447,13 @@ static bool talker_forward_decode(const TalkerWeights * tw, KVCache * kv, ggml_backend_sched_t sched, const float * input_embed_1, + bool use_flash_attn, + bool clamp_fp16, TalkerForwardOutput * out) { if (kv->cur_len + 1 > kv->max_seq_len) { fprintf(stderr, "[TalkerForward] FATAL: decode would overflow cache (%d + 1 > %d)\n", kv->cur_len, kv->max_seq_len); return false; } - return talker_forward_core(tw, kv, sched, input_embed_1, 1, kv->cur_len, NULL, out); + return talker_forward_core(tw, kv, sched, input_embed_1, 1, kv->cur_len, use_flash_attn, clamp_fp16, NULL, out); } diff --git a/tests/abi-c.c b/tests/abi-c.c index a2da9cb..04e77ae 100644 --- a/tests/abi-c.c +++ b/tests/abi-c.c @@ -1,4 +1,4 @@ -/* tests/abi-c.c : link-only ABI smoke test for qwen.h. +/* tests/abi-c.c: link-only ABI smoke test for qwen.h. * * Compiled in pure C99 with -Wall -Werror -pedantic. The purpose of this * test is NOT to run a full synthesis (no GGUF loaded, no model required); @@ -22,11 +22,23 @@ #include #include +static bool stub_cancel(void * ud) { + (void) ud; + return false; +} + +static bool stub_on_chunk(const float * samples, int n_samples, void * ud) { + (void) samples; + (void) n_samples; + (void) ud; + return true; +} + /* Counter incremented by the stub log callback. The probe checks that at * least one log line was routed through the callback by triggering a - * qt_init failure (which emits a [qwen] ERROR line via qt_log). */ -static int g_log_lines = 0; -static enum qt_log_level g_last_log_level = QT_LOG_DEBUG; + * qt_init failure (which emits a [Qwen] ERROR line via qt_log). */ +static int g_log_lines = 0; +static enum qt_log_level g_last_log_level = QT_LOG_DEBUG; static char g_last_log_msg[512] = { 0 }; static void stub_log(enum qt_log_level level, const char * msg, void * user_data) { @@ -55,29 +67,44 @@ int main(void) { struct qt_tts_params params; qt_tts_default_params(¶ms); - /* Sanity-check a few default values, including the abi_version. */ - if (params.max_new_tokens != 2048 || params.temperature != 0.9f) { + /* Sanity-check a few default values, including the abi_version and + * the new use_fa / clamp_fp16 / on_chunk / chunk_duration_sec slots. */ + if (params.max_new_tokens != 2048 || params.chunk_duration_sec <= 0.0f) { fprintf(stderr, "[Probe] default values do not match\n"); return 1; } if (iparams.abi_version != QT_ABI_VERSION || params.abi_version != QT_ABI_VERSION) { - fprintf(stderr, "[Probe] abi_version not set by qwen_*_default_params\n"); + fprintf(stderr, "[Probe] abi_version not set by qt_*_default_params\n"); + return 1; + } + if (!iparams.use_fa || iparams.clamp_fp16) { + fprintf(stderr, "[Probe] init_params defaults do not match (use_fa=true, clamp_fp16=false)\n"); + return 1; + } + if (QT_CODEC_SAMPLE_RATE != 24000) { + fprintf(stderr, "[Probe] QT_CODEC_SAMPLE_RATE is not 24000\n"); return 1; } - /* Touch every output struct field so the compiler validates the - * layout end-to-end without ever needing a model. */ + /* Touch every reference-pointer field, every callback typedef and + * every output struct field so the compiler validates the layout + * end-to-end without ever needing a model. */ + params.cancel = stub_cancel; + params.cancel_user_data = NULL; + params.on_chunk = stub_on_chunk; + params.on_chunk_user_data = NULL; + struct qt_audio audio = { 0 }; qt_audio_free(&audio); - /* Install the log callback before the failing init so the [qwen] + /* Install the log callback before the failing init so the [Qwen] * ERROR line lands on stub_log instead of stderr. */ qt_log_set(stub_log, NULL); /* Call every entry through its early-return path. qt_init returns - * NULL on missing talker_path / codec_path, qt_synthesize fails on - * NULL handle, qt_free is safe on NULL. None of these load a model, - * but the linker must resolve every name to satisfy the call. */ + * NULL on missing paths, qt_synthesize / qt_duration_sec_to_tokens + * fail on NULL handle, qt_free is safe on NULL. None of these load a + * model, but the linker must resolve every name to satisfy the call. */ struct qt_context * dummy = qt_init(NULL); if (dummy != NULL) { fprintf(stderr, "[Probe] qt_init(NULL) was supposed to return NULL\n"); @@ -102,17 +129,14 @@ int main(void) { return 6; } if (g_last_log_level != QT_LOG_ERROR) { - fprintf(stderr, "[Probe] last log level was %d, expected %d\n", (int) g_last_log_level, - (int) QT_LOG_ERROR); + fprintf(stderr, "[Probe] last log level was %d, expected %d\n", (int) g_last_log_level, (int) QT_LOG_ERROR); return 7; } printf("[Probe] qt_log_set routed %d line(s), last: '%s'\n", g_log_lines, g_last_log_msg); printf("[Probe] qt_last_error reads '%s'\n", err); /* abi_version validation : a struct claiming a future ABI must be - * rejected up front, before any allocation. Both paths are filled - * with placeholders so the NULL guard does not short-circuit the - * abi_version branch. */ + * rejected up front, before any allocation. */ struct qt_init_params future_iparams; qt_init_default_params(&future_iparams); future_iparams.talker_path = "irrelevant.gguf"; @@ -132,8 +156,14 @@ int main(void) { return 3; } + int frames = qt_duration_sec_to_tokens(NULL, 1.0f); + if (frames < 1) { + fprintf(stderr, "[Probe] qt_duration_sec_to_tokens returned %d, expected >= 1\n", frames); + return 4; + } + /* Restore the default stderr fallback before exit so the trailing - * [qwen] log lines from the cleanup paths land where the user + * [Qwen] log lines from the cleanup paths land where the user * expects them. */ qt_log_set(NULL, NULL); diff --git a/tests/debug-clone-cossim.py b/tests/debug-clone-cossim.py index 728a944..81d3aad 100755 --- a/tests/debug-clone-cossim.py +++ b/tests/debug-clone-cossim.py @@ -13,7 +13,7 @@ length, the shorter one padded with tts_pad / truncated as needed. Cote Python the speaker embedding is captured directly via model.extract_speaker_embedding, and the reference codec frames via -model.speech_tokenizer.encode. Both intermediates land as speaker-emb.bin +model.speech_tokenizer.encode. Both intermediates land as spk-emb.bin and ref-codes.bin and are compared against the C++ side dumps emitted by pipeline-tts.cpp when --ref-wav and --ref-text are set. @@ -38,9 +38,6 @@ CKPT = "../checkpoints/Qwen3-TTS-12Hz-1.7B-Base" DUMP_CPP = "cpp/clone" DUMP_PT = "python/clone" -DEFAULT_REF_AUDIO = "../examples/freeman.wav" -DEFAULT_REF_TEXT = "../examples/freeman.txt" - # Mode B adds two pre-talker stages to the standard list : the speaker # embedding extracted from the reference audio (ECAPA forward, projected to # talker hidden), and the reference codec frames at 12.5 Hz. Plus three @@ -65,7 +62,7 @@ STAGES_CLONE = cc.STAGES_STANDARD + [ ("SpkBlock3", "spk-block3.bin"), ("SpkMFA", "spk-mfa.bin"), ("SpkASP", "spk-asp.bin"), - ("SpeakerEmb", "speaker-emb.bin"), + ("SpeakerEmb", "spk-emb.bin"), ] def install_clone_hooks(model, dump_dir): @@ -271,9 +268,9 @@ def dump_mel_mag_python(ref_wav, dump_dir): def main(): ap = argparse.ArgumentParser() ap.add_argument("--prompt", default="../examples/prompt.txt") - ap.add_argument("--ref-wav", default=DEFAULT_REF_AUDIO, + ap.add_argument("--ref-wav", default="../examples/freeman.wav", help="reference WAV path for voice cloning") - ap.add_argument("--ref-text-file", default=DEFAULT_REF_TEXT, + ap.add_argument("--ref-text", default="../examples/freeman.txt", help="path to a UTF-8 file with the transcript of ref-wav") ap.add_argument("--seed", type=int, default=42) ap.add_argument("--lang", default="english") @@ -297,7 +294,7 @@ def main(): with open(args.prompt, "r", encoding="utf-8") as f: text = f.read().strip() - with open(args.ref_text_file, "r", encoding="utf-8") as f: + with open(args.ref_text, "r", encoding="utf-8") as f: ref_text = f.read().strip() print(f"[Input] Prompt: {len(text)} chars: {text[:60]}{'...' if len(text) > 60 else ''}") print(f"[Input] RefAudio: {args.ref_wav}") @@ -356,7 +353,7 @@ def main(): # Extract speaker embedding via ECAPA forward, projected to talker hidden. spk_emb = model.extract_speaker_embedding(audio=ref_wav, sr=ref_sr) print(f"[Python] SpeakerEmb shape: {tuple(spk_emb.shape)} dtype: {spk_emb.dtype}") - cc.save_dump(os.path.join(DUMP_PT, "speaker-emb.bin"), spk_emb) + cc.save_dump(os.path.join(DUMP_PT, "spk-emb.bin"), spk_emb) # Encode the reference audio to 16 codebook codes at 12.5 Hz. The encode # call returns shape [T_codec, K=16] after the internal transpose, while @@ -460,7 +457,7 @@ def main(): "--seed", str(args.seed), "--text", text, "--ref-wav", args.ref_wav, - "--ref-text", ref_text, + "--ref-text", args.ref_text, "--lang", args.lang, "--max-new", str(args.max_new_tokens), "--dump", DUMP_CPP, diff --git a/tests/debug-customvoice-cossim.py b/tests/debug-customvoice-cossim.py index 5606f60..e81a32c 100755 --- a/tests/debug-customvoice-cossim.py +++ b/tests/debug-customvoice-cossim.py @@ -35,12 +35,10 @@ CKPT = "../checkpoints/Qwen3-TTS-12Hz-1.7B-CustomVoice" DUMP_CPP = "cpp/customvoice" DUMP_PT = "python/customvoice" -DEFAULT_SPEAKER = "vivian" - def main(): ap = argparse.ArgumentParser() ap.add_argument("--prompt", default="../examples/prompt.txt") - ap.add_argument("--speaker", default=DEFAULT_SPEAKER, + ap.add_argument("--speaker", default="vivian", help="speaker preset key (lowercase), validated by the model") ap.add_argument("--instruct", default="", help="optional style instruction, empty disables the instruct prefix") diff --git a/tests/debug-tts-cossim.py b/tests/debug-tts-cossim.py index 9be0c40..036c4f7 100755 --- a/tests/debug-tts-cossim.py +++ b/tests/debug-tts-cossim.py @@ -32,12 +32,10 @@ CKPT = "../checkpoints/Qwen3-TTS-12Hz-1.7B-VoiceDesign" DUMP_CPP = "cpp/tts" DUMP_PT = "python/tts" -DEFAULT_INSTRUCT = "male, young adult, moderate pitch" - def main(): ap = argparse.ArgumentParser() ap.add_argument("--prompt", default="../examples/prompt.txt") - ap.add_argument("--instruct", default=DEFAULT_INSTRUCT, + ap.add_argument("--instruct", default="male, young adult, moderate pitch", help="natural language style instruction") ap.add_argument("--seed", type=int, default=42) ap.add_argument("--lang", default="english") diff --git a/tools/qwen-tts.cpp b/tools/qwen-tts.cpp index 605aee3..36cd785 100644 --- a/tools/qwen-tts.cpp +++ b/tools/qwen-tts.cpp @@ -13,7 +13,6 @@ // --text or stdin if --text is absent. #include "audio-io.h" -#include "pipeline-codec.h" #include "qwen.h" #include @@ -58,6 +57,9 @@ static void print_usage(const char * prog) { " --sub-temp Sub-talker temperature (default: 0.9)\n" " --sub-top-k Sub-talker top-k (default: 50)\n" " --sub-top-p Sub-talker top-p (default: 1.0)\n\n" + "Backend options:\n" + " --no-fa Disable flash attention (manual F32 attention chain)\n" + " --clamp-fp16 Clamp hidden states + V to FP16 range (sub Ampere CUDA)\n\n" "Debug:\n" " --dump Dump intermediate tensors for cossim debug\n", prog); @@ -86,6 +88,8 @@ struct Args { float subtalker_top_p; float subtalker_temperature; bool subtalker_do_sample; + bool use_fa; + bool clamp_fp16; }; // Read all of stdin into a string. Trims trailing newlines so a piped @@ -143,6 +147,8 @@ static bool parse_args(int argc, char ** argv, Args & a) { a.subtalker_top_k = 50; a.subtalker_top_p = 1.0f; a.subtalker_temperature = 0.9f; + a.use_fa = true; + a.clamp_fp16 = false; for (int i = 1; i < argc; i++) { const char * arg = argv[i]; if (std::strcmp(arg, "-h") == 0 || std::strcmp(arg, "--help") == 0) { @@ -194,6 +200,10 @@ static bool parse_args(int argc, char ** argv, Args & a) { a.subtalker_top_k = std::atoi(argv[++i]); } else if (std::strcmp(arg, "--sub-top-p") == 0 && i + 1 < argc) { a.subtalker_top_p = (float) std::atof(argv[++i]); + } else if (std::strcmp(arg, "--no-fa") == 0) { + a.use_fa = false; + } else if (std::strcmp(arg, "--clamp-fp16") == 0) { + a.clamp_fp16 = true; } else if (std::strcmp(arg, "-o") == 0 && i + 1 < argc) { a.out_wav = argv[++i]; } else { @@ -213,6 +223,8 @@ static int run(const Args & a) { qt_init_default_params(&iparams); iparams.talker_path = a.model; iparams.codec_path = a.codec; + iparams.use_fa = a.use_fa; + iparams.clamp_fp16 = a.clamp_fp16; qt_context * q = qt_init(&iparams); if (!q) { @@ -246,7 +258,7 @@ static int run(const Args & a) { int ref_n_samples = 0; if (a.ref_wav) { int T_in = 0; - float * raw = audio_read_mono(a.ref_wav, TOKENIZER_SAMPLE_RATE, &T_in); + float * raw = audio_read_mono(a.ref_wav, QT_CODEC_SAMPLE_RATE, &T_in); if (!raw || T_in <= 0) { fprintf(stderr, "[CLI] ERROR: cannot read --ref-wav '%s'\n", a.ref_wav); if (raw) {