cli, pipeline: decode --ref-audio once at the CLI, pipeline takes a mono buffer

Clone mode used to read the same WAV twice (speaker encoder then codec
encoder), which failed silently on Windows where pipeline_codec_encode
returned empty on the second pass. PipelineTTSSynthesizeParams now takes
ref_audio_24k + ref_n_samples instead of a path. The CLI calls
audio_read_mono once, holds the buffer in a unique_ptr until synthesis
returns, and feeds the same pointer to both encoders.

src/pipeline-tts.cpp

@@ -279,18 +279,21 @@ bool pipeline_tts_synthesize(PipelineTTS *                       pt,
     const std::string   speaker  = params.speaker ? params.speaker : "";
     const std::string   ref_text = params.ref_text ? params.ref_text : "";
 
-    // Voice clone mode A : if ref_audio is given, run the speaker
+    // Voice clone mode A : if ref_audio_24k is given, run the speaker
-    // encoder on the WAV and feed the resulting embedding straight into
+    // encoder on the pre-decoded mono buffer and feed the resulting
-    // the prompt builder. Mutually exclusive with --speaker.
+    // embedding straight into the prompt builder. Mutually exclusive
+    // with --speaker.
+    const bool         has_ref_audio = (params.ref_audio_24k != NULL) && (params.ref_n_samples > 0);
     std::vector<float> ref_spk_emb;
     const float *      ref_spk_emb_ptr = NULL;
-    if (params.ref_audio && params.ref_audio[0]) {
+    if (has_ref_audio) {
         if (!pt->has_speaker_encoder) {
             fprintf(stderr,
                     "[Pipeline] FATAL: --ref-audio requires a model with a loaded speaker encoder (Base only)\n");
             return false;
         }
-        if (!speaker_encoder_extract(&pt->speaker_encoder, pt->sched, params.ref_audio, ref_spk_emb, params.dump_dir)) {
+        if (!speaker_encoder_extract(&pt->speaker_encoder, pt->sched, params.ref_audio_24k, params.ref_n_samples,
+                                     ref_spk_emb, params.dump_dir)) {
             return false;
         }
         if ((int) ref_spk_emb.size() != pt->talker.hidden_size) {
@@ -309,25 +312,18 @@ bool pipeline_tts_synthesize(PipelineTTS *                       pt,
     std::vector<int32_t> ref_codes;
     int                  ref_codes_T = 0;
     if (!ref_text.empty()) {
-        if (!params.ref_audio || !params.ref_audio[0]) {
+        if (!has_ref_audio) {
             fprintf(stderr, "[Pipeline] FATAL: --ref-text requires --ref-audio\n");
             return false;
         }
-        // audio_read_mono returns f32 mono at the codec sample rate. The
+        // The codec hop is 1920 samples at 24 kHz so n_samples must be
-        // codec hop is 1920 samples at 24 kHz so n_samples must be a
+        // a multiple of 1920. Truncate to the nearest hop boundary.
-        // multiple of 1920. Truncate to the nearest hop boundary.
+        if (params.ref_n_samples < QWEN_TOKENIZER_HOP_LENGTH) {
-        int     T_codec_audio = 0;
+            fprintf(stderr, "[Pipeline] FATAL: ref_audio too short for ICL (%d samples)\n", params.ref_n_samples);
-        float * raw           = audio_read_mono(params.ref_audio, QWEN_TOKENIZER_SAMPLE_RATE, &T_codec_audio);
-        if (!raw || T_codec_audio < QWEN_TOKENIZER_HOP_LENGTH) {
-            fprintf(stderr, "[Pipeline] FATAL: cannot read ref_audio for ICL '%s'\n", params.ref_audio);
-            if (raw) {
-                std::free(raw);
-            }
             return false;
         }
-        int aligned_T = (T_codec_audio / QWEN_TOKENIZER_HOP_LENGTH) * QWEN_TOKENIZER_HOP_LENGTH;
+        int aligned_T = (params.ref_n_samples / QWEN_TOKENIZER_HOP_LENGTH) * QWEN_TOKENIZER_HOP_LENGTH;
-        ref_codes     = pipeline_codec_encode(&pt->codec, raw, aligned_T, params.dump_dir);
+        ref_codes     = pipeline_codec_encode(&pt->codec, params.ref_audio_24k, aligned_T, params.dump_dir);
-        std::free(raw);
         if (ref_codes.empty()) {
             fprintf(stderr, "[Pipeline] FATAL: pipeline_codec_encode returned empty codes\n");
             return false;

src/pipeline-tts.h

@@ -111,24 +111,25 @@ struct BPETokenizer;
 // captures step 0 prefill activations plus the codes-full / output-audio
 // dumps under the named directory ; debug only, slows the run.
 struct PipelineTTSSynthesizeParams {
-    const char * text;
+    const char *  text;
-    const char * lang;
+    const char *  lang;
-    const char * instruct;
+    const char *  instruct;
-    const char * speaker;
+    const char *  speaker;
-    const char * ref_audio;
+    const float * ref_audio_24k;
-    const char * ref_text;
+    int           ref_n_samples;
-    int64_t      seed;
+    const char *  ref_text;
-    int          max_new_tokens;
+    int64_t       seed;
-    bool         do_sample;
+    int           max_new_tokens;
-    float        temperature;
+    bool          do_sample;
-    int          top_k;
+    float         temperature;
-    float        top_p;
+    int           top_k;
-    float        repetition_penalty;
+    float         top_p;
-    bool         subtalker_do_sample;
+    float         repetition_penalty;
-    float        subtalker_temperature;
+    bool          subtalker_do_sample;
-    int          subtalker_top_k;
+    float         subtalker_temperature;
-    float        subtalker_top_p;
+    int           subtalker_top_k;
-    const char * dump_dir;
+    float         subtalker_top_p;
+    const char *  dump_dir;
 };
 
 // Output of one synthesis call. audio is a 24 kHz mono F32 PCM buffer

src/speaker-encoder-extract.h

@@ -31,18 +31,24 @@
 #include <vector>
 
 // Public entry point. Returns true on success, fills emb_out with the
-// 2048-dim f32 embedding. When dump_dir is non NULL, also writes the post
+// 2048-dim f32 embedding. The audio buffer must already be mono at
-// mel_spectrogram tensor to mel-spk.bin under that directory using the
+// sw->sample_rate (24 kHz). When dump_dir is non NULL, also writes the
-// debug.h header format. Quiet otherwise.
+// post mel_spectrogram tensor to mel-spk.bin under that directory using
+// the debug.h header format. Quiet otherwise.
 static bool speaker_encoder_extract(const SpeakerEncoderWeights * sw,
                                     ggml_backend_sched_t          sched,
-                                    const char *                  wav_path,
+                                    const float *                 audio,
+                                    int                           n_samples,
                                     std::vector<float> &          emb_out,
                                     const char *                  dump_dir = NULL) {
     if (sw->weight_buf == NULL) {
         fprintf(stderr, "[SpkExtract] FATAL: speaker encoder weights not loaded\n");
         return false;
     }
+    if (!audio || n_samples <= 0) {
+        fprintf(stderr, "[SpkExtract] FATAL: empty audio buffer\n");
+        return false;
+    }
 
     AudioMelConfig mel_cfg;
     mel_cfg.sample_rate = sw->sample_rate;
@@ -52,18 +58,8 @@ static bool speaker_encoder_extract(const SpeakerEncoderWeights * sw,
     mel_cfg.fmin        = 0.0f;
     mel_cfg.fmax        = 12000.0f;
 
-    // Load WAV, mono mix, resample to 24 kHz. audio_read_mono allocates
+    const int     T_in = n_samples;
-    // with malloc, wrap in a unique_ptr for clean release.
+    const float * raw  = audio;
-    int     T_in = 0;
-    float * raw  = audio_read_mono(wav_path, sw->sample_rate, &T_in);
-    if (!raw || T_in <= 0) {
-        fprintf(stderr, "[SpkExtract] FATAL: cannot read WAV '%s'\n", wav_path);
-        if (raw) {
-            std::free(raw);
-        }
-        return false;
-    }
-    std::unique_ptr<float, void (*)(void *)> raw_holder(raw, std::free);
 
     const int pad   = (mel_cfg.n_fft - mel_cfg.hop) / 2;  // 384
     const int T_pad = T_in + 2 * pad;
@@ -290,7 +286,6 @@ static bool speaker_encoder_extract(const SpeakerEncoderWeights * sw,
     ggml_backend_sched_reset(sched);
     ggml_free(gctx);
 
-    fprintf(stderr, "[SpkExtract] Extracted %d-dim embedding from %s (%d samples, padded %d)\n", sw->enc_dim, wav_path,
+    fprintf(stderr, "[SpkExtract] Extracted %d-dim embedding (%d samples, padded %d)\n", sw->enc_dim, T_in, T_pad);
-            T_in, T_pad);
     return true;
 }

tools/qwen-tts.cpp

@@ -20,6 +20,7 @@
 #include <cstdlib>
 #include <cstring>
 #include <iostream>
+#include <memory>
 #include <random>
 #include <sstream>
 #include <stdexcept>
@@ -282,6 +283,31 @@ static int run(const Args & a) {
         ref_text = ref_text_buf.c_str();
     }
 
+    // Decode the reference WAV once, mono at the codec sample rate. The
+    // pipeline consumes the buffer directly so the WAV is read exactly
+    // once regardless of how many encoders need the audio (speaker
+    // encoder embedding + codec encoder RVQ codes for ICL mode B).
+    std::vector<float>                       ref_audio_buf;
+    std::unique_ptr<float, void (*)(void *)> raw_holder(NULL, std::free);
+    const float *                            ref_audio_24k = NULL;
+    int                                      ref_n_samples = 0;
+    if (a.ref_audio) {
+        int     T_in = 0;
+        float * raw  = audio_read_mono(a.ref_audio, QWEN_TOKENIZER_SAMPLE_RATE, &T_in);
+        if (!raw || T_in <= 0) {
+            fprintf(stderr, "[CLI] ERROR: cannot read --ref-audio '%s'\n", a.ref_audio);
+            if (raw) {
+                std::free(raw);
+            }
+            pipeline_tts_free(&pt);
+            backend_release(bp.backend, bp.cpu_backend);
+            return 1;
+        }
+        raw_holder.reset(raw);
+        ref_audio_24k = raw;
+        ref_n_samples = T_in;
+    }
+
     // Resolve output WAV format string : wav16 / wav24 / wav32. Default
     // wav16 mirrors the omnivoice.cpp default.
     WavFormat wav_fmt;
@@ -333,7 +359,8 @@ static int run(const Args & a) {
     p.lang                        = a.lang;
     p.instruct                    = a.instruct;
     p.speaker                     = a.speaker;
-    p.ref_audio                   = a.ref_audio;
+    p.ref_audio_24k               = ref_audio_24k;
+    p.ref_n_samples               = ref_n_samples;
     p.ref_text                    = ref_text;
     p.seed                        = seed;
     p.max_new_tokens              = a.max_new_tokens;