enne2/qwentts.cpp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

cli, pipeline: decode --ref-audio once at the CLI, pipeline takes a mono buffer

Browse Source 
Clone mode used to read the same WAV twice (speaker encoder then codec
encoder), which failed silently on Windows where pipeline_codec_encode
returned empty on the second pass. PipelineTTSSynthesizeParams now takes
ref_audio_24k + ref_n_samples instead of a path. The CLI calls
audio_read_mono once, holds the buffer in a unique_ptr until synthesis
returns, and feeds the same pointer to both encoders.
This commit is contained in:
Pascal
2026-05-11 18:14:59 +02:00
parent d700ffea43
commit 7f532f1ed0
4 changed files with 75 additions and 56 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/pipeline-tts.cpp
+15 -19
View File
@@ -279,18 +279,21 @@ bool pipeline_tts_synthesize(PipelineTTS * pt,
const std::string speaker = params.speaker ? params.speaker : "";
const std::string ref_text = params.ref_text ? params.ref_text : "";
// Voice clone mode A : if ref_audio is given, run the speaker
// encoder on the WAV and feed the resulting embedding straight into
// the prompt builder. Mutually exclusive with --speaker.
// Voice clone mode A : if ref_audio_24k is given, run the speaker
// encoder on the pre-decoded mono buffer and feed the resulting
// embedding straight into the prompt builder. Mutually exclusive
// with --speaker.
const bool has_ref_audio = (params.ref_audio_24k != NULL) && (params.ref_n_samples > 0);
std::vector<float> ref_spk_emb;
const float * ref_spk_emb_ptr = NULL;
if (params.ref_audio && params.ref_audio[0]) {
if (has_ref_audio) {
if (!pt->has_speaker_encoder) {
fprintf(stderr,
"[Pipeline] FATAL: --ref-audio requires a model with a loaded speaker encoder (Base only)\n");
return false;
}
if (!speaker_encoder_extract(&pt->speaker_encoder, pt->sched, params.ref_audio, ref_spk_emb, params.dump_dir)) {
if (!speaker_encoder_extract(&pt->speaker_encoder, pt->sched, params.ref_audio_24k, params.ref_n_samples,
ref_spk_emb, params.dump_dir)) {
return false;
}
if ((int) ref_spk_emb.size() != pt->talker.hidden_size) {
@@ -309,25 +312,18 @@ bool pipeline_tts_synthesize(PipelineTTS * pt,
std::vector<int32_t> ref_codes;
int ref_codes_T = 0;
if (!ref_text.empty()) {
if (!params.ref_audio || !params.ref_audio[0]) {
if (!has_ref_audio) {
fprintf(stderr, "[Pipeline] FATAL: --ref-text requires --ref-audio\n");
return false;
}
// audio_read_mono returns f32 mono at the codec sample rate. The
// codec hop is 1920 samples at 24 kHz so n_samples must be a
// multiple of 1920. Truncate to the nearest hop boundary.
int T_codec_audio = 0;
float * raw = audio_read_mono(params.ref_audio, QWEN_TOKENIZER_SAMPLE_RATE, &T_codec_audio);
if (!raw || T_codec_audio < QWEN_TOKENIZER_HOP_LENGTH) {
fprintf(stderr, "[Pipeline] FATAL: cannot read ref_audio for ICL '%s'\n", params.ref_audio);
if (raw) {
std::free(raw);
}
// The codec hop is 1920 samples at 24 kHz so n_samples must be
// a multiple of 1920. Truncate to the nearest hop boundary.
if (params.ref_n_samples < QWEN_TOKENIZER_HOP_LENGTH) {
fprintf(stderr, "[Pipeline] FATAL: ref_audio too short for ICL (%d samples)\n", params.ref_n_samples);
return false;
}
int aligned_T = (T_codec_audio / QWEN_TOKENIZER_HOP_LENGTH) * QWEN_TOKENIZER_HOP_LENGTH;
ref_codes = pipeline_codec_encode(&pt->codec, raw, aligned_T, params.dump_dir);
std::free(raw);
int aligned_T = (params.ref_n_samples / QWEN_TOKENIZER_HOP_LENGTH) * QWEN_TOKENIZER_HOP_LENGTH;
ref_codes = pipeline_codec_encode(&pt->codec, params.ref_audio_24k, aligned_T, params.dump_dir);
if (ref_codes.empty()) {
fprintf(stderr, "[Pipeline] FATAL: pipeline_codec_encode returned empty codes\n");
return false;
src/pipeline-tts.h
+2 -1
View File
@@ -115,7 +115,8 @@ struct PipelineTTSSynthesizeParams {
const char * lang;
const char * instruct;
const char * speaker;
const char * ref_audio;
const float * ref_audio_24k;
int ref_n_samples;
const char * ref_text;
int64_t seed;
int max_new_tokens;
src/speaker-encoder-extract.h
+13 -18
View File
@@ -31,18 +31,24 @@
#include <vector>
// Public entry point. Returns true on success, fills emb_out with the
// 2048-dim f32 embedding. When dump_dir is non NULL, also writes the post
// mel_spectrogram tensor to mel-spk.bin under that directory using the
// debug.h header format. Quiet otherwise.
// 2048-dim f32 embedding. The audio buffer must already be mono at
// sw->sample_rate (24 kHz). When dump_dir is non NULL, also writes the
// post mel_spectrogram tensor to mel-spk.bin under that directory using
// the debug.h header format. Quiet otherwise.
static bool speaker_encoder_extract(const SpeakerEncoderWeights * sw,
ggml_backend_sched_t sched,
const char * wav_path,
const float * audio,
int n_samples,
std::vector<float> & emb_out,
const char * dump_dir = NULL) {
if (sw->weight_buf == NULL) {
fprintf(stderr, "[SpkExtract] FATAL: speaker encoder weights not loaded\n");
return false;
}
if (!audio || n_samples <= 0) {
fprintf(stderr, "[SpkExtract] FATAL: empty audio buffer\n");
return false;
}
AudioMelConfig mel_cfg;
mel_cfg.sample_rate = sw->sample_rate;
@@ -52,18 +58,8 @@ static bool speaker_encoder_extract(const SpeakerEncoderWeights * sw,
mel_cfg.fmin = 0.0f;
mel_cfg.fmax = 12000.0f;
// Load WAV, mono mix, resample to 24 kHz. audio_read_mono allocates
// with malloc, wrap in a unique_ptr for clean release.
int T_in = 0;
float * raw = audio_read_mono(wav_path, sw->sample_rate, &T_in);
if (!raw || T_in <= 0) {
fprintf(stderr, "[SpkExtract] FATAL: cannot read WAV '%s'\n", wav_path);
if (raw) {
std::free(raw);
}
return false;
}
std::unique_ptr<float, void (*)(void *)> raw_holder(raw, std::free);
const int T_in = n_samples;
const float * raw = audio;
const int pad = (mel_cfg.n_fft - mel_cfg.hop) / 2; // 384
const int T_pad = T_in + 2 * pad;
@@ -290,7 +286,6 @@ static bool speaker_encoder_extract(const SpeakerEncoderWeights * sw,
ggml_backend_sched_reset(sched);
ggml_free(gctx);
fprintf(stderr, "[SpkExtract] Extracted %d-dim embedding from %s (%d samples, padded %d)\n", sw->enc_dim, wav_path,
T_in, T_pad);
fprintf(stderr, "[SpkExtract] Extracted %d-dim embedding (%d samples, padded %d)\n", sw->enc_dim, T_in, T_pad);
return true;
}
tools/qwen-tts.cpp
+28 -1
View File
@@ -20,6 +20,7 @@
#include <cstdlib>
#include <cstring>
#include <iostream>
#include <memory>
#include <random>
#include <sstream>
#include <stdexcept>
@@ -282,6 +283,31 @@ static int run(const Args & a) {
ref_text = ref_text_buf.c_str();
}
// Decode the reference WAV once, mono at the codec sample rate. The
// pipeline consumes the buffer directly so the WAV is read exactly
// once regardless of how many encoders need the audio (speaker
// encoder embedding + codec encoder RVQ codes for ICL mode B).
std::vector<float> ref_audio_buf;
std::unique_ptr<float, void (*)(void *)> raw_holder(NULL, std::free);
const float * ref_audio_24k = NULL;
int ref_n_samples = 0;
if (a.ref_audio) {
int T_in = 0;
float * raw = audio_read_mono(a.ref_audio, QWEN_TOKENIZER_SAMPLE_RATE, &T_in);
if (!raw || T_in <= 0) {
fprintf(stderr, "[CLI] ERROR: cannot read --ref-audio '%s'\n", a.ref_audio);
if (raw) {
std::free(raw);
}
pipeline_tts_free(&pt);
backend_release(bp.backend, bp.cpu_backend);
return 1;
}
raw_holder.reset(raw);
ref_audio_24k = raw;
ref_n_samples = T_in;
}
// Resolve output WAV format string : wav16 / wav24 / wav32. Default
// wav16 mirrors the omnivoice.cpp default.
WavFormat wav_fmt;
@@ -333,7 +359,8 @@ static int run(const Args & a) {
p.lang = a.lang;
p.instruct = a.instruct;
p.speaker = a.speaker;
p.ref_audio = a.ref_audio;
p.ref_audio_24k = ref_audio_24k;
p.ref_n_samples = ref_n_samples;
p.ref_text = ref_text;
p.seed = seed;
p.max_new_tokens = a.max_new_tokens;