enne2/qwentts.cpp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

cli, pipeline: decode --ref-audio once at the CLI, pipeline takes a mono buffer

Browse Source 
Clone mode used to read the same WAV twice (speaker encoder then codec
encoder), which failed silently on Windows where pipeline_codec_encode
returned empty on the second pass. PipelineTTSSynthesizeParams now takes
ref_audio_24k + ref_n_samples instead of a path. The CLI calls
audio_read_mono once, holds the buffer in a unique_ptr until synthesis
returns, and feeds the same pointer to both encoders.
This commit is contained in:
Pascal
2026-05-11 18:14:59 +02:00
parent d700ffea43
commit 7f532f1ed0
4 changed files with 75 additions and 56 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/pipeline-tts.cpp
+15 -19
View File
@@ -279,18 +279,21 @@ bool pipeline_tts_synthesize(PipelineTTS * pt,
const std::string speaker = params.speaker ? params.speaker : "";
const std::string ref_text = params.ref_text ? params.ref_text : "";
// Voice clone mode A : if ref_audio is given, run the speaker
// encoder on the WAV and feed the resulting embedding straight into
// the prompt builder. Mutually exclusive with --speaker.
// Voice clone mode A : if ref_audio_24k is given, run the speaker
// encoder on the pre-decoded mono buffer and feed the resulting
// embedding straight into the prompt builder. Mutually exclusive
// with --speaker.
const bool has_ref_audio = (params.ref_audio_24k != NULL) && (params.ref_n_samples > 0);
std::vector<float> ref_spk_emb;
const float * ref_spk_emb_ptr = NULL;
if (params.ref_audio && params.ref_audio[0]) {
if (has_ref_audio) {
if (!pt->has_speaker_encoder) {
fprintf(stderr,
"[Pipeline] FATAL: --ref-audio requires a model with a loaded speaker encoder (Base only)\n");
return false;
}
if (!speaker_encoder_extract(&pt->speaker_encoder, pt->sched, params.ref_audio, ref_spk_emb, params.dump_dir)) {
if (!speaker_encoder_extract(&pt->speaker_encoder, pt->sched, params.ref_audio_24k, params.ref_n_samples,
ref_spk_emb, params.dump_dir)) {
return false;
}
if ((int) ref_spk_emb.size() != pt->talker.hidden_size) {
@@ -309,25 +312,18 @@ bool pipeline_tts_synthesize(PipelineTTS * pt,
std::vector<int32_t> ref_codes;
int ref_codes_T = 0;
if (!ref_text.empty()) {
if (!params.ref_audio || !params.ref_audio[0]) {
if (!has_ref_audio) {
fprintf(stderr, "[Pipeline] FATAL: --ref-text requires --ref-audio\n");
return false;
}
// audio_read_mono returns f32 mono at the codec sample rate. The
// codec hop is 1920 samples at 24 kHz so n_samples must be a
// multiple of 1920. Truncate to the nearest hop boundary.
int T_codec_audio = 0;
float * raw = audio_read_mono(params.ref_audio, QWEN_TOKENIZER_SAMPLE_RATE, &T_codec_audio);
if (!raw || T_codec_audio < QWEN_TOKENIZER_HOP_LENGTH) {
fprintf(stderr, "[Pipeline] FATAL: cannot read ref_audio for ICL '%s'\n", params.ref_audio);
if (raw) {
std::free(raw);
}
// The codec hop is 1920 samples at 24 kHz so n_samples must be
// a multiple of 1920. Truncate to the nearest hop boundary.
if (params.ref_n_samples < QWEN_TOKENIZER_HOP_LENGTH) {
fprintf(stderr, "[Pipeline] FATAL: ref_audio too short for ICL (%d samples)\n", params.ref_n_samples);
return false;
}
int aligned_T = (T_codec_audio / QWEN_TOKENIZER_HOP_LENGTH) * QWEN_TOKENIZER_HOP_LENGTH;
ref_codes = pipeline_codec_encode(&pt->codec, raw, aligned_T, params.dump_dir);
std::free(raw);
int aligned_T = (params.ref_n_samples / QWEN_TOKENIZER_HOP_LENGTH) * QWEN_TOKENIZER_HOP_LENGTH;
ref_codes = pipeline_codec_encode(&pt->codec, params.ref_audio_24k, aligned_T, params.dump_dir);
if (ref_codes.empty()) {
fprintf(stderr, "[Pipeline] FATAL: pipeline_codec_encode returned empty codes\n");
return false;
src/pipeline-tts.h
+19 -18
View File
@@ -111,24 +111,25 @@ struct BPETokenizer;
// captures step 0 prefill activations plus the codes-full / output-audio
// dumps under the named directory ; debug only, slows the run.
struct PipelineTTSSynthesizeParams {
const char * text;
const char * lang;
const char * instruct;
const char * speaker;
const char * ref_audio;
const char * ref_text;
int64_t seed;
int max_new_tokens;
bool do_sample;
float temperature;
int top_k;
float top_p;
float repetition_penalty;
bool subtalker_do_sample;
float subtalker_temperature;
int subtalker_top_k;
float subtalker_top_p;
const char * dump_dir;
const char * text;
const char * lang;
const char * instruct;
const char * speaker;
const float * ref_audio_24k;
int ref_n_samples;
const char * ref_text;
int64_t seed;
int max_new_tokens;
bool do_sample;
float temperature;
int top_k;
float top_p;
float repetition_penalty;
bool subtalker_do_sample;
float subtalker_temperature;
int subtalker_top_k;
float subtalker_top_p;
const char * dump_dir;
};
// Output of one synthesis call. audio is a 24 kHz mono F32 PCM buffer
src/speaker-encoder-extract.h
+13 -18
View File
@@ -31,18 +31,24 @@
#include <vector>
// Public entry point. Returns true on success, fills emb_out with the
// 2048-dim f32 embedding. When dump_dir is non NULL, also writes the post
// mel_spectrogram tensor to mel-spk.bin under that directory using the
// debug.h header format. Quiet otherwise.
// 2048-dim f32 embedding. The audio buffer must already be mono at
// sw->sample_rate (24 kHz). When dump_dir is non NULL, also writes the
// post mel_spectrogram tensor to mel-spk.bin under that directory using
// the debug.h header format. Quiet otherwise.
static bool speaker_encoder_extract(const SpeakerEncoderWeights * sw,
ggml_backend_sched_t sched,
const char * wav_path,
const float * audio,
int n_samples,
std::vector<float> & emb_out,
const char * dump_dir = NULL) {
if (sw->weight_buf == NULL) {
fprintf(stderr, "[SpkExtract] FATAL: speaker encoder weights not loaded\n");
return false;
}
if (!audio || n_samples <= 0) {
fprintf(stderr, "[SpkExtract] FATAL: empty audio buffer\n");
return false;
}
AudioMelConfig mel_cfg;
mel_cfg.sample_rate = sw->sample_rate;
@@ -52,18 +58,8 @@ static bool speaker_encoder_extract(const SpeakerEncoderWeights * sw,
mel_cfg.fmin = 0.0f;
mel_cfg.fmax = 12000.0f;
// Load WAV, mono mix, resample to 24 kHz. audio_read_mono allocates
// with malloc, wrap in a unique_ptr for clean release.
int T_in = 0;
float * raw = audio_read_mono(wav_path, sw->sample_rate, &T_in);
if (!raw || T_in <= 0) {
fprintf(stderr, "[SpkExtract] FATAL: cannot read WAV '%s'\n", wav_path);
if (raw) {
std::free(raw);
}
return false;
}
std::unique_ptr<float, void (*)(void *)> raw_holder(raw, std::free);
const int T_in = n_samples;
const float * raw = audio;
const int pad = (mel_cfg.n_fft - mel_cfg.hop) / 2; // 384
const int T_pad = T_in + 2 * pad;
@@ -290,7 +286,6 @@ static bool speaker_encoder_extract(const SpeakerEncoderWeights * sw,
ggml_backend_sched_reset(sched);
ggml_free(gctx);
fprintf(stderr, "[SpkExtract] Extracted %d-dim embedding from %s (%d samples, padded %d)\n", sw->enc_dim, wav_path,
T_in, T_pad);
fprintf(stderr, "[SpkExtract] Extracted %d-dim embedding (%d samples, padded %d)\n", sw->enc_dim, T_in, T_pad);
return true;
}