tts: seed the vocoder left context with the ICL reference tail

The upstream pipeline decodes reference plus generated codes then trims,
giving the first generated frames causal context from the reference; the
generated only decode started the vocoder from an empty state and
colored the clone onset. Seed both decode paths with the last
min(ref_T, left_ctx_frames) reference frames: the streaming decoder
takes them below its emit cursor so they are never emitted, the buffered
path prepends them and strips their samples. Raising
codec_left_context_sec past the reference duration reproduces the
upstream full reference decode exactly.
This commit is contained in:
Pascal
2026-07-04 22:47:41 +02:00
parent 3e3b6c1712
commit 4057d0331f
2 changed files with 51 additions and 7 deletions
+20
View File
@@ -112,6 +112,26 @@ struct codec_chunked_decoder_stream {
by_k.assign((size_t) K, {});
}
// Seed the left context with the tail of the ICL reference codes so
// the first emitted chunk draws causal context from the reference
// instead of an empty decoder state, matching the upstream pipeline
// which decodes reference plus generated then trims. ref_kt is K
// major [K, ref_T]; the last min(ref_T, left_ctx_frames) frames are
// kept. Call once, after init and before any push_frame; the seeded
// frames sit below emit_start_frame so they are never emitted.
void seed_reference(const int32_t * ref_kt, int ref_T) {
int seed = ref_T < left_ctx_frames ? ref_T : left_ctx_frames;
if (seed <= 0) {
return;
}
for (int k = 0; k < K; k++) {
const int32_t * row = ref_kt + (size_t) k * (size_t) ref_T + (size_t) (ref_T - seed);
by_k[(size_t) k].insert(by_k[(size_t) k].end(), row, row + seed);
}
T_so_far = seed;
emit_start_frame = seed;
}
// Append one frame (K int32 codes, one per codebook). Drain any
// chunks that became emittable. Returns false on decode failure or
// when cb returns false (cancellation).
+31 -7
View File
@@ -583,6 +583,11 @@ qt_status pipeline_tts_synthesize(PipelineTTS * pt,
codec_chunked_decoder_stream stream;
if (streaming) {
stream.init(num_codebooks, chunk_frames, left_ctx_frames);
// ICL clone: the reference tail seeds the decoder left context
// so the onset is voiced with the reference's causal state.
if (ref_codes_ptr != NULL) {
stream.seed_reference(ref_codes_ptr, ref_codes_T);
}
}
for (int step = 0; step < params->max_new_tokens; step++) {
@@ -794,23 +799,42 @@ qt_status pipeline_tts_synthesize(PipelineTTS * pt,
// equivalent to a single pipeline_codec_decode call when T_frames
// fits in one chunk, bounded VRAM beyond that. Transpose codes from
// [T_frames, K] to [K, T_frames] because codec_chunked_decode
// expects K major layout.
const int T_frames = (int) all_codes.size();
std::vector<int32_t> codes_kt((size_t) num_codebooks * (size_t) T_frames);
for (int t = 0; t < T_frames; t++) {
for (int k = 0; k < num_codebooks; k++) {
codes_kt[(size_t) k * (size_t) T_frames + (size_t) t] = all_codes[(size_t) t][(size_t) k];
// expects K major layout. On the ICL clone path the tail of the
// reference codes prepends the buffer so the onset is voiced with
// the reference's causal state, mirroring the upstream pipeline
// which decodes reference plus generated then trims; the seeded
// samples strip from the front afterwards. Raising
// codec_left_context_sec past the reference duration reproduces the
// upstream full reference decode exactly.
const int T_frames = (int) all_codes.size();
int seed = 0;
if (ref_codes_ptr != NULL) {
seed = ref_codes_T < left_ctx_frames ? ref_codes_T : left_ctx_frames;
}
const int T_dec = seed + T_frames;
std::vector<int32_t> codes_kt((size_t) num_codebooks * (size_t) T_dec);
for (int k = 0; k < num_codebooks; k++) {
int32_t * row = codes_kt.data() + (size_t) k * (size_t) T_dec;
if (seed > 0) {
std::memcpy(row, ref_codes_ptr + (size_t) k * (size_t) ref_codes_T + (size_t) (ref_codes_T - seed),
(size_t) seed * sizeof(int32_t));
}
for (int t = 0; t < T_frames; t++) {
row[(size_t) (seed + t)] = all_codes[(size_t) t][(size_t) k];
}
}
Timer t_codec;
std::vector<float> audio =
codec_chunked_decode(&pt->codec, codes_kt.data(), num_codebooks, T_frames, chunk_frames, left_ctx_frames);
codec_chunked_decode(&pt->codec, codes_kt.data(), num_codebooks, T_dec, chunk_frames, left_ctx_frames);
perf.codec_ms += t_codec.ms();
if (audio.empty()) {
qt_set_error("pipeline_tts_synthesize: codec decode returned no audio");
qt_log(QT_LOG_ERROR, "[Pipeline] codec decode returned no audio");
return QT_STATUS_GENERATE_FAILED;
}
if (seed > 0) {
audio.erase(audio.begin(), audio.begin() + (size_t) seed * (size_t) TOKENIZER_HOP_LENGTH);
}
if (params->dump_dir) {
DebugDumper d;