tts: seed the vocoder left context with the ICL reference tail
The upstream pipeline decodes reference plus generated codes then trims, giving the first generated frames causal context from the reference; the generated only decode started the vocoder from an empty state and colored the clone onset. Seed both decode paths with the last min(ref_T, left_ctx_frames) reference frames: the streaming decoder takes them below its emit cursor so they are never emitted, the buffered path prepends them and strips their samples. Raising codec_left_context_sec past the reference duration reproduces the upstream full reference decode exactly.
This commit is contained in:
@@ -112,6 +112,26 @@ struct codec_chunked_decoder_stream {
|
||||
by_k.assign((size_t) K, {});
|
||||
}
|
||||
|
||||
// Seed the left context with the tail of the ICL reference codes so
|
||||
// the first emitted chunk draws causal context from the reference
|
||||
// instead of an empty decoder state, matching the upstream pipeline
|
||||
// which decodes reference plus generated then trims. ref_kt is K
|
||||
// major [K, ref_T]; the last min(ref_T, left_ctx_frames) frames are
|
||||
// kept. Call once, after init and before any push_frame; the seeded
|
||||
// frames sit below emit_start_frame so they are never emitted.
|
||||
void seed_reference(const int32_t * ref_kt, int ref_T) {
|
||||
int seed = ref_T < left_ctx_frames ? ref_T : left_ctx_frames;
|
||||
if (seed <= 0) {
|
||||
return;
|
||||
}
|
||||
for (int k = 0; k < K; k++) {
|
||||
const int32_t * row = ref_kt + (size_t) k * (size_t) ref_T + (size_t) (ref_T - seed);
|
||||
by_k[(size_t) k].insert(by_k[(size_t) k].end(), row, row + seed);
|
||||
}
|
||||
T_so_far = seed;
|
||||
emit_start_frame = seed;
|
||||
}
|
||||
|
||||
// Append one frame (K int32 codes, one per codebook). Drain any
|
||||
// chunks that became emittable. Returns false on decode failure or
|
||||
// when cb returns false (cancellation).
|
||||
|
||||
+29
-5
@@ -583,6 +583,11 @@ qt_status pipeline_tts_synthesize(PipelineTTS * pt,
|
||||
codec_chunked_decoder_stream stream;
|
||||
if (streaming) {
|
||||
stream.init(num_codebooks, chunk_frames, left_ctx_frames);
|
||||
// ICL clone: the reference tail seeds the decoder left context
|
||||
// so the onset is voiced with the reference's causal state.
|
||||
if (ref_codes_ptr != NULL) {
|
||||
stream.seed_reference(ref_codes_ptr, ref_codes_T);
|
||||
}
|
||||
}
|
||||
|
||||
for (int step = 0; step < params->max_new_tokens; step++) {
|
||||
@@ -794,23 +799,42 @@ qt_status pipeline_tts_synthesize(PipelineTTS * pt,
|
||||
// equivalent to a single pipeline_codec_decode call when T_frames
|
||||
// fits in one chunk, bounded VRAM beyond that. Transpose codes from
|
||||
// [T_frames, K] to [K, T_frames] because codec_chunked_decode
|
||||
// expects K major layout.
|
||||
// expects K major layout. On the ICL clone path the tail of the
|
||||
// reference codes prepends the buffer so the onset is voiced with
|
||||
// the reference's causal state, mirroring the upstream pipeline
|
||||
// which decodes reference plus generated then trims; the seeded
|
||||
// samples strip from the front afterwards. Raising
|
||||
// codec_left_context_sec past the reference duration reproduces the
|
||||
// upstream full reference decode exactly.
|
||||
const int T_frames = (int) all_codes.size();
|
||||
std::vector<int32_t> codes_kt((size_t) num_codebooks * (size_t) T_frames);
|
||||
for (int t = 0; t < T_frames; t++) {
|
||||
int seed = 0;
|
||||
if (ref_codes_ptr != NULL) {
|
||||
seed = ref_codes_T < left_ctx_frames ? ref_codes_T : left_ctx_frames;
|
||||
}
|
||||
const int T_dec = seed + T_frames;
|
||||
std::vector<int32_t> codes_kt((size_t) num_codebooks * (size_t) T_dec);
|
||||
for (int k = 0; k < num_codebooks; k++) {
|
||||
codes_kt[(size_t) k * (size_t) T_frames + (size_t) t] = all_codes[(size_t) t][(size_t) k];
|
||||
int32_t * row = codes_kt.data() + (size_t) k * (size_t) T_dec;
|
||||
if (seed > 0) {
|
||||
std::memcpy(row, ref_codes_ptr + (size_t) k * (size_t) ref_codes_T + (size_t) (ref_codes_T - seed),
|
||||
(size_t) seed * sizeof(int32_t));
|
||||
}
|
||||
for (int t = 0; t < T_frames; t++) {
|
||||
row[(size_t) (seed + t)] = all_codes[(size_t) t][(size_t) k];
|
||||
}
|
||||
}
|
||||
Timer t_codec;
|
||||
std::vector<float> audio =
|
||||
codec_chunked_decode(&pt->codec, codes_kt.data(), num_codebooks, T_frames, chunk_frames, left_ctx_frames);
|
||||
codec_chunked_decode(&pt->codec, codes_kt.data(), num_codebooks, T_dec, chunk_frames, left_ctx_frames);
|
||||
perf.codec_ms += t_codec.ms();
|
||||
if (audio.empty()) {
|
||||
qt_set_error("pipeline_tts_synthesize: codec decode returned no audio");
|
||||
qt_log(QT_LOG_ERROR, "[Pipeline] codec decode returned no audio");
|
||||
return QT_STATUS_GENERATE_FAILED;
|
||||
}
|
||||
if (seed > 0) {
|
||||
audio.erase(audio.begin(), audio.begin() + (size_t) seed * (size_t) TOKENIZER_HOP_LENGTH);
|
||||
}
|
||||
|
||||
if (params->dump_dir) {
|
||||
DebugDumper d;
|
||||
|
||||
Reference in New Issue
Block a user