talker, code predictor: persistent KV cache
Adds a per-layer K/V ring (kv-cache.h) backed by a dedicated backend buffer, sized at init for max_seq_len positions. The talker holds a 4096 position cache (896 MB f32) for the LM context, the code predictor a 16 position cache (~80 KB) reset every frame. talker_forward splits into prefill (resets the cache and writes T positions in one shot) and decode (appends one position, reads the [0, cur_len+1) window). code_predictor_step does the same with a T=2 prefill plus 14 single token decodes. Bit identical audio output, validated by sha256 against the pre KV cache run on a 64 frame F32 reference seed=42. Walltime drops ~10% on a single utterance ; the win scales with sequence length and unlocks frame by frame streaming.
This commit is contained in:
@@ -11,6 +11,7 @@
|
||||
#include "code-predictor-weights.h"
|
||||
#include "ggml-backend.h"
|
||||
#include "gguf-weights.h"
|
||||
#include "kv-cache.h"
|
||||
#include "pipeline-codec.h"
|
||||
#include "speaker-encoder-weights.h"
|
||||
#include "talker-weights.h"
|
||||
@@ -88,6 +89,12 @@ struct PipelineTTS {
|
||||
BackendPair bp;
|
||||
ggml_backend_t backend;
|
||||
ggml_backend_sched_t sched;
|
||||
|
||||
// Persistent KV caches : the talker holds the LM context, the
|
||||
// predictor holds one frame's 16 sub-steps and gets reset every
|
||||
// frame in code_predictor_step.
|
||||
KVCache talker_kv;
|
||||
KVCache code_predictor_kv;
|
||||
};
|
||||
|
||||
// Open the talker GGUF and the codec GGUF, load every module on the
|
||||
|
||||
Reference in New Issue
Block a user