tts: persistent graph arenas and padded attention windows
Rebuild each forward into a persistent arena per graph shape class (one for the talker, two for the code predictor prefill and step flavors that alternate within a frame) so nodes keep stable addresses and the CUDA graph cache replays its executable instead of reinstantiating. Pad the talker attention window to 256 and fix the predictor window to the frame cache size so decode shapes hold across steps, with the causal mask carrying neg inf over the padded tail. Drop the per step ggml context churn and the trailing sched resets: one talker step plus 15 predictor micro steps per frame no longer pay a full build/alloc/free cycle each.
This commit is contained in:
@@ -14,6 +14,7 @@
|
||||
#include "code-predictor-weights.h"
|
||||
#include "ggml-backend.h"
|
||||
#include "gguf-weights.h"
|
||||
#include "graph-arena.h"
|
||||
#include "kv-cache.h"
|
||||
#include "pipeline-codec.h"
|
||||
#include "qwen.h"
|
||||
@@ -126,6 +127,15 @@ struct PipelineTTS {
|
||||
// frame in code_predictor_step.
|
||||
KVCache talker_kv;
|
||||
KVCache code_predictor_kv;
|
||||
|
||||
// Persistent graph arenas, one per graph shape class. Stable node
|
||||
// addresses across rebuilds keep the backend CUDA graph cache hot:
|
||||
// the talker shares one arena for prefill and decode, the predictor
|
||||
// splits prefill (T=2) and step (T=1) so the two flavors that
|
||||
// alternate within a frame each keep their own executable.
|
||||
GraphArena talker_arena;
|
||||
GraphArena cp_prefill_arena;
|
||||
GraphArena cp_step_arena;
|
||||
};
|
||||
|
||||
// Open the talker GGUF and the codec GGUF, load every module on the
|
||||
|
||||
Reference in New Issue
Block a user