predictor: static graphs replayed with a single id upload per step

The 15 predictor flavors build and allocate once at load, positions,
kv rows, and mask baked as never freed graph outputs, and replay
directly on the backend. The prefill slices the last position before
lm_head so every flavor reads one logits row at offset zero. Replaces
the per step graph rebuild, sched allocation, and debug prints.
This commit is contained in:
Pascal
2026-07-06 21:46:08 +02:00
parent 66ebf32b2b
commit 2862be4152
6 changed files with 170 additions and 135 deletions
+17 -10
View File
@@ -196,12 +196,14 @@ int main(int argc, char ** argv) {
return names;
};
// The adapter always drives the streaming pipeline : on_chunk routes to
// the shared sink, which either streams to the socket (pcm) or fills a
// one-shot buffer (wav). Either way the audio path is identical. A
// registered voice wins over a model speaker of the same name and
// injects the pre-extracted reference latents. A name matching
// neither is rejected instead of silently generating voiceless.
// pcm drives the streaming pipeline : on_chunk routes each decoded
// frame to the shared sink for lowest latency. wav leaves on_chunk
// unset so the pipeline runs the buffered chunked codec path (batch
// decode, talker uninterrupted) and the whole utterance pushes to
// the sink once. A registered voice wins over a model speaker of
// the same name and injects the pre-extracted reference latents. A
// name matching neither is rejected instead of silently generating
// voiceless.
be.synthesize = [q, &lang](const tts_request & req, const tts_sink & sink, std::string & err) -> int {
struct qt_tts_params p;
qt_tts_default_params(&p);
@@ -259,13 +261,18 @@ int main(int argc, char ** argv) {
// Trampoline : the C ABI on_chunk forwards to the C++ sink.
const tts_sink * sink_ptr = &sink;
p.on_chunk = [](const float * s, int ns, void * u) -> bool {
return (*static_cast<const tts_sink *>(u))(s, ns);
};
p.on_chunk_user_data = (void *) sink_ptr;
if (req.format == "pcm") {
p.on_chunk = [](const float * s, int ns, void * u) -> bool {
return (*static_cast<const tts_sink *>(u))(s, ns);
};
p.on_chunk_user_data = (void *) sink_ptr;
}
struct qt_audio out = {};
enum qt_status rc = qt_synthesize(q, &p, &out);
if (rc == QT_STATUS_OK && p.on_chunk == NULL && out.n_samples > 0) {
sink(out.samples, out.n_samples);
}
qt_audio_free(&out);
if (rc != QT_STATUS_OK) {
err = qt_last_error();