predictor: unroll the frame into one cgraph and sample in standard ops

One static frame graph per batch width replaces the per step chain:
prefill and the 15 acoustic steps run in a single backend compute.
This is the target architecture for the llama.cpp Qwen3-TTS port and
serves as its working GGML reference while under test.

Sampling is a plain op chain batched over slots: temperature, argsort
top_k (descending order is guaranteed on every backend, unlike top_k),
softmax, cumsum, cdf crossing against a per step philox uniform.
Greedy draws with u = 0 and lands on the argmax. Faster than the
fused sampling op under CUDA graph capture, greedy codes stay exact
against the Python reference on CPU, CUDA and Vulkan.

Opt in single slot latency mode (--codec-fused on qwen-tts and
tts-server, codec_fused in qt_init_params): the codec stream tail
joins the frame graph at T=1, codes read through a device view, one
80 ms chunk per compute with no host round trip.

Predictor 3.34 -> 3.11 ms/frame on CUDA, end to end -4%.

src/code-predictor-forward.h

@@ -31,10 +31,9 @@
 //   - one private embedding table and one private linear head per
 //     acoustic codebook (1..15)
 //
-// Graph metadata lives in caller owned static graphs, one per flavor
+// Graph metadata lives in a caller owned static frame graph per batch
-// and batch width N (prefill plus one per acoustic step), built lazily
+// width N, built lazily on the first frame at a given N, then replayed
-// on the first frame at a given N, then replayed directly on the
+// directly on the backend with an N * 4 byte code id upload per call.
-// backend with an N * 4 byte code id upload per call.
 
 #include "code-predictor-graph.h"
 #include "code-predictor-weights.h"
@@ -44,7 +43,7 @@
 #include "ggml.h"
 #include "kv-cache.h"
 #include "qt-error.h"
-#include "sampling.h"
+#include "sampling-graph.h"
 
 #include <cmath>
 #include <cstdint>
@@ -78,7 +77,13 @@ static struct ggml_tensor * code_predictor_attn_f32(struct ggml_context * ctx,
 
 // Node budget for one predictor graph, same accounting as the talker.
 static int code_predictor_graph_max_nodes(int n_layers) {
-    return 48 * n_layers + 64;
+    return 48 * n_layers + 96;
+}
+
+// Node budget for the unrolled frame graph: the prefill and every
+// acoustic step chained in one cgraph.
+static int code_predictor_frame_graph_max_nodes(int n_layers, int n_passes) {
+    return n_passes * code_predictor_graph_max_nodes(n_layers);
 }
 
 // One batched Qwen3 decoder block, KV cached over sets [0, N). x holds
@@ -199,23 +204,81 @@ static struct ggml_tensor * code_predictor_layer_forward(struct ggml_context *
     return x;
 }
 
-// Build one static batched predictor graph over sets [0, N). A non
+// Baked inputs of one predictor pass: positions, kv rows and the
-// NULL hidden_bridge selects the T=2 prefill flavor reading
+// causal mask upload once after allocation, then every replay reuses
-// [talker_hidden, embed(c0)] per slot through lm_head[0]; otherwise
+// them.
-// the graph is the single token step for g_head, appending at the
+struct CodePredPassBake {
-// fixed cache row g_head + 1. The logits node holds the last position
+    struct ggml_tensor * pos;
-// of every slot as [Vg, N]. use_flash_attn / clamp_fp16 apply to every
+    struct ggml_tensor * rows;
-// layer.
+    struct ggml_tensor * mask;
-static bool code_predictor_graph_build(const CodePredictorWeights * cw,
+    int                  T;
-                                       KVCache *                    kv,
+    int                  n_past;
-                                       ggml_backend_t               backend,
+};
-                                       struct ggml_tensor *         embd_table,
+
-                                       struct ggml_tensor *         hidden_bridge,
+// Upload the baked inputs of every recorded pass. n_kv_pad is the
-                                       int                          g_head,
+// constant mask width shared by all flavors.
-                                       int                          N,
+static void code_predictor_bake_upload(const std::vector<CodePredPassBake> & bake, int N, int n_kv_pad) {
-                                       bool                         use_flash_attn,
+    for (const CodePredPassBake & b : bake) {
-                                       bool                         clamp_fp16,
+        std::vector<int32_t> pos((size_t) b.T * (size_t) N);
-                                       CodePredGraph *              cp) {
+        for (int n = 0; n < N; n++) {
+            for (int t = 0; t < b.T; t++) {
+                pos[(size_t) n * (size_t) b.T + (size_t) t] = b.n_past + t;
+            }
+        }
+        ggml_backend_tensor_set(b.pos, pos.data(), 0, pos.size() * sizeof(int32_t));
+
+        std::vector<int64_t> rows((size_t) b.T * (size_t) N);
+        for (int n = 0; n < N; n++) {
+            for (int t = 0; t < b.T; t++) {
+                rows[(size_t) n * (size_t) b.T + (size_t) t] = (int64_t) (b.n_past + t);
+            }
+        }
+        ggml_backend_tensor_set(b.rows, rows.data(), 0, rows.size() * sizeof(int64_t));
+
+        std::vector<ggml_fp16_t> mask((size_t) n_kv_pad * (size_t) b.T * (size_t) N);
+        const ggml_fp16_t        zero    = ggml_fp32_to_fp16(0.0f);
+        const ggml_fp16_t        neg_inf = ggml_fp32_to_fp16(-INFINITY);
+        for (size_t i = 0; i < mask.size(); i++) {
+            mask[i] = neg_inf;
+        }
+        for (int n = 0; n < N; n++) {
+            for (int q = 0; q < b.T; q++) {
+                const int q_pos = b.n_past + q;
+                for (int k = 0; k <= q_pos; k++) {
+                    mask[((size_t) n * (size_t) b.T + (size_t) q) * (size_t) n_kv_pad + (size_t) k] = zero;
+                }
+            }
+        }
+        ggml_backend_tensor_set(b.mask, mask.data(), 0, mask.size() * sizeof(ggml_fp16_t));
+    }
+}
+
+// Append one predictor pass to an existing graph. A non NULL
+// hidden_bridge selects the T=2 prefill flavor reading [talker_hidden,
+// embed(c0)] per slot through lm_head[0]; otherwise the pass is the
+// single token step for g_head, appending at the fixed cache row
+// g_head + 1. The pass ends in its sampling tail: it gathers its input
+// ids from row g_head of the persistent sp->codes accumulator (row 0
+// is the host written c0) and writes the ids it samples from
+// lm_head[g_head] into row g_head + 1, so passes replay with no logits
+// readback. Passes chained in one graph execute in node insertion
+// order on the direct backend compute path, so each pass reads the
+// codes row and the kv rows its predecessors wrote. use_flash_attn /
+// clamp_fp16 apply to every layer. logits_out receives the pass
+// logits, bake records the inputs to upload after allocation.
+static void code_predictor_pass_append(struct ggml_context *           gctx,
+                                       struct ggml_cgraph *            gf,
+                                       const CodePredictorWeights *    cw,
+                                       KVCache *                       kv,
+                                       struct ggml_tensor *            embd_table,
+                                       struct ggml_tensor *            hidden_bridge,
+                                       SamplerInputs *                 sp,
+                                       int                             g_head,
+                                       int                             N,
+                                       bool                            use_flash_attn,
+                                       bool                            clamp_fp16,
+                                       struct ggml_tensor **           logits_out,
+                                       std::vector<CodePredPassBake> & bake) {
     const int T        = hidden_bridge ? 2 : 1;
     const int n_past   = hidden_bridge ? 0 : g_head + 1;
     const int n_layers = cw->num_hidden_layers;
@@ -224,37 +287,23 @@ static bool code_predictor_graph_build(const CodePredictorWeights * cw,
     // constant width keeps every flavor at the same mask shape.
     const int n_kv_pad = kv->max_seq_len;
 
-    const int    max_nodes = code_predictor_graph_max_nodes(n_layers);
-    const size_t bytes =
-        ggml_tensor_overhead() * (size_t) max_nodes + ggml_graph_overhead_custom((size_t) max_nodes, false);
-    struct ggml_init_params gp = { bytes, NULL, true };
-    cp->ctx                    = ggml_init(gp);
-    if (!cp->ctx) {
-        fprintf(stderr, "[CodePredictor] FATAL: graph ctx allocation failed\n");
-        return false;
-    }
-    struct ggml_context * gctx = cp->ctx;
-
     // Inputs: one code id per slot gathered in graph from embd_table,
-    // positions, kv rows and the attention mask. The prefill path
+    // positions, kv rows and the attention mask. The ids come from row
-    // (T == 2) concats each slot's resident talker hidden ahead of
+    // g_head of the persistent codes accumulator, written either by
-    // embed(c0), both on device: the per slot sequence is
+    // the host (row 0, c0) or by the previous graph's sampling tail,
-    // [talker_hidden, embed(c0)] with zero row upload. Steps (T == 1)
+    // so replays upload nothing per step. The prefill path (T == 2)
-    // are pure gathers: the only per step upload is N * 4 bytes of
+    // concats each slot's resident talker hidden ahead of embed(c0),
-    // code ids.
+    // both on device: the per slot sequence is [talker_hidden,
-    struct ggml_tensor * ids_in  = ggml_new_tensor_1d(gctx, GGML_TYPE_I32, N);
+    // embed(c0)] with zero row upload.
     struct ggml_tensor * pos_in  = ggml_new_tensor_1d(gctx, GGML_TYPE_I32, T * N);
     struct ggml_tensor * mask_in = ggml_new_tensor_4d(gctx, GGML_TYPE_F16, n_kv_pad, T, 1, N);
     struct ggml_tensor * rows_in = ggml_new_tensor_3d(gctx, GGML_TYPE_I64, T, 1, N);
-    ggml_set_name(ids_in, "sub_code_ids");
     ggml_set_name(pos_in, "positions");
     ggml_set_name(mask_in, "causal_mask");
     ggml_set_name(rows_in, "kv_rows");
-    // ids uploads before every replay; pos, rows, and mask bake once,
+    // pos, rows, and mask bake once, so they also carry the output
-    // so they also carry the output flag: the allocator never frees an
+    // flag: the allocator never frees an output, which keeps their
-    // output, which keeps their slots out of the intermediate reuse
+    // slots out of the intermediate reuse pool across replays.
-    // pool across replays.
-    ggml_set_input(ids_in);
     ggml_set_input(pos_in);
     ggml_set_output(pos_in);
     ggml_set_input(mask_in);
@@ -262,6 +311,9 @@ static bool code_predictor_graph_build(const CodePredictorWeights * cw,
     ggml_set_input(rows_in);
     ggml_set_output(rows_in);
 
+    struct ggml_tensor * ids_in = ggml_view_1d(gctx, sp->codes, N, (size_t) g_head * sp->codes->nb[1]);
+    ggml_set_name(ids_in, "sub_code_ids");
+
     struct ggml_tensor * x_in = ggml_get_rows(gctx, embd_table, ids_in);  // [hidden, N]
     if (T == 2) {
         struct ggml_tensor * bridge_cols =
@@ -273,8 +325,6 @@ static bool code_predictor_graph_build(const CodePredictorWeights * cw,
     }
     ggml_set_name(x_in, "sub_input");
 
-    struct ggml_cgraph * gf = ggml_new_graph_custom(gctx, max_nodes, false);
-
     // small_to_mtp projection: Linear(talker_hidden -> hidden) with bias.
     // When absent (Identity case) the input is already at predictor hidden.
     struct ggml_tensor * h = x_in;
@@ -306,144 +356,107 @@ static bool code_predictor_graph_build(const CodePredictorWeights * cw,
     ggml_set_output(logits);
     ggml_build_forward_expand(gf, logits);
 
+    ggml_build_forward_expand(gf, sampler_tail_build(gctx, logits, sp, g_head));
+
+    bake.push_back({ pos_in, rows_in, mask_in, T, n_past });
+    *logits_out = logits;
+}
+
+// Build the unrolled frame graph over sets [0, N): the T=2 prefill and
+// the n_acoustic - 1 steps chained in one static cgraph, so a frame
+// replays in a single backend compute. talker_embd_table feeds the
+// prefill c0 embedding, each step embeds through its own codebook
+// table. Pass order in the node list carries the data dependencies:
+// every step reads the codes row and the kv rows its predecessors
+// wrote.
+static bool code_predictor_frame_graph_build(const CodePredictorWeights * cw,
+                                             KVCache *                    kv,
+                                             ggml_backend_t               backend,
+                                             struct ggml_tensor *         talker_embd_table,
+                                             struct ggml_tensor *         hidden_bridge,
+                                             SamplerInputs *              sp,
+                                             int                          N,
+                                             bool                         use_flash_attn,
+                                             bool                         clamp_fp16,
+                                             CodePredGraph *              cp) {
+    const int n_layers   = cw->num_hidden_layers;
+    const int n_acoustic = cw->num_acoustic_codebooks;
+    const int max_nodes  = code_predictor_frame_graph_max_nodes(n_layers, n_acoustic);
+
+    const size_t bytes =
+        ggml_tensor_overhead() * (size_t) max_nodes + ggml_graph_overhead_custom((size_t) max_nodes, false);
+    struct ggml_init_params gp = { bytes, NULL, true };
+    cp->ctx                    = ggml_init(gp);
+    if (!cp->ctx) {
+        fprintf(stderr, "[CodePredictor] FATAL: frame graph ctx allocation failed\n");
+        return false;
+    }
+    struct ggml_cgraph * gf = ggml_new_graph_custom(cp->ctx, max_nodes, false);
+
+    std::vector<CodePredPassBake> bake;
+    bake.reserve((size_t) n_acoustic);
+    struct ggml_tensor * logits = NULL;
+
+    code_predictor_pass_append(cp->ctx, gf, cw, kv, talker_embd_table, hidden_bridge, sp, 0, N, use_flash_attn,
+                               clamp_fp16, &logits, bake);
+    for (int g = 1; g < n_acoustic; g++) {
+        code_predictor_pass_append(cp->ctx, gf, cw, kv, cw->codec_embedding[(size_t) (g - 1)], NULL, sp, g, N,
+                                   use_flash_attn, clamp_fp16, &logits, bake);
+    }
+
     cp->galloc = ggml_gallocr_new(ggml_backend_get_default_buffer_type(backend));
     if (!cp->galloc || !ggml_gallocr_alloc_graph(cp->galloc, gf)) {
-        fprintf(stderr, "[CodePredictor] FATAL: graph allocation failed\n");
+        fprintf(stderr, "[CodePredictor] FATAL: frame graph allocation failed\n");
         code_predictor_graph_free(cp);
         return false;
     }
-
+    code_predictor_bake_upload(bake, N, kv->max_seq_len);
-    {
-        std::vector<int32_t> pos((size_t) T * (size_t) N);
-        for (int n = 0; n < N; n++) {
-            for (int t = 0; t < T; t++) {
-                pos[(size_t) n * (size_t) T + (size_t) t] = n_past + t;
-            }
-        }
-        ggml_backend_tensor_set(pos_in, pos.data(), 0, pos.size() * sizeof(int32_t));
-
-        std::vector<int64_t> rows((size_t) T * (size_t) N);
-        for (int n = 0; n < N; n++) {
-            for (int t = 0; t < T; t++) {
-                rows[(size_t) n * (size_t) T + (size_t) t] = (int64_t) (n_past + t);
-            }
-        }
-        ggml_backend_tensor_set(rows_in, rows.data(), 0, rows.size() * sizeof(int64_t));
-    }
-
-    {
-        std::vector<ggml_fp16_t> mask((size_t) n_kv_pad * (size_t) T * (size_t) N);
-        const ggml_fp16_t        zero    = ggml_fp32_to_fp16(0.0f);
-        const ggml_fp16_t        neg_inf = ggml_fp32_to_fp16(-INFINITY);
-        for (size_t i = 0; i < mask.size(); i++) {
-            mask[i] = neg_inf;
-        }
-        for (int n = 0; n < N; n++) {
-            for (int q = 0; q < T; q++) {
-                const int q_pos = n_past + q;
-                for (int k = 0; k <= q_pos; k++) {
-                    mask[((size_t) n * (size_t) T + (size_t) q) * (size_t) n_kv_pad + (size_t) k] = zero;
-                }
-            }
-        }
-        ggml_backend_tensor_set(mask_in, mask.data(), 0, mask.size() * sizeof(ggml_fp16_t));
-    }
 
     cp->gf     = gf;
-    cp->ids_in = ids_in;
     cp->logits = logits;
     cp->N      = N;
     return true;
 }
 
-// Replay one static predictor graph: upload the N code ids, run the
+// Run the predictor for one audio frame through the unrolled frame
-// graph directly on the backend, read the [Vg, N] logits back.
+// graph, all N slots in lockstep: the host writes c0 into row 0 of the
-static bool code_predictor_replay(CodePredGraph *      cp,
+// codes accumulator, uploads the per frame sampler state, replays one
-                                  ggml_backend_t       backend,
+// graph, then reads the [N, 16] accumulator back in one transfer. Per
-                                  const int32_t *      code_ids,
+// slot temperature controls greedy (temperature <= 0) vs stochastic;
-                                  int                  N,
+// seed and subseq_base index each slot's Philox stream, subseq_base[i]
-                                  std::vector<float> * logits_out) {
+// being the subsequence of slot i's c0 sample (the 15 acoustic samples
-    ggml_backend_tensor_set(cp->ids_in, code_ids, 0, (size_t) N * sizeof(int32_t));
+// consume subseq_base[i] + 1 .. subseq_base[i] + 15). Fills out->codes
-    if (ggml_backend_graph_compute(backend, cp->gf) != GGML_STATUS_SUCCESS) {
+// as [N * 16] slot major. dump_dir may be NULL and applies to slot 0.
-        fprintf(stderr, "[CodePredictor] FATAL: graph compute failed\n");
+static bool code_predictor_frame_step(const CodePredictorWeights * cw,
-        return false;
+                                      ggml_backend_t               backend,
-    }
+                                      CodePredGraph *              frame_graph,
-    logits_out->resize((size_t) cp->logits->ne[0] * (size_t) N);
+                                      SamplerInputs *              sp,
-    ggml_backend_tensor_get(cp->logits, logits_out->data(), 0, logits_out->size() * sizeof(float));
+                                      const int32_t *              c0,
-    return true;
+                                      int                          N,
-}
+                                      const float *                temperature,
-
+                                      const int64_t *              seed,
-// Run the predictor for one audio frame over the static graphs, all N
+                                      const int64_t *              subseq_base,
-// slots in lockstep. The prefill replay consumes the persistent hidden
+                                      const char *                 dump_dir,
-// bridge columns already written by the talker graph and the sampled
+                                      CodePredictorOutput *        out) {
-// c0 of every slot; the 14 step replays each feed the codes sampled
-// just before. Per slot sampling parameters control greedy
-// (temperature <= 0) vs stochastic; seed and subseq_base index each
-// slot's own Philox stream, so per slot outputs match a single
-// sequence run bit for bit. subseq_base[i] is the subsequence of slot
-// i's c0 sample for this step; its 15 acoustic samples consume
-// subseq_base[i] + 1 .. subseq_base[i] + 15. Fills out->codes as
-// [N * 16] slot major. dump_dir may be NULL and applies to slot 0.
-static bool code_predictor_step(const CodePredictorWeights * cw,
-                                ggml_backend_t               backend,
-                                CodePredGraph *              prefill_graph,
-                                CodePredGraph *              step_graphs,
-                                const int32_t *              c0,
-                                int                          N,
-                                const float *                temperature,
-                                const int *                  top_k,
-                                const float *                top_p,
-                                const int64_t *              seed,
-                                const int64_t *              subseq_base,
-                                const char *                 dump_dir,
-                                CodePredictorOutput *        out) {
     const int n_acoustic = cw->num_acoustic_codebooks;
     const int n_codes    = n_acoustic + 1;
 
-    out->codes.assign((size_t) N * (size_t) n_codes, 0);
+    ggml_backend_tensor_set(sp->codes, c0, 0, (size_t) N * sizeof(int32_t));
-    for (int i = 0; i < N; i++) {
+    sampler_inputs_upload(sp, temperature, seed, subseq_base, N);
-        out->codes[(size_t) i * (size_t) n_codes] = c0[i];
-    }
 
-    std::vector<float>   logits;
+    if (ggml_backend_graph_compute(backend, frame_graph->gf) != GGML_STATUS_SUCCESS) {
-    std::vector<int32_t> ids((size_t) N);
+        fprintf(stderr, "[CodePredictor] FATAL: frame graph compute failed\n");
-
-    if (!code_predictor_replay(prefill_graph, backend, c0, N, &logits)) {
         return false;
     }
-    const int V0 = (int) (logits.size() / (size_t) N);
-    for (int i = 0; i < N; i++) {
-        float u_g = 0.0f;
-        int cg = sample_top_k_p(logits.data() + (size_t) i * (size_t) V0, V0, temperature[i], top_k[i], top_p[i], 1.0f,
-                                nullptr, 0, seed[i], subseq_base[i] + 1, &u_g);
-        if (cg < 0) {
-            fprintf(stderr, "[CodePredictor] FATAL: sample returned no candidate at g=0 (slot %d)\n", i);
-            return false;
-        }
-        out->codes[(size_t) i * (size_t) n_codes + 1] = cg;
-    }
 
-    // Decode loop: 14 batched single-token replays. At step g
+    // One readback per frame: the accumulator is row major over code
-    // (g=1..14) every slot feeds the id of the code it just sampled,
+    // groups, out->codes is slot major.
-    // gathered in graph from the group's private embedding table, and
+    std::vector<int32_t> acc((size_t) N * (size_t) n_codes);
-    // reads lm_head[g].
+    ggml_backend_tensor_get(sp->codes, acc.data(), 0, acc.size() * sizeof(int32_t));
-    for (int g = 1; g < n_acoustic; g++) {
+    out->codes.resize((size_t) N * (size_t) n_codes);
-        for (int i = 0; i < N; i++) {
+    for (int i = 0; i < N; i++) {
-            ids[(size_t) i] = out->codes[(size_t) i * (size_t) n_codes + (size_t) g];
+        for (int g = 0; g < n_codes; g++) {
-        }
+            out->codes[(size_t) i * (size_t) n_codes + (size_t) g] = acc[(size_t) g * (size_t) N + (size_t) i];
-        if (!code_predictor_replay(&step_graphs[(size_t) (g - 1)], backend, ids.data(), N, &logits)) {
-            return false;
-        }
-        const int Vg = (int) (logits.size() / (size_t) N);
-        for (int i = 0; i < N; i++) {
-            float u_g = 0.0f;
-            int   cg  = sample_top_k_p(logits.data() + (size_t) i * (size_t) Vg, Vg, temperature[i], top_k[i], top_p[i],
-                                       1.0f, nullptr, 0, seed[i], subseq_base[i] + 1 + g, &u_g);
-            if (cg < 0) {
-                fprintf(stderr, "[CodePredictor] FATAL: sample returned no candidate at g=%d (slot %d)\n", g, i);
-                return false;
-            }
-            out->codes[(size_t) i * (size_t) n_codes + (size_t) (g + 1)] = cg;
         }
     }
 

src/code-predictor-graph.h

@@ -13,7 +13,6 @@ struct CodePredGraph {
     struct ggml_context * ctx    = nullptr;
     struct ggml_cgraph *  gf     = nullptr;
     ggml_gallocr_t        galloc = nullptr;
-    struct ggml_tensor *  ids_in = nullptr;  // [N] i32, one code id per slot
     struct ggml_tensor *  logits = nullptr;  // [Vg, N] f32
     int                   N      = 0;        // batch width this build covers
 };
@@ -28,7 +27,6 @@ static void code_predictor_graph_free(CodePredGraph * cp) {
         cp->ctx = nullptr;
     }
     cp->gf     = nullptr;
-    cp->ids_in = nullptr;
     cp->logits = nullptr;
     cp->N      = 0;
 }

src/pipeline-codec.cpp

@@ -478,6 +478,66 @@ void pipeline_codec_snap_free(CodecStateSnap * s) {
     *s = {};
 }
 
+// Append the streaming decode chain to an existing graph: quantizer,
+// pre conv, sliding window transformer, upsample stage and DAC, all
+// threaded through the [set0, set0 + M) slices of the persistent
+// stream state. codes_in [T, K, M] i32, pos_in [T * M] i32, rows_in
+// [T, 1, M] i64 and mask_in [ring, T, 1, M] f32 come from the caller.
+// Returns the clamped audio tensor [T * hop, 1, M], expanded into gf.
+static struct ggml_tensor * codec_stream_chain_append(PipelineCodec *       pc,
+                                                      struct ggml_context * gctx,
+                                                      struct ggml_cgraph *  gf,
+                                                      struct ggml_tensor *  codes_in,
+                                                      struct ggml_tensor *  pos_in,
+                                                      struct ggml_tensor *  rows_in,
+                                                      struct ggml_tensor *  mask_in,
+                                                      int                   set0,
+                                                      int                   M) {
+    // Lane state slices [t, c, M] at set0: contiguous suffix or prefix
+    // spans of the [t, c, S] owners, so views reshape freely.
+    auto slice = [&](struct ggml_tensor * owner) {
+        return ggml_view_3d(gctx, owner, owner->ne[0], owner->ne[1], M, owner->nb[1], owner->nb[2],
+                            (size_t) set0 * owner->nb[2]);
+    };
+    QwenUpsampleStreamState up_sl;
+    QwenDACStreamState      dac_sl;
+    for (int i = 0; i < pc->upsample.num_blocks; i++) {
+        up_sl.dw[i] = slice(pc->stream_up.dw[i]);
+    }
+    dac_sl.pre = slice(pc->stream_dac.pre);
+    for (int i = 0; i < DAC_NUM_BLOCKS; i++) {
+        dac_sl.carry[i] = slice(pc->stream_dac.carry[i]);
+        for (int r = 0; r < DAC_RES_UNITS; r++) {
+            dac_sl.ru[i][r] = slice(pc->stream_dac.ru[i][r]);
+        }
+    }
+    dac_sl.post = slice(pc->stream_dac.post);
+
+    // KVCache facade over the lane set span: reuse the multi-set ring
+    // tensors with per graph 4D views built inside tok_trans; staging
+    // binds the last set through a single set span.
+    KVCache * kv = &pc->stream_kv;
+
+    // Same module chain as pipeline_codec_decode with the stateful
+    // batched variants threaded through the persistent stream slices.
+    // The quantizer uses the alignment safe variant: no scheduler input
+    // duplication happens on the direct backend compute path. Codes
+    // flatten to [T * M, K] column gathers via the lane major layout.
+    struct ggml_tensor * h = quant_decode_stream_batch(gctx, &pc->qdec, codes_in);  // [512, T, M] C-first
+    h                      = ggml_cont(gctx, ggml_transpose(gctx, h));              // [T, 512, M] T-first
+    h = qwen_causal_conv1d_stream(gctx, gf, pc->pre_conv_w, pc->pre_conv_b, h, 3, 1, slice(pc->stream_pre_conv));
+    h = ggml_cont(gctx, ggml_transpose(gctx, h));                                   // [1024, T, M] C-first
+    h = tok_trans_forward_stream_span(gctx, gf, &pc->transformer, h, pos_in, mask_in, rows_in, kv, set0, M);
+    h = ggml_cont(gctx, ggml_transpose(gctx, h));                                   // [T, 1024, M] T-first
+    h = upsample_stage_forward_stream(gctx, gf, &pc->upsample, h, &up_sl);
+    h = dac_decoder_forward_stream(gctx, gf, &pc->dac, h, &dac_sl);
+    h = ggml_clamp(gctx, h, -1.0f, 1.0f);
+    ggml_set_name(h, "audio_out");
+    ggml_set_output(h);
+    ggml_build_forward_expand(gf, h);
+    return h;
+}
+
 // Build the static stream graph of chunk width T = 1 << cls: the
 // same module chain as the T=1 frame graph, every stream state and
 // KV ring tensor shared across classes, inputs and intermediates
@@ -527,48 +587,7 @@ static bool codec_stream_graph_build(PipelineCodec * pc, int cls, int M, bool st
     ggml_set_input(rows_in);
     ggml_set_input(mask_in);
 
-    // Lane state slices [t, c, M] at set0: contiguous suffix or prefix
+    struct ggml_tensor * h = codec_stream_chain_append(pc, gctx, gf, codes_in, pos_in, rows_in, mask_in, set0, M);
-    // spans of the [t, c, S] owners, so views reshape freely.
-    auto slice = [&](struct ggml_tensor * owner) {
-        return ggml_view_3d(gctx, owner, owner->ne[0], owner->ne[1], M, owner->nb[1], owner->nb[2],
-                            (size_t) set0 * owner->nb[2]);
-    };
-    QwenUpsampleStreamState up_sl;
-    QwenDACStreamState      dac_sl;
-    for (int i = 0; i < pc->upsample.num_blocks; i++) {
-        up_sl.dw[i] = slice(pc->stream_up.dw[i]);
-    }
-    dac_sl.pre = slice(pc->stream_dac.pre);
-    for (int i = 0; i < DAC_NUM_BLOCKS; i++) {
-        dac_sl.carry[i] = slice(pc->stream_dac.carry[i]);
-        for (int r = 0; r < DAC_RES_UNITS; r++) {
-            dac_sl.ru[i][r] = slice(pc->stream_dac.ru[i][r]);
-        }
-    }
-    dac_sl.post = slice(pc->stream_dac.post);
-
-    // KVCache facade over the lane set span: reuse the multi-set ring
-    // tensors with per graph 4D views built inside tok_trans; staging
-    // binds the last set through a single set span.
-    KVCache * kv = &pc->stream_kv;
-
-    // Same module chain as pipeline_codec_decode with the stateful
-    // batched variants threaded through the persistent stream slices.
-    // The quantizer uses the alignment safe variant: no scheduler input
-    // duplication happens on the direct backend compute path. Codes
-    // flatten to [T * M, K] column gathers via the lane major layout.
-    struct ggml_tensor * h = quant_decode_stream_batch(gctx, &pc->qdec, codes_in);  // [512, T, M] C-first
-    h                      = ggml_cont(gctx, ggml_transpose(gctx, h));              // [T, 512, M] T-first
-    h = qwen_causal_conv1d_stream(gctx, gf, pc->pre_conv_w, pc->pre_conv_b, h, 3, 1, slice(pc->stream_pre_conv));
-    h = ggml_cont(gctx, ggml_transpose(gctx, h));                                   // [1024, T, M] C-first
-    h = tok_trans_forward_stream_span(gctx, gf, &pc->transformer, h, pos_in, mask_in, rows_in, kv, set0, M);
-    h = ggml_cont(gctx, ggml_transpose(gctx, h));                                   // [T, 1024, M] T-first
-    h = upsample_stage_forward_stream(gctx, gf, &pc->upsample, h, &up_sl);
-    h = dac_decoder_forward_stream(gctx, gf, &pc->dac, h, &dac_sl);
-    h = ggml_clamp(gctx, h, -1.0f, 1.0f);
-    ggml_set_name(h, "audio_out");
-    ggml_set_output(h);
-    ggml_build_forward_expand(gf, h);
 
     sg->galloc = ggml_gallocr_new(ggml_backend_get_default_buffer_type(pc->backend));
     if (!sg->galloc || !ggml_gallocr_alloc_graph(sg->galloc, gf)) {
@@ -668,6 +687,97 @@ bool pipeline_codec_decode_stream_batch(PipelineCodec * pc, const int32_t * code
     return codec_stream_run(pc, sg, codes, T, M, 0, audio_out);
 }
 
+// Append the fused streaming tail: ring inputs created here, the chain
+// appended over the lane sets [set0 = 0, M). The caller owns gctx/gf
+// and allocates the whole graph afterwards.
+bool pipeline_codec_stream_tail_append(PipelineCodec *       pc,
+                                       struct ggml_context * gctx,
+                                       struct ggml_cgraph *  gf,
+                                       struct ggml_tensor *  codes_in,
+                                       int                   T,
+                                       int                   M,
+                                       CodecStreamTail *     tail) {
+    const int ring = CODEC_STREAM_RING;
+    if (!pc->stream_ready) {
+        qt_log(QT_LOG_ERROR, "[Pipeline] stream tail append before stream state init");
+        return false;
+    }
+    if (pc->transformer.sliding_window + T > ring) {
+        qt_log(QT_LOG_ERROR, "[Pipeline] sliding window %d plus chunk %d exceeds KV ring %d",
+               pc->transformer.sliding_window, T, ring);
+        return false;
+    }
+    if (M < 1 || M > pc->stream_sets - 1) {
+        qt_log(QT_LOG_ERROR, "[Pipeline] invalid stream tail lane count %d", M);
+        return false;
+    }
+
+    tail->T = T;
+    tail->M = M;
+
+    tail->pos  = ggml_new_tensor_1d(gctx, GGML_TYPE_I32, T * M);
+    tail->rows = ggml_new_tensor_3d(gctx, GGML_TYPE_I64, T, 1, M);
+    tail->mask = ggml_new_tensor_4d(gctx, GGML_TYPE_F32, ring, T, 1, M);
+    ggml_set_name(tail->pos, "codec_positions");
+    ggml_set_name(tail->rows, "codec_kv_rows");
+    ggml_set_name(tail->mask, "codec_ring_mask");
+    ggml_set_input(tail->pos);
+    ggml_set_input(tail->rows);
+    ggml_set_input(tail->mask);
+
+    tail->out = codec_stream_chain_append(pc, gctx, gf, codes_in, tail->pos, tail->rows, tail->mask, 0, M);
+    return tail->out != NULL;
+}
+
+// Upload the per frame ring inputs of the fused tail from the lane
+// positions at [set0, set0 + M).
+bool pipeline_codec_stream_tail_upload(PipelineCodec * pc, CodecStreamTail * tail, int set0) {
+    const int T    = tail->T;
+    const int M    = tail->M;
+    const int ring = CODEC_STREAM_RING;
+    if (!tail->out) {
+        return false;
+    }
+
+    tail->pos_buf.resize((size_t) T * (size_t) M);
+    tail->rows_buf.resize((size_t) T * (size_t) M);
+    static thread_local std::vector<int> pos0;
+    pos0.assign((size_t) M, 0);
+    for (int m = 0; m < M; m++) {
+        int p   = pc->stream_pos[(size_t) (set0 + m)];
+        pos0[m] = p;
+        for (int t = 0; t < T; t++) {
+            tail->pos_buf[(size_t) m * (size_t) T + (size_t) t]  = p + t;
+            tail->rows_buf[(size_t) m * (size_t) T + (size_t) t] = (int64_t) ((p + t) % ring);
+        }
+    }
+    ggml_backend_tensor_set(tail->pos, tail->pos_buf.data(), 0, tail->pos_buf.size() * sizeof(int32_t));
+    ggml_backend_tensor_set(tail->rows, tail->rows_buf.data(), 0, tail->rows_buf.size() * sizeof(int64_t));
+
+    tok_trans_build_stream_mask(pos0.data(), T, M, ring, pc->transformer.sliding_window, tail->mask_buf);
+    ggml_backend_tensor_set(tail->mask, tail->mask_buf.data(), 0, tail->mask_buf.size() * sizeof(float));
+    return true;
+}
+
+// Read every non NULL lane's audio from the fused tail output and
+// advance the lane positions. The caller has computed the graph.
+bool pipeline_codec_stream_tail_read(PipelineCodec * pc, CodecStreamTail * tail, int set0, float ** audio_out) {
+    const int T = tail->T;
+    const int M = tail->M;
+    if (!tail->out) {
+        return false;
+    }
+    const size_t lane_samples = (size_t) T * (size_t) TOKENIZER_HOP_LENGTH;
+    for (int m = 0; m < M; m++) {
+        if (audio_out && audio_out[m]) {
+            ggml_backend_tensor_get(tail->out, audio_out[m], (size_t) m * lane_samples * sizeof(float),
+                                    lane_samples * sizeof(float));
+        }
+        pc->stream_pos[(size_t) (set0 + m)] += T;
+    }
+    return true;
+}
+
 bool pipeline_codec_decode_stream(PipelineCodec * pc, const int32_t * codes, int T, float * audio_out) {
     int cls = 0;
     while ((1 << cls) < T) {

src/pipeline-codec.h

@@ -194,6 +194,36 @@ bool pipeline_codec_stream_reset(PipelineCodec * pc, int set);
 // NULL discards that lane's audio. Every lane's position advances.
 bool pipeline_codec_decode_stream_batch(PipelineCodec * pc, const int32_t * codes, int T, int M, float ** audio_out);
 
+// Fused streaming tail: the decode chain appended to a caller owned
+// graph, codes read in graph from a caller provided [T, K, M] i32
+// tensor instead of a host upload. The caller allocates the graph,
+// uploads the per frame ring inputs with
+// pipeline_codec_stream_tail_upload before each compute, and reads the
+// audio and advances the lane positions with
+// pipeline_codec_stream_tail_read after it. Lanes bind the state sets
+// [set0, set0 + M).
+struct CodecStreamTail {
+    struct ggml_tensor * pos  = nullptr;  // [T * M] i32
+    struct ggml_tensor * rows = nullptr;  // [T, 1, M] i64
+    struct ggml_tensor * mask = nullptr;  // [ring, T, 1, M] f32
+    struct ggml_tensor * out  = nullptr;  // [T * 1920, 1, M] f32
+    int                  T    = 0;
+    int                  M    = 0;
+    std::vector<int32_t> pos_buf;
+    std::vector<int64_t> rows_buf;
+    std::vector<float>   mask_buf;
+};
+
+bool pipeline_codec_stream_tail_append(PipelineCodec *       pc,
+                                       struct ggml_context * gctx,
+                                       struct ggml_cgraph *  gf,
+                                       struct ggml_tensor *  codes_in,
+                                       int                   T,
+                                       int                   M,
+                                       CodecStreamTail *     tail);
+bool pipeline_codec_stream_tail_upload(PipelineCodec * pc, CodecStreamTail * tail, int set0);
+bool pipeline_codec_stream_tail_read(PipelineCodec * pc, CodecStreamTail * tail, int set0, float ** audio_out);
+
 // Decode one chunk of T frames through the STAGING set (the last
 // one). codes is [T, K]; audio_out receives T * TOKENIZER_HOP_LENGTH
 // samples, NULL discards the audio (ICL reference priming). The

src/pipeline-tts.cpp

@@ -105,32 +105,154 @@ static void parse_generation_defaults(const GGUFModel & gf, GenerationDefaults &
     g.max_new_tokens        = (int) gf_get_u32(gf, "generation.max_new_tokens");
 }
 
-// Ensure the static predictor graph set for batch width N exists:
+// Build the fused frame graph for the single slot mode: the unrolled
-// prefill (T=2 through lm_head[0]) plus one T=1 step per acoustic
+// predictor passes followed by the codec stream tail at T=1 over lane
-// codebook after the first. Built lazily on the first frame at a given
+// set 0, one cgraph, one compute per frame. The codec reads the codes
-// width, then replayed for the process lifetime.
+// straight from the sampler accumulator through a device view, so the
-static bool pipeline_tts_cp_graphs_ensure(PipelineTTS * pt, int N) {
+// codes never round trip the host between the predictor and the
+// decode. Requires the codec stream state (reset at slot admit) to be
+// allocated, which holds by the time the first frame builds this.
+static bool pipeline_tts_fused_graph_build(PipelineTTS * pt, CodePredGraphSet & s) {
+    const CodePredictorWeights * cw = &pt->code_predictor;
+    KVCache *                    kv = &pt->code_predictor_kv;
+
+    const int n_layers   = cw->num_hidden_layers;
+    const int n_acoustic = cw->num_acoustic_codebooks;
+    const int n_codes    = n_acoustic + 1;
+    const int max_nodes  = code_predictor_frame_graph_max_nodes(n_layers, n_acoustic) + 4096;
+
+    if (n_codes != TOKENIZER_NUM_CODEBOOKS) {
+        qt_log(QT_LOG_ERROR, "[Pipeline] fused tail codebook mismatch: %d vs %d", n_codes, TOKENIZER_NUM_CODEBOOKS);
+        return false;
+    }
+
+    CodePredGraph * cp = &s.fused;
+
+    const size_t bytes =
+        ggml_tensor_overhead() * (size_t) max_nodes + ggml_graph_overhead_custom((size_t) max_nodes, false);
+    struct ggml_init_params gp = { bytes, NULL, true };
+    cp->ctx                    = ggml_init(gp);
+    if (!cp->ctx) {
+        qt_log(QT_LOG_ERROR, "[Pipeline] fused graph ctx allocation failed");
+        return false;
+    }
+    struct ggml_cgraph * gf = ggml_new_graph_custom(cp->ctx, max_nodes, false);
+
+    std::vector<CodePredPassBake> bake;
+    bake.reserve((size_t) n_acoustic);
+    struct ggml_tensor * logits = NULL;
+
+    code_predictor_pass_append(cp->ctx, gf, cw, kv, pt->talker.codec_embedding, pt->hidden_bridge, &s.sampler, 0, 1,
+                               pt->use_flash_attn, pt->clamp_fp16, &logits, bake);
+    for (int g = 1; g < n_acoustic; g++) {
+        code_predictor_pass_append(cp->ctx, gf, cw, kv, cw->codec_embedding[(size_t) (g - 1)], NULL, &s.sampler, g, 1,
+                                   pt->use_flash_attn, pt->clamp_fp16, &logits, bake);
+    }
+
+    // Codes bridge: the [1, 16] accumulator is 16 contiguous i32, c0
+    // first, exactly the [T=1, K=16, M=1] layout the quantizer reads.
+    struct ggml_tensor * codes_in = ggml_view_3d(cp->ctx, s.sampler.codes, 1, n_codes, 1, s.sampler.codes->nb[1],
+                                                 (size_t) n_codes * s.sampler.codes->nb[1], 0);
+    ggml_set_name(codes_in, "fused_codes_bridge");
+
+    if (!pipeline_codec_stream_tail_append(&pt->codec, cp->ctx, gf, codes_in, 1, 1, &s.tail)) {
+        code_predictor_graph_free(cp);
+        s.tail = {};
+        return false;
+    }
+
+    cp->galloc = ggml_gallocr_new(ggml_backend_get_default_buffer_type(pt->backend));
+    if (!cp->galloc || !ggml_gallocr_alloc_graph(cp->galloc, gf)) {
+        qt_log(QT_LOG_ERROR, "[Pipeline] fused graph allocation failed");
+        code_predictor_graph_free(cp);
+        s.tail = {};
+        return false;
+    }
+    code_predictor_bake_upload(bake, 1, kv->max_seq_len);
+
+    cp->gf     = gf;
+    cp->logits = logits;
+    cp->N      = 1;
+    return true;
+}
+
+// Run one fused frame for the single slot: sampler upload, codec ring
+// upload, one compute, codes readback, then audio readback with the
+// lane position advance. Mirrors code_predictor_frame_step for N=1
+// plus the codec side of the graph.
+static bool pipeline_tts_fused_frame_step(PipelineTTS *         pt,
+                                          CodePredGraphSet &    gs,
+                                          const int32_t *       c0,
+                                          const float *         temperature,
+                                          const int64_t *       seed,
+                                          const int64_t *       subseq_base,
+                                          float *               audio,
+                                          CodePredictorOutput * out) {
+    SamplerInputs * sp      = &gs.sampler;
+    const int       n_codes = pt->num_code_groups;
+
+    ggml_backend_tensor_set(sp->codes, c0, 0, sizeof(int32_t));
+    sampler_inputs_upload(sp, temperature, seed, subseq_base, 1);
+    if (!pipeline_codec_stream_tail_upload(&pt->codec, &gs.tail, 0)) {
+        return false;
+    }
+
+    if (ggml_backend_graph_compute(pt->backend, gs.fused.gf) != GGML_STATUS_SUCCESS) {
+        qt_log(QT_LOG_ERROR, "[Pipeline] fused frame graph compute failed");
+        return false;
+    }
+
+    // N=1: the accumulator rows are single scalars, so the slot major
+    // output equals the row major readback.
+    out->codes.resize((size_t) n_codes);
+    ggml_backend_tensor_get(sp->codes, out->codes.data(), 0, (size_t) n_codes * sizeof(int32_t));
+
+    float * outs[1] = { audio };
+    return pipeline_codec_stream_tail_read(&pt->codec, &gs.tail, 0, outs);
+}
+
+// Ensure the static predictor graph set for batch width N exists: the
+// frame graph, or its fused flavor carrying the codec stream tail,
+// over one persistent sampler state. Built lazily on the first frame
+// at a given width, then replayed for the process lifetime.
+static bool pipeline_tts_cp_graphs_ensure(PipelineTTS * pt, int N, bool fused) {
     if ((int) pt->cp_graphs.size() < N) {
         pt->cp_graphs.resize((size_t) N);
     }
     CodePredGraphSet & s = pt->cp_graphs[(size_t) (N - 1)];
-    if (s.prefill.ctx) {
+    if (fused ? s.fused.ctx != NULL : s.frame.ctx != NULL) {
         return true;
     }
-    if (!code_predictor_graph_build(&pt->code_predictor, &pt->code_predictor_kv, pt->backend,
+
-                                    pt->talker.codec_embedding, pt->hidden_bridge, 0, N, pt->use_flash_attn,
+    // Sampler inputs and the codes accumulator, resident on the
-                                    pt->clamp_fp16, &s.prefill)) {
+    // backend for the process lifetime: every graph of the set views
-        return false;
+    // them, so they allocate before any graph builds.
-    }
+    const int n_steps = pt->num_code_groups - 1;
-    s.steps.resize((size_t) (pt->num_code_groups - 2));
+    if (!s.sampler_ctx) {
-    for (size_t g = 0; g < s.steps.size(); g++) {
+        struct ggml_init_params gp = { ggml_tensor_overhead() * 8, NULL, true };
-        if (!code_predictor_graph_build(&pt->code_predictor, &pt->code_predictor_kv, pt->backend,
+        s.sampler_ctx              = ggml_init(gp);
-                                        pt->code_predictor.codec_embedding[g], NULL, (int) g + 1, N, pt->use_flash_attn,
+        if (s.sampler_ctx) {
-                                        pt->clamp_fp16, &s.steps[g])) {
+            sampler_inputs_build(s.sampler_ctx, &s.sampler, N, n_steps, pt->gen_defaults.subtalker_top_k);
+            s.sampler_buf = ggml_backend_alloc_ctx_tensors(s.sampler_ctx, pt->backend);
+        }
+        if (!s.sampler_ctx || !s.sampler_buf) {
+            qt_log(QT_LOG_ERROR, "[Pipeline] sampler state allocation failed (N=%d)", N);
+            if (s.sampler_ctx) {
+                ggml_free(s.sampler_ctx);
+                s.sampler_ctx = NULL;
+            }
             return false;
         }
+        ggml_backend_buffer_clear(s.sampler_buf, 0);
     }
-    return true;
+
+    if (fused) {
+        return pipeline_tts_fused_graph_build(pt, s);
+    }
+
+    return code_predictor_frame_graph_build(&pt->code_predictor, &pt->code_predictor_kv, pt->backend,
+                                            pt->talker.codec_embedding, pt->hidden_bridge, &s.sampler, N,
+                                            pt->use_flash_attn, pt->clamp_fp16, &s.frame);
 }
 
 bool pipeline_tts_load(PipelineTTS * pt,
@@ -140,7 +262,8 @@ bool pipeline_tts_load(PipelineTTS * pt,
                        bool          use_fa,
                        bool          clamp_fp16,
                        int           max_batch,
-                       float         codec_chunk_sec) {
+                       float         codec_chunk_sec,
+                       bool          codec_fused) {
     pt->bp                  = bp;
     pt->backend             = bp.backend;
     pt->sched               = NULL;
@@ -161,6 +284,10 @@ bool pipeline_tts_load(PipelineTTS * pt,
     pt->use_flash_attn = use_fa && bp.has_gpu;
     pt->clamp_fp16     = clamp_fp16;
 
+    // Fused codec tail: the frame graph decodes its own audio chunk,
+    // single slot latency mode, effective at max_batch 1 only.
+    pt->codec_fused = codec_fused && pt->max_batch == 1;
+
     if (!gf_load(&pt->gguf_talker, talker_gguf_path)) {
         qt_log(QT_LOG_ERROR, "[Pipeline] failed to load talker GGUF: %s", talker_gguf_path);
         return false;
@@ -221,7 +348,7 @@ bool pipeline_tts_load(PipelineTTS * pt,
     // one buys nothing and redecodes frames for nothing.
     pt->codec_left_ctx_frames = 2 * pt->codec.transformer.sliding_window;
 
-    // Scheduler shared by talker_forward_* and code_predictor_step.
+    // Scheduler shared by talker_forward_* and the predictor frame step.
     // Routes ops the GPU backend cannot run (typical case: K-quant
     // get_rows on CUDA) to the CPU backend. 4096 nodes covers the 28L
     // Qwen3 talker graph (~48 ops per layer with KV cache writes) with
@@ -317,13 +444,10 @@ bool pipeline_tts_load(PipelineTTS * pt,
     // on first use.
     pt->talker_decode_graphs.resize(((size_t) pt->talker_kv.max_seq_len + 255) / 256);
     bool graphs_ok = graph_arena_init(&pt->talker_arena, talker_graph_max_nodes(pt->talker.num_hidden_layers)) &&
-                     pipeline_tts_cp_graphs_ensure(pt, 1);
+                     pipeline_tts_cp_graphs_ensure(pt, 1, false);
     if (!graphs_ok) {
         for (size_t n = 0; n < pt->cp_graphs.size(); n++) {
-            code_predictor_graph_free(&pt->cp_graphs[n].prefill);
+            code_predictor_graph_set_free(&pt->cp_graphs[n]);
-            for (size_t g = 0; g < pt->cp_graphs[n].steps.size(); g++) {
-                code_predictor_graph_free(&pt->cp_graphs[n].steps[g]);
-            }
         }
         pt->cp_graphs.clear();
         pt->talker_decode_graphs.clear();
@@ -355,10 +479,7 @@ bool pipeline_tts_load(PipelineTTS * pt,
 
 void pipeline_tts_free(PipelineTTS * pt) {
     for (size_t n = 0; n < pt->cp_graphs.size(); n++) {
-        code_predictor_graph_free(&pt->cp_graphs[n].prefill);
+        code_predictor_graph_set_free(&pt->cp_graphs[n]);
-        for (size_t g = 0; g < pt->cp_graphs[n].steps.size(); g++) {
-            code_predictor_graph_free(&pt->cp_graphs[n].steps[g]);
-        }
     }
     pt->cp_graphs.clear();
     for (size_t g = 0; g < pt->talker_decode_graphs.size(); g++) {
@@ -1204,9 +1325,17 @@ void tts_engine_step(TtsEngine * e, std::vector<TtsJob *> * retired) {
     for (int i = 0; i < N; i++) {
         any_live = any_live || e->slots[(size_t) i].has_frame;
     }
+    bool fused_frame = false;
     if (any_live) {
         CodePredictorOutput cp;
-        if (!pipeline_tts_cp_graphs_ensure(pt, N)) {
+        std::vector<float>  fused_audio;
+
+        // Fused single slot mode: predictor and codec run in one graph,
+        // gated on the slot actually streaming (codec_set 0 reset at
+        // admit) so the stream state and the audio consumer both exist.
+        fused_frame = pt->codec_fused && N == 1 && e->slots[0].has_frame && e->slots[0].codec_set == 0;
+
+        if (!pipeline_tts_cp_graphs_ensure(pt, N, fused_frame)) {
             qt_set_error("pipeline_tts_synthesize: code predictor graph build failed (N=%d)", N);
             for (TtsSlot & s : e->slots) {
                 s.finished   = true;
@@ -1217,8 +1346,6 @@ void tts_engine_step(TtsEngine * e, std::vector<TtsJob *> * retired) {
             CodePredGraphSet &   gs = pt->cp_graphs[(size_t) (N - 1)];
             std::vector<int32_t> c0s((size_t) N, 0);
             std::vector<float>   temps((size_t) N, 0.0f);
-            std::vector<int>     top_ks((size_t) N, 0);
-            std::vector<float>   top_ps((size_t) N, 1.0f);
             std::vector<int64_t> seeds((size_t) N, 0);
             std::vector<int64_t> subseqs((size_t) N, 0);
             const char *         cp_dump = NULL;
@@ -1230,8 +1357,6 @@ void tts_engine_step(TtsEngine * e, std::vector<TtsJob *> * retired) {
                 const struct qt_tts_params * p = s.job->params;
                 c0s[(size_t) i]                = s.pending_c0;
                 temps[(size_t) i]              = s.subtk_T;
-                top_ks[(size_t) i]             = p->subtalker_top_k;
-                top_ps[(size_t) i]             = p->subtalker_top_p;
                 seeds[(size_t) i]              = s.job->resolved_seed;
                 subseqs[(size_t) i]            = s.subseq_counter - 1;
                 if (N == 1 && s.step == 0 && p->dump_dir) {
@@ -1239,9 +1364,17 @@ void tts_engine_step(TtsEngine * e, std::vector<TtsJob *> * retired) {
                 }
             }
             Timer t_pred;
-            if (!code_predictor_step(&pt->code_predictor, pt->backend, &gs.prefill, gs.steps.data(), c0s.data(), N,
+            bool  pred_ok;
-                                     temps.data(), top_ks.data(), top_ps.data(), seeds.data(), subseqs.data(), cp_dump,
+            if (fused_frame) {
-                                     &cp)) {
+                fused_audio.resize((size_t) TOKENIZER_HOP_LENGTH);
+                pred_ok = pipeline_tts_fused_frame_step(pt, gs, c0s.data(), temps.data(), seeds.data(), subseqs.data(),
+                                                        fused_audio.data(), &cp);
+            } else {
+                pred_ok =
+                    code_predictor_frame_step(&pt->code_predictor, pt->backend, &gs.frame, &gs.sampler, c0s.data(), N,
+                                              temps.data(), seeds.data(), subseqs.data(), cp_dump, &cp);
+            }
+            if (!pred_ok) {
                 for (TtsSlot & s : e->slots) {
                     s.finished   = true;
                     s.fin_status = QT_STATUS_GENERATE_FAILED;
@@ -1271,6 +1404,17 @@ void tts_engine_step(TtsEngine * e, std::vector<TtsJob *> * retired) {
                     s.all_codes.push_back(codes);
                     s.talker_history.push_back(s.pending_c0);
 
+                    // Fused mode: this frame's audio came out of the
+                    // same compute, dispatch it now instead of staging
+                    // through the shared codec flush below.
+                    if (fused_frame && p->on_chunk && s.fin_status != QT_STATUS_CANCELLED) {
+                        if (!p->on_chunk(fused_audio.data(), TOKENIZER_HOP_LENGTH, p->on_chunk_user_data)) {
+                            qt_log(QT_LOG_INFO, "[Pipeline] on_chunk callback aborted the synthesis (fused)");
+                            s.finished   = true;
+                            s.fin_status = QT_STATUS_CANCELLED;
+                        }
+                    }
+
                     // Streaming slots stage this frame through
                     // all_codes.back() and has_frame; the shared codec
                     // flush after this loop decodes every lane in one
@@ -1344,7 +1488,7 @@ void tts_engine_step(TtsEngine * e, std::vector<TtsJob *> * retired) {
     // every retiring lane leaves with its audio fully dispatched before
     // the swap remove below. Zero rows only ever exist in that single
     // row flush, which keeps the per lane audio blocks free of padding.
-    if (e->codec_M > 0) {
+    if (e->codec_M > 0 && !fused_frame) {
         const int num_cg     = pt->num_code_groups;
         bool      any_finish = false;
         bool      any_stage  = false;

src/pipeline-tts.h

@@ -19,6 +19,7 @@
 #include "kv-cache.h"
 #include "pipeline-codec.h"
 #include "qwen.h"
+#include "sampling-graph.h"
 #include "speaker-encoder-weights.h"
 #include "talker-decode-graph.h"
 #include "talker-weights.h"
@@ -90,13 +91,36 @@ struct PromptCache {
     size_t                              max_prefix_entries;
 };
 
-// One set of static predictor graphs for a given batch width: the T=2
+// One set of static predictor graphs for a given batch width: the
-// prefill plus one T=1 step per acoustic codebook after the first.
+// frame graph replays the prefill and every acoustic step in one
+// compute, the fused flavor appends the codec stream tail. The
+// sampler inputs and the codes accumulator live in their own context
+// backed by a persistent backend buffer: every graph of the set reads
+// and writes them across replays, so they never enter gallocr pools.
 struct CodePredGraphSet {
-    CodePredGraph              prefill;
+    CodePredGraph         frame;  // prefill and every acoustic step in one cgraph
-    std::vector<CodePredGraph> steps;
+    CodePredGraph         fused;  // frame graph with the codec stream tail appended
+    CodecStreamTail       tail;   // codec side of the fused graph, tensors live in fused.ctx
+    SamplerInputs         sampler;
+    struct ggml_context * sampler_ctx = nullptr;
+    ggml_backend_buffer_t sampler_buf = nullptr;
 };
 
+static inline void code_predictor_graph_set_free(CodePredGraphSet * s) {
+    code_predictor_graph_free(&s->frame);
+    code_predictor_graph_free(&s->fused);
+    s->tail = {};
+    if (s->sampler_buf) {
+        ggml_backend_buffer_free(s->sampler_buf);
+        s->sampler_buf = nullptr;
+    }
+    if (s->sampler_ctx) {
+        ggml_free(s->sampler_ctx);
+        s->sampler_ctx = nullptr;
+    }
+    s->sampler = SamplerInputs();
+}
+
 struct PipelineTTS {
     GGUFModel             gguf_talker;
     TalkerWeights         talker;
@@ -146,6 +170,7 @@ struct PipelineTTS {
     // on sub Ampere CUDA targets.
     bool use_flash_attn;
     bool clamp_fp16;
+    bool codec_fused;  // frame graph carries the codec stream tail, single slot latency mode
 
     // Persistent KV caches, one set per slot: the talker holds the LM
     // contexts, the predictor holds one frame's 16 sub-steps per slot,
@@ -190,7 +215,8 @@ bool pipeline_tts_load(PipelineTTS * pt,
                        bool          use_fa,
                        bool          clamp_fp16,
                        int           max_batch,
-                       float         codec_chunk_sec);
+                       float         codec_chunk_sec,
+                       bool          codec_fused);
 
 void pipeline_tts_free(PipelineTTS * pt);
 

src/qwen.cpp

@@ -225,6 +225,7 @@ void qt_init_default_params(struct qt_init_params * p) {
     p->max_batch   = 1;
 
     p->codec_chunk_sec = QT_CODEC_CHUNK_SEC_DEFAULT;
+    p->codec_fused     = false;
 }
 
 void qt_tts_default_params(struct qt_tts_params * p) {
@@ -362,7 +363,7 @@ struct qt_context * qt_init(const struct qt_init_params * params) {
         }
 
         if (!pipeline_tts_load(&q->pt, params->talker_path, params->codec_path, q->bp, params->use_fa,
-                               params->clamp_fp16, max_batch, chunk_sec)) {
+                               params->clamp_fp16, max_batch, chunk_sec, params->codec_fused)) {
             qt_throw("qt_init: pipeline_tts_load failed for '%s' / '%s'", params->talker_path, params->codec_path);
         }
 

src/qwen.h

@@ -157,11 +157,19 @@ struct qt_init_params {
     // frames at 12.5 Hz). The streaming path frames its own chunks
     // through the persistent codec stream state and reads none of this.
     float codec_chunk_sec;
+
+    // Fuse the codec streaming tail into the predictor frame graph:
+    // one compute per frame delivers its 80 ms audio chunk with no
+    // host round trip between the predictor and the decode. Single
+    // slot latency mode: requires max_batch 1 and a streaming
+    // synthesis (on_chunk); throughput drops against the buffered
+    // flush, which stays the default.
+    bool codec_fused;
 };
 
 // Initialise to the standard defaults: both paths NULL (caller must set
 // them before calling qt_init), use_fa true, clamp_fp16 false,
-// max_batch 1, codec_chunk_sec 24.0.
+// max_batch 1, codec_chunk_sec 24.0, codec_fused false.
 QT_API void qt_init_default_params(struct qt_init_params * p);
 
 // Allocate every module described by params. Returns NULL on any

src/sampling-graph.h

@@ -0,0 +1,117 @@
+#pragma once
+// sampling-graph.h: the predictor sampling tail in standard ops, so
+// the whole frame decodes on the backend without per step logits
+// readbacks. Each tail applies the per step temperature, keeps the
+// top_k candidates, then draws one token where the cdf crosses the
+// per step uniform u. top_k bakes from the generation defaults at
+// build; nucleus filtering is not applied. Greedy slots upload u = 0,
+// which lands the draw on each slot's first (highest) candidate.
+//
+// Sampler inputs and the codes accumulator live in a caller owned
+// persistent context, never in gallocr input buffers. The uniform
+// draws stay on the host (philox depends only on seed and subsequence)
+// and upload once per frame inside the state tensor.
+
+#include "ggml-backend.h"
+#include "ggml.h"
+#include "philox.h"
+
+#include <vector>
+
+struct SamplerInputs {
+    struct ggml_tensor * state   = nullptr;  // [2, N, n_steps] f32, per slot (temperature, u)
+    struct ggml_tensor * codes   = nullptr;  // [N, n_codes] i32, row g holds code g of every slot
+    int                  n_steps = 0;        // sampled codes per frame (semantic + acoustic)
+    int                  N       = 0;
+    int                  top_k   = 0;        // candidate count baked into every tail
+};
+
+// Create the sampler tensors inside pctx. The caller allocates pctx
+// into a persistent backend buffer afterwards.
+static inline void sampler_inputs_build(struct ggml_context * pctx, SamplerInputs * sp, int N, int n_steps, int top_k) {
+    sp->n_steps = n_steps;
+    sp->N       = N;
+    sp->top_k   = top_k;
+    sp->state   = ggml_new_tensor_3d(pctx, GGML_TYPE_F32, 2, N, n_steps);
+    sp->codes   = ggml_new_tensor_2d(pctx, GGML_TYPE_I32, N, n_steps + 1);
+    ggml_set_name(sp->state, "sampler.state");
+    ggml_set_name(sp->codes, "sampler.codes");
+}
+
+// Upload the per frame sampler state. Greedy slots (temperature <= 0)
+// carry temperature 1 and u 0, which selects the argmax through the
+// descending candidate order. subseq_base[i] indexes slot i's philox
+// stream: draw g uses subsequence subseq_base[i] + 1 + g.
+static inline void sampler_inputs_upload(SamplerInputs * sp,
+                                         const float *   temperature,
+                                         const int64_t * seed,
+                                         const int64_t * subseq_base,
+                                         int             N) {
+    std::vector<float> st((size_t) 2 * (size_t) N * (size_t) sp->n_steps);
+
+    for (int g = 0; g < sp->n_steps; g++) {
+        for (int i = 0; i < N; i++) {
+            const bool greedy = temperature[i] <= 0.0f;
+            float      u      = 0.0f;
+            if (!greedy) {
+                philox_uniform_fill(seed[i], subseq_base[i] + 1 + g, 0u, &u, 1);
+            }
+            float * row = st.data() + ((size_t) g * (size_t) N + (size_t) i) * 2;
+            row[0]      = greedy ? 1.0f : temperature[i];
+            row[1]      = u;
+        }
+    }
+    ggml_backend_tensor_set(sp->state, st.data(), 0, st.size() * sizeof(float));
+}
+
+// One sampling tail: reads this step's per slot (temperature, u) from
+// the state, draws one token id per slot from logits [n_vocab, N] and
+// writes the N ids to row step_idx + 1 of the codes accumulator. Every
+// gather batches over the slot dim through 3D get_rows.
+static inline struct ggml_tensor * sampler_tail_build(struct ggml_context * gctx,
+                                                      struct ggml_tensor *  logits,
+                                                      SamplerInputs *       sp,
+                                                      int                   step_idx) {
+    const int64_t n_vocab = logits->ne[0];
+    const int64_t N       = logits->ne[1];
+
+    struct ggml_tensor * temp =
+        ggml_view_2d(gctx, sp->state, 1, N, sp->state->nb[1], (size_t) step_idx * sp->state->nb[2]);
+    struct ggml_tensor * u =
+        ggml_view_2d(gctx, sp->state, 1, N, sp->state->nb[1], sp->state->nb[0] + (size_t) step_idx * sp->state->nb[2]);
+
+    struct ggml_tensor * cur = ggml_div(gctx, logits, temp);
+
+    // keep each slot's top_k candidates, logits and ids in descending
+    // order. argsort guarantees the order on every backend (top_k does
+    // not), and the descending layout is what makes the u = 0 draw an
+    // argmax.
+    struct ggml_tensor * candidates = NULL;
+    if (sp->top_k > 0 && sp->top_k < n_vocab) {
+        struct ggml_tensor * order = ggml_argsort(gctx, cur, GGML_SORT_ORDER_DESC);
+        struct ggml_tensor * idx =
+            ggml_cont(gctx, ggml_view_2d(gctx, order, sp->top_k, N, order->nb[1], 0));  // [top_k, N] i32
+        struct ggml_tensor * a3d = ggml_reshape_3d(gctx, cur, 1, n_vocab, N);
+        cur                      = ggml_reshape_2d(gctx, ggml_get_rows(gctx, a3d, idx), sp->top_k, N);
+        candidates               = idx;
+    }
+
+    // draw one token per slot: find where the cdf crosses u
+    struct ggml_tensor * probs  = ggml_soft_max(gctx, cur);
+    struct ggml_tensor * cumsum = ggml_cumsum(gctx, probs);
+
+    struct ggml_tensor * diff       = ggml_sub(gctx, cumsum, u);
+    struct ggml_tensor * cross_mask = ggml_step(gctx, diff);
+    struct ggml_tensor * idxf       = ggml_sum_rows(gctx, cross_mask);  // [1, N]
+    struct ggml_tensor * idx =
+        ggml_cast(gctx, ggml_scale_bias(gctx, idxf, -1.0f, (float) cross_mask->ne[0]), GGML_TYPE_I32);
+
+    if (candidates) {
+        struct ggml_tensor * cand_3d = ggml_reshape_3d(gctx, candidates, 1, candidates->ne[0], N);
+        idx                          = ggml_get_rows(gctx, cand_3d, idx);  // [1, 1, N]
+    }
+
+    struct ggml_tensor * ids = ggml_reshape_1d(gctx, idx, N);
+    struct ggml_tensor * dst = ggml_view_1d(gctx, sp->codes, N, (size_t) (step_idx + 1) * sp->codes->nb[1]);
+    return ggml_cpy(gctx, ids, dst);
+}

tools/qwen-tts.cpp

@@ -54,6 +54,7 @@ static void print_usage(const char * prog) {
             "  --ref-text <path>       Transcript file for the reference (enables ICL clone mode)\n"
             "  --max-new <n>           Max new audio frames (default: 2048)\n"
             "  --codec-chunk-dur <f>   Codec decode chunk duration in seconds (default: 24.0)\n"
+            "  --codec-fused           Decode each frame's audio inside the predictor graph (streaming only)\n"
             "  --stream-by-line        Flush synthesis at each newline, one WAV header per line (-o '-')\n\n"
             "Sampling:\n"
             "  --seed <int>            Sampling seed (default: -1 for random)\n"
@@ -100,6 +101,7 @@ struct Args {
     bool         clamp_fp16;
     bool         stream_by_line;
     float        codec_chunk_sec;
+    bool         codec_fused;
 };
 
 // Read all of stdin into a string. Binary mode on Windows so UTF-16 input
@@ -198,6 +200,7 @@ static bool parse_args(int argc, char ** argv, Args & a) {
     // Chunk sentinel : qt_init resolves a non positive value to the
     // library default.
     a.codec_chunk_sec       = 0.0f;
+    a.codec_fused           = false;
     for (int i = 1; i < argc; i++) {
         const char * arg = argv[i];
         if (std::strcmp(arg, "-h") == 0 || std::strcmp(arg, "--help") == 0) {
@@ -257,6 +260,8 @@ static bool parse_args(int argc, char ** argv, Args & a) {
             a.clamp_fp16 = true;
         } else if (std::strcmp(arg, "--stream-by-line") == 0) {
             a.stream_by_line = true;
+        } else if (std::strcmp(arg, "--codec-fused") == 0) {
+            a.codec_fused = true;
         } else if (std::strcmp(arg, "--codec-chunk-dur") == 0 && i + 1 < argc) {
             a.codec_chunk_sec = (float) std::atof(argv[++i]);
         } else if (std::strcmp(arg, "-o") == 0 && i + 1 < argc) {
@@ -281,6 +286,7 @@ static int run(const Args & a) {
     iparams.use_fa          = a.use_fa;
     iparams.clamp_fp16      = a.clamp_fp16;
     iparams.codec_chunk_sec = a.codec_chunk_sec;
+    iparams.codec_fused     = a.codec_fused;
 
     qt_context * q = qt_init(&iparams);
     if (!q) {

tools/tts-server.cpp

@@ -52,7 +52,8 @@ static void print_usage(const char * prog) {
             "  --max-batch <n>         Concurrent requests batched on the GPU (default: 1)\n"
             "  --no-fa                 Disable flash attention\n"
             "  --clamp-fp16            Clamp hidden states to FP16 range\n"
-            "  --codec-chunk-dur <f>   Codec decode chunk duration in seconds, wav responses (default: 24.0)\n",
+            "  --codec-chunk-dur <f>   Codec decode chunk duration in seconds, wav responses (default: 24.0)\n"
+            "  --codec-fused           Decode each frame's audio inside the predictor graph (max-batch 1, streaming)\n",
             prog);
 }
 
@@ -71,6 +72,7 @@ int main(int argc, char ** argv) {
     server_config cfg;
     bool          use_fa          = true;
     bool          clamp_fp16      = false;
+    bool          codec_fused     = false;
     int           max_batch       = 1;
     // Chunk sentinel : qt_init resolves a non positive value to the
     // library default.
@@ -96,6 +98,8 @@ int main(int argc, char ** argv) {
             clamp_fp16 = true;
         } else if (!std::strcmp(arg, "--max-batch") && i + 1 < argc) {
             max_batch = std::atoi(argv[++i]);
+        } else if (!std::strcmp(arg, "--codec-fused")) {
+            codec_fused = true;
         } else if (!std::strcmp(arg, "--codec-chunk-dur") && i + 1 < argc) {
             codec_chunk_dur = (float) std::atof(argv[++i]);
         } else if (!std::strcmp(arg, "--help") || !std::strcmp(arg, "-h")) {
@@ -121,6 +125,7 @@ int main(int argc, char ** argv) {
     iparams.clamp_fp16      = clamp_fp16;
     iparams.max_batch       = max_batch;
     iparams.codec_chunk_sec = codec_chunk_dur;
+    iparams.codec_fused     = codec_fused;
 
     struct qt_context * q = qt_init(&iparams);
     if (!q) {