predictor: unroll the frame into one cgraph and sample in standard ops
One static frame graph per batch width replaces the per step chain: prefill and the 15 acoustic steps run in a single backend compute. This is the target architecture for the llama.cpp Qwen3-TTS port and serves as its working GGML reference while under test. Sampling is a plain op chain batched over slots: temperature, argsort top_k (descending order is guaranteed on every backend, unlike top_k), softmax, cumsum, cdf crossing against a per step philox uniform. Greedy draws with u = 0 and lands on the argmax. Faster than the fused sampling op under CUDA graph capture, greedy codes stay exact against the Python reference on CPU, CUDA and Vulkan. Opt in single slot latency mode (--codec-fused on qwen-tts and tts-server, codec_fused in qt_init_params): the codec stream tail joins the frame graph at T=1, codes read through a device view, one 80 ms chunk per compute with no host round trip. Predictor 3.34 -> 3.11 ms/frame on CUDA, end to end -4%.
This commit is contained in:
@@ -54,6 +54,7 @@ static void print_usage(const char * prog) {
|
||||
" --ref-text <path> Transcript file for the reference (enables ICL clone mode)\n"
|
||||
" --max-new <n> Max new audio frames (default: 2048)\n"
|
||||
" --codec-chunk-dur <f> Codec decode chunk duration in seconds (default: 24.0)\n"
|
||||
" --codec-fused Decode each frame's audio inside the predictor graph (streaming only)\n"
|
||||
" --stream-by-line Flush synthesis at each newline, one WAV header per line (-o '-')\n\n"
|
||||
"Sampling:\n"
|
||||
" --seed <int> Sampling seed (default: -1 for random)\n"
|
||||
@@ -100,6 +101,7 @@ struct Args {
|
||||
bool clamp_fp16;
|
||||
bool stream_by_line;
|
||||
float codec_chunk_sec;
|
||||
bool codec_fused;
|
||||
};
|
||||
|
||||
// Read all of stdin into a string. Binary mode on Windows so UTF-16 input
|
||||
@@ -198,6 +200,7 @@ static bool parse_args(int argc, char ** argv, Args & a) {
|
||||
// Chunk sentinel : qt_init resolves a non positive value to the
|
||||
// library default.
|
||||
a.codec_chunk_sec = 0.0f;
|
||||
a.codec_fused = false;
|
||||
for (int i = 1; i < argc; i++) {
|
||||
const char * arg = argv[i];
|
||||
if (std::strcmp(arg, "-h") == 0 || std::strcmp(arg, "--help") == 0) {
|
||||
@@ -257,6 +260,8 @@ static bool parse_args(int argc, char ** argv, Args & a) {
|
||||
a.clamp_fp16 = true;
|
||||
} else if (std::strcmp(arg, "--stream-by-line") == 0) {
|
||||
a.stream_by_line = true;
|
||||
} else if (std::strcmp(arg, "--codec-fused") == 0) {
|
||||
a.codec_fused = true;
|
||||
} else if (std::strcmp(arg, "--codec-chunk-dur") == 0 && i + 1 < argc) {
|
||||
a.codec_chunk_sec = (float) std::atof(argv[++i]);
|
||||
} else if (std::strcmp(arg, "-o") == 0 && i + 1 < argc) {
|
||||
@@ -281,6 +286,7 @@ static int run(const Args & a) {
|
||||
iparams.use_fa = a.use_fa;
|
||||
iparams.clamp_fp16 = a.clamp_fp16;
|
||||
iparams.codec_chunk_sec = a.codec_chunk_sec;
|
||||
iparams.codec_fused = a.codec_fused;
|
||||
|
||||
qt_context * q = qt_init(&iparams);
|
||||
if (!q) {
|
||||
|
||||
Reference in New Issue
Block a user