Merge pull request #5 from andimarafioti/codex/voice-ref-abi

Add voice reference extraction ABI

README.md

@@ -149,6 +149,16 @@ qt_audio_free(&audio);
 qt_free(q);
 ```
 
+Base voice-clone latents can also be precomputed in-process, replacing
+the `qwen-codec --talker ref.wav` shell-out: `qt_extract_voice_ref`
+takes the decoded `.wav` contents as mono float32 PCM at 24 kHz and
+fills a `struct qt_voice_ref` with the `.spk`-equivalent speaker
+embedding plus the `.rvq`-equivalent `[num_codebooks, ref_T]` code
+matrix. Pass those buffers back through `qt_tts_params.ref_spk_emb` /
+`ref_codes`, and for reference-WAV-plus-transcription ICL mode keep
+passing the transcript as `qt_tts_params.ref_text`. Release the buffers
+with `qt_voice_ref_free`.
+
 `tests/abi-c.c` is built with `-std=c99 -Wall -Werror -pedantic` on
 every build (the `test-abi-c` target), so any regression that breaks
 plain C consumability fails the build, not just an opt-in target.

src/qwen.cpp

@@ -20,6 +20,7 @@
 #include "bpe.h"
 #include "pipeline-tts.h"
 #include "qt-error.h"
+#include "speaker-encoder-extract.h"
 #include "version.h"
 
 #include <atomic>
@@ -27,9 +28,11 @@
 #include <cstdio>
 #include <cstdlib>
 #include <cstring>
+#include <new>
 #include <random>
 #include <stdexcept>
 #include <string>
+#include <vector>
 
 // Internal definition of the opaque handle. C++ types are fine here
 // because nothing in this struct ever crosses the public ABI boundary :
@@ -300,6 +303,121 @@ void qt_free(struct qt_context * q) {
     delete q;
 }
 
+void qt_voice_ref_free(struct qt_voice_ref * ref) {
+    if (!ref) {
+        return;
+    }
+    if (ref->ref_spk_emb) {
+        std::free(ref->ref_spk_emb);
+    }
+    if (ref->ref_codes) {
+        std::free(ref->ref_codes);
+    }
+    ref->ref_spk_emb   = nullptr;
+    ref->ref_spk_dim   = 0;
+    ref->ref_codes     = nullptr;
+    ref->ref_T         = 0;
+    ref->num_codebooks = 0;
+}
+
+enum qt_status qt_extract_voice_ref(struct qt_context *   q,
+                                    const float *         ref_audio_24k,
+                                    int                   ref_n_samples,
+                                    struct qt_voice_ref * out) {
+    if (out) {
+        qt_voice_ref_free(out);
+    }
+    if (!q || !ref_audio_24k || !out) {
+        qt_set_error("qt_extract_voice_ref: q, ref_audio_24k or out is NULL");
+        return QT_STATUS_INVALID_PARAMS;
+    }
+    if (ref_n_samples < TOKENIZER_HOP_LENGTH) {
+        qt_set_error("qt_extract_voice_ref: ref_audio_24k too short for RVQ encode (%d samples, need at least %d)",
+                     ref_n_samples, TOKENIZER_HOP_LENGTH);
+        return QT_STATUS_INVALID_PARAMS;
+    }
+
+    const std::string & mt = q->pt.model_type;
+    if (mt != "base") {
+        qt_set_error("qt_extract_voice_ref: voice references are only valid for base models (loaded: %s)", mt.c_str());
+        return QT_STATUS_MODE_INVALID;
+    }
+    if (!q->pt.has_speaker_encoder) {
+        qt_set_error("qt_extract_voice_ref: loaded base model has no speaker encoder");
+        return QT_STATUS_GENERATE_FAILED;
+    }
+    if (q->pt.num_code_groups <= 0) {
+        qt_set_error("qt_extract_voice_ref: invalid codebook count %d", q->pt.num_code_groups);
+        return QT_STATUS_GENERATE_FAILED;
+    }
+
+    try {
+        std::vector<float> emb;
+        if (!speaker_encoder_extract(&q->pt.speaker_encoder, q->pt.sched, ref_audio_24k, ref_n_samples, emb)) {
+            qt_set_error("qt_extract_voice_ref: speaker embedding extraction failed");
+            return QT_STATUS_GENERATE_FAILED;
+        }
+        if ((int) emb.size() != q->pt.talker.hidden_size) {
+            qt_set_error("qt_extract_voice_ref: speaker embedding size %zu mismatches talker hidden %d", emb.size(),
+                         q->pt.talker.hidden_size);
+            return QT_STATUS_GENERATE_FAILED;
+        }
+
+        const int            aligned_n = (ref_n_samples / TOKENIZER_HOP_LENGTH) * TOKENIZER_HOP_LENGTH;
+        const int            ref_T     = aligned_n / TOKENIZER_HOP_LENGTH;
+        std::vector<int32_t> codes     = pipeline_codec_encode(&q->pt.codec, ref_audio_24k, aligned_n);
+        if (codes.empty()) {
+            qt_set_error("qt_extract_voice_ref: pipeline_codec_encode returned empty codes");
+            return QT_STATUS_GENERATE_FAILED;
+        }
+        const int num_codebooks = q->pt.num_code_groups;
+        if ((codes.size() % (size_t) num_codebooks) != 0) {
+            qt_set_error("qt_extract_voice_ref: encoded code count %zu is not divisible by %d", codes.size(),
+                         num_codebooks);
+            return QT_STATUS_GENERATE_FAILED;
+        }
+        const int codes_T = (int) (codes.size() / (size_t) num_codebooks);
+        if (codes_T != ref_T) {
+            qt_set_error("qt_extract_voice_ref: encoded frame count %d mismatches aligned frame count %d", codes_T,
+                         ref_T);
+            return QT_STATUS_GENERATE_FAILED;
+        }
+
+        const size_t emb_bytes   = emb.size() * sizeof(float);
+        const size_t codes_bytes = codes.size() * sizeof(int32_t);
+        float *      emb_copy    = (float *) std::malloc(emb_bytes);
+        int32_t *    codes_copy  = (int32_t *) std::malloc(codes_bytes);
+        if (!emb_copy || !codes_copy) {
+            std::free(emb_copy);
+            std::free(codes_copy);
+            qt_set_error("qt_extract_voice_ref: malloc failed for %zu emb bytes and %zu code bytes", emb_bytes,
+                         codes_bytes);
+            return QT_STATUS_OOM;
+        }
+        std::memcpy(emb_copy, emb.data(), emb_bytes);
+        std::memcpy(codes_copy, codes.data(), codes_bytes);
+
+        out->ref_spk_emb   = emb_copy;
+        out->ref_spk_dim   = (int) emb.size();
+        out->ref_codes     = codes_copy;
+        out->ref_T         = ref_T;
+        out->num_codebooks = num_codebooks;
+
+        qt_log(QT_LOG_INFO, "[Qwen] Extracted voice ref: spk_dim=%d K=%d T=%d (%d/%d samples)", out->ref_spk_dim,
+               out->num_codebooks, out->ref_T, aligned_n, ref_n_samples);
+        return QT_STATUS_OK;
+    } catch (const std::bad_alloc &) {
+        qt_set_error("qt_extract_voice_ref: out of memory");
+        qt_voice_ref_free(out);
+        return QT_STATUS_OOM;
+    } catch (const std::exception & e) {
+        qt_set_error("%s", e.what());
+        qt_log(QT_LOG_ERROR, "[Qwen] %s", e.what());
+        qt_voice_ref_free(out);
+        return QT_STATUS_GENERATE_FAILED;
+    }
+}
+
 enum qt_status qt_synthesize(struct qt_context * q, const struct qt_tts_params * params, struct qt_audio * out) {
     if (!q || !params) {
         qt_set_error("qt_synthesize: q or params is NULL");

src/qwen.h

@@ -137,6 +137,44 @@ QT_API struct qt_context * qt_init(const struct qt_init_params * params);
 // Safe on NULL.
 QT_API void qt_free(struct qt_context * q);
 
+// Precomputed Base-model voice reference latents. Plain POD: both
+// pointers are malloc allocated by qt_extract_voice_ref, owned by the
+// struct, released by qt_voice_ref_free. Do not free either pointer
+// directly nor reassign without freeing first. Zero initialise before
+// first use: `struct qt_voice_ref ref = {0};`.
+//
+// ref_spk_emb is the speaker embedding equivalent to a raw .spk file.
+// ref_codes is the RVQ code matrix equivalent to a raw .rvq file,
+// laid out [num_codebooks, ref_T] row-major (T fastest).
+struct qt_voice_ref {
+    float *   ref_spk_emb;
+    int       ref_spk_dim;
+    int32_t * ref_codes;
+    int       ref_T;
+    int       num_codebooks;
+};
+
+// Extract reusable voice-clone conditioning from a decoded reference
+// .wav/audio buffer: mono float32 PCM at 24 kHz. Requires a loaded Base
+// model with speaker encoder weights. The speaker embedding consumes the
+// full input buffer, matching --ref-wav clone mode A. RVQ encoding
+// truncates to the codec hop boundary, matching qwen-codec --talker
+// ref.wav / --ref-rvq.
+// For reference-WAV-plus-transcription ICL mode, pass the returned
+// ref_spk_emb and ref_codes back to qt_synthesize together with the
+// transcript in qt_tts_params.ref_text.
+//
+// On success fills out with malloc-owned buffers. On failure leaves out
+// empty and stores a diagnostic in qt_last_error().
+QT_API enum qt_status qt_extract_voice_ref(struct qt_context *   q,
+                                           const float *         ref_audio_24k,
+                                           int                   ref_n_samples,
+                                           struct qt_voice_ref * out);
+
+// Release the speaker embedding and RVQ code buffers and reset the
+// struct to empty. Safe on a zero initialised struct.
+QT_API void qt_voice_ref_free(struct qt_voice_ref * ref);
+
 // Cooperative cancellation callback. Returns true to request the
 // synthesis to abort. Polled at the top of every Talker decode step in
 // the autoregressive loop, so the cancel granularity is roughly one

tests/abi-c.c

@@ -94,6 +94,9 @@ int main(void) {
     struct qt_audio audio = { 0 };
     qt_audio_free(&audio);
 
+    struct qt_voice_ref voice_ref = { 0 };
+    qt_voice_ref_free(&voice_ref);
+
     /* Install the log callback before the failing init so the [Qwen]
      * ERROR line lands on stub_log instead of stderr. */
     qt_log_set(stub_log, NULL);
@@ -136,9 +139,9 @@ int main(void) {
      * rejected up front, before any allocation. */
     struct qt_init_params future_iparams;
     qt_init_default_params(&future_iparams);
-    future_iparams.talker_path = "irrelevant.gguf";
-    future_iparams.codec_path  = "irrelevant.gguf";
-    future_iparams.abi_version = QT_ABI_VERSION + 1;
+    future_iparams.talker_path   = "irrelevant.gguf";
+    future_iparams.codec_path    = "irrelevant.gguf";
+    future_iparams.abi_version   = QT_ABI_VERSION + 1;
     struct qt_context * rejected = qt_init(&future_iparams);
     if (rejected != NULL) {
         fprintf(stderr, "[Probe] qt_init accepted a future abi_version\n");
@@ -153,6 +156,13 @@ int main(void) {
         return 3;
     }
 
+    rc = qt_extract_voice_ref(NULL, NULL, 0, &voice_ref);
+    if (rc != QT_STATUS_INVALID_PARAMS) {
+        fprintf(stderr, "[Probe] qt_extract_voice_ref(NULL) returned %d, expected %d\n", (int) rc,
+                (int) QT_STATUS_INVALID_PARAMS);
+        return 11;
+    }
+
     int frames = qt_duration_sec_to_tokens(NULL, 1.0f);
     if (frames < 1) {
         fprintf(stderr, "[Probe] qt_duration_sec_to_tokens returned %d, expected >= 1\n", frames);
@@ -177,6 +187,7 @@ int main(void) {
 
     qt_free(NULL);
     qt_audio_free(&audio);
+    qt_voice_ref_free(&voice_ref);
 
     return 0;
 }