diff --git a/src/causal-trans-conv.h b/src/causal-trans-conv.h index 9f4ba01..87816ac 100644 --- a/src/causal-trans-conv.h +++ b/src/causal-trans-conv.h @@ -36,12 +36,14 @@ static struct ggml_tensor * qwen_load_ctw_f32(WeightCtx * wctx, const GGUFModel fprintf(stderr, "[CausalTransConv] FATAL: tensor '%s' not found\n", name.c_str()); exit(1); } - // Source dtype is F32 in the F32 master, F16 in the quantized variants - // since 3D conv weights cannot be Q8_0 / Q4_K_M and ggml falls back to - // F16 in the quantizer. Both paths cast to F32 here ; the K*OC*IC + // Source dtype follows the GGUF norm (pure llama.cpp policy). The F32 + // master keeps tensors in F32, the BF16 variant keeps them in their + // source BF16, and the K-quant variants land them in F16 through the + // aligned fallback (kernel rows of width K=2 do not divide a K-quant + // block size). All three are widened to F32 here ; the K*OC*IC // permutation always lands in a freshly allocated F32 buffer anyway. - if (src->type != GGML_TYPE_F32 && src->type != GGML_TYPE_F16) { - fprintf(stderr, "[CausalTransConv] FATAL: '%s' expected F32 or F16, got type %d\n", name.c_str(), + if (src->type != GGML_TYPE_F32 && src->type != GGML_TYPE_F16 && src->type != GGML_TYPE_BF16) { + fprintf(stderr, "[CausalTransConv] FATAL: '%s' expected F32, F16 or BF16, got type %d\n", name.c_str(), (int) src->type); exit(1); } @@ -60,7 +62,10 @@ static struct ggml_tensor * qwen_load_ctw_f32(WeightCtx * wctx, const GGUFModel if (src->type == GGML_TYPE_F32) { return ((const float *) raw)[idx]; } - return ggml_fp16_to_fp32(((const ggml_fp16_t *) raw)[idx]); + if (src->type == GGML_TYPE_F16) { + return ggml_fp16_to_fp32(((const ggml_fp16_t *) raw)[idx]); + } + return ggml_bf16_to_fp32(((const ggml_bf16_t *) raw)[idx]); }; for (int ic = 0; ic < IC; ic++) { diff --git a/src/convnext-block.h b/src/convnext-block.h index a135de3..aa5577f 100644 --- a/src/convnext-block.h +++ b/src/convnext-block.h @@ -77,7 +77,7 @@ static bool qwen_upsample_stage_load(QwenUpsampleStage * stage, const GGUFModel QwenConvNeXtBlock & cn = stage->convnext[i]; snprintf(name, sizeof(name), "tok_dec.upsample.%d.dwconv.weight", i); - cn.dwconv_w = gf_load_tensor(&wctx, gf, name); + cn.dwconv_w = gf_load_conv(&wctx, gf, name); snprintf(name, sizeof(name), "tok_dec.upsample.%d.dwconv.bias", i); cn.dwconv_b = gf_load_tensor(&wctx, gf, name); snprintf(name, sizeof(name), "tok_dec.upsample.%d.norm.weight", i); diff --git a/src/dac-decoder-v2.h b/src/dac-decoder-v2.h index 0298f3a..a7ae529 100644 --- a/src/dac-decoder-v2.h +++ b/src/dac-decoder-v2.h @@ -150,7 +150,7 @@ static bool qwen_dac_decoder_load(QwenDACDecoder * d, const GGUFModel & gf, ggml WeightCtx wctx; wctx_init(&wctx, n_tensors); - d->conv_pre_w = gf_load_tensor(&wctx, gf, "tok_dec.dec.0.conv.weight"); + d->conv_pre_w = gf_load_conv(&wctx, gf, "tok_dec.dec.0.conv.weight"); d->conv_pre_b = gf_load_tensor(&wctx, gf, "tok_dec.dec.0.conv.bias"); for (int i = 0; i < QWEN_DAC_NUM_BLOCKS; i++) { @@ -182,17 +182,17 @@ static bool qwen_dac_decoder_load(QwenDACDecoder * d, const GGUFModel & gf, ggml qwen_dac_load_snakebeta(&wctx, gf, &ru.act1, std::string(rp) + ".act1.alpha", std::string(rp) + ".act1.beta"); - ru.c1w = gf_load_tensor(&wctx, gf, std::string(rp) + ".conv1.weight"); + ru.c1w = gf_load_conv(&wctx, gf, std::string(rp) + ".conv1.weight"); ru.c1b = gf_load_tensor(&wctx, gf, std::string(rp) + ".conv1.bias"); qwen_dac_load_snakebeta(&wctx, gf, &ru.act2, std::string(rp) + ".act2.alpha", std::string(rp) + ".act2.beta"); - ru.c2w = gf_load_tensor(&wctx, gf, std::string(rp) + ".conv2.weight"); + ru.c2w = gf_load_conv(&wctx, gf, std::string(rp) + ".conv2.weight"); ru.c2b = gf_load_tensor(&wctx, gf, std::string(rp) + ".conv2.bias"); } } qwen_dac_load_snakebeta(&wctx, gf, &d->snake_post, "tok_dec.dec.5.snake.alpha", "tok_dec.dec.5.snake.beta"); - d->conv_post_w = gf_load_tensor(&wctx, gf, "tok_dec.dec.6.conv.weight"); + d->conv_post_w = gf_load_conv(&wctx, gf, "tok_dec.dec.6.conv.weight"); d->conv_post_b = gf_load_tensor(&wctx, gf, "tok_dec.dec.6.conv.bias"); if (!wctx_alloc(&wctx, backend)) { diff --git a/src/encoder-downsample.h b/src/encoder-downsample.h index 1930e04..320d471 100644 --- a/src/encoder-downsample.h +++ b/src/encoder-downsample.h @@ -30,7 +30,7 @@ static bool qwen_encoder_downsample_load(QwenEncoderDownsample * d, const GGUFMo WeightCtx wctx; wctx_init(&wctx, 4); - d->weight = gf_load_tensor(&wctx, gf, "tok_enc.downsample.weight"); + d->weight = gf_load_conv(&wctx, gf, "tok_enc.downsample.weight"); if (!wctx_alloc(&wctx, backend)) { fprintf(stderr, "[EncDownsample] FATAL: backend allocation failed\n"); return false; diff --git a/src/gguf-weights.h b/src/gguf-weights.h index 2708099..6785122 100644 --- a/src/gguf-weights.h +++ b/src/gguf-weights.h @@ -266,6 +266,91 @@ static struct ggml_tensor * gf_load_tensor_f32(WeightCtx * wctx, const GGUFModel return tensor; } +// Load a Conv1d / Conv1dDW kernel weight, forcing F16 storage on the +// backend regardless of the source GGUF dtype. +// +// TODO upstream GGML : ggml_conv_1d and ggml_conv_1d_dw in +// ggml/src/ggml.c hardcode dst_type = GGML_TYPE_F16 in their internal +// ggml_im2col call (currently ggml.c lines around 4508 and 4542). +// ggml_conv_2d at the equivalent site uses the adaptive pattern +// dst_type = a->type (around line 4595). Two backend bugs follow. +// +// 1) CPU im2col dispatches on dst->type. The im2col_f16 path +// asserts src0->type == GGML_TYPE_F16, so a F32 or BF16 conv +// kernel crashes on CPU. +// 2) ggml_conv_1d lowers to ggml_mul_mat(im2col, reshape(weight)) +// where the weight ends up as src1, not src0. The Vulkan fast +// path ggml_vk_get_dequantize_mul_mat_vec asserts b_type in +// {F32, F16, Q8_1}, so a BF16 conv kernel crashes on Vulkan +// even when im2col itself succeeds. +// +// Vulkan does ship a pipeline_im2col_f32 and CUDA handles both F32 +// and F16 cleanly, so the fix upstream is to align ggml_conv_1d and +// ggml_conv_1d_dw on ggml_conv_2d's adaptive a->type pattern. Until +// that lands, the only safe assumption across CPU, CUDA, and Vulkan +// is F16 kernels everywhere, so we mirror ggml_conv_1d's hardcoded +// choice here and load every conv kernel as F16 regardless of source +// dtype. F16 source is a direct passthrough, F32 and BF16 widen +// through a F32 staging buffer. +static struct ggml_tensor * gf_load_conv(WeightCtx * wctx, const GGUFModel & gf, const std::string & name) { + int64_t idx = gguf_find_tensor(gf.gguf, name.c_str()); + if (idx < 0) { + fprintf(stderr, "[GGUF] FATAL: tensor '%s' not found\n", name.c_str()); + exit(1); + } + struct ggml_tensor * src = ggml_get_tensor(gf.meta, name.c_str()); + int n_dims = ggml_n_dims(src); + int64_t ne[4] = { 1, 1, 1, 1 }; + for (int i = 0; i < n_dims; i++) { + ne[i] = src->ne[i]; + } + + // F16 source : direct passthrough, no conversion. + if (src->type == GGML_TYPE_F16) { + return gf_load_tensor(wctx, gf, name); + } + if (src->type != GGML_TYPE_F32 && src->type != GGML_TYPE_BF16) { + fprintf(stderr, "[GGUF] FATAL: gf_load_conv unsupported source type %s for '%s'\n", ggml_type_name(src->type), + name.c_str()); + exit(1); + } + + // Allocate F16 backend tensor in the WeightCtx graph. + struct ggml_tensor * tensor = ggml_new_tensor(wctx->ctx, GGML_TYPE_F16, n_dims, ne); + ggml_set_name(tensor, name.c_str()); + + size_t n = (size_t) ggml_nelements(src); + size_t raw_off = gguf_get_tensor_offset(gf.gguf, idx); + const void * raw = gf.mapping + gf.data_offset + raw_off; + + // The staging vector owns float[] buffers to keep memory alive + // until wctx_alloc copies it to the backend. n F16 elements + // occupy n * 2 bytes, which fits in (n + 1) / 2 floats. The + // pending entry references the same buffer reinterpreted as + // ggml_fp16_t and carries the exact F16 byte count. + size_t n_floats = (n + 1) / 2; + auto buf = std::make_unique(n_floats); + ggml_fp16_t * data = (ggml_fp16_t *) buf.get(); + + if (src->type == GGML_TYPE_F32) { + ggml_fp32_to_fp16_row((const float *) raw, data, (int) n); + } else { + // BF16 source : widen to F32 first, then narrow to F16 in + // one pass to preserve mantissa bits the BF16-to-F16 direct + // cast would otherwise leave undefined. + std::vector f32(n); + const uint16_t * p = (const uint16_t *) raw; + for (size_t i = 0; i < n; i++) { + f32[i] = ggml_bf16_to_fp32(*(const ggml_bf16_t *) &p[i]); + } + ggml_fp32_to_fp16_row(f32.data(), data, (int) n); + } + + wctx->pending.push_back({ tensor, (const void *) data, n * sizeof(ggml_fp16_t), 0 }); + wctx->staging.push_back(std::move(buf)); + return tensor; +} + // Get raw pointer to tensor data in the mmapped file. // Useful for CPU-side operations (e.g. bf16 embed lookup for lyrics). // Returns NULL if not found. @@ -290,53 +375,6 @@ static enum ggml_type gf_get_type(const GGUFModel & gf, const std::string & name return src->type; } -// Load a Conv1d weight onto an F16 backend tensor regardless of the source -// dtype. Mandatory on ARM aarch64 : the CPU im2col op asserts src0 is F16, -// while x86 silently accepts BF16 / F32. F16 source memcpy passes through ; -// F32 / BF16 widen ; Q8_0 / Q4_K / Q5_K / Q6_K dequantize via type traits. -// The destination tensor must be allocated as GGML_TYPE_F16. -static void gf_load_conv_f16(struct ggml_tensor * dst, const GGUFModel & gf, const std::string & name) { - struct ggml_tensor * src = ggml_get_tensor(gf.meta, name.c_str()); - if (!src) { - fprintf(stderr, "[GGUF] FATAL: tensor '%s' not in meta context\n", name.c_str()); - exit(1); - } - GGML_ASSERT(dst->type == GGML_TYPE_F16); - GGML_ASSERT(ggml_nelements(dst) == ggml_nelements(src)); - - const void * raw = gf_get_data(gf, name.c_str()); - size_t n = (size_t) ggml_nelements(src); - - // F16 source : direct memcpy, no conversion needed. - if (src->type == GGML_TYPE_F16) { - ggml_backend_tensor_set(dst, raw, 0, ggml_nbytes(dst)); - return; - } - - // All other types widen / dequantize to F32, then cast down to F16. - std::vector f32(n); - if (src->type == GGML_TYPE_F32) { - memcpy(f32.data(), raw, n * sizeof(float)); - } else if (src->type == GGML_TYPE_BF16) { - const uint16_t * p = (const uint16_t *) raw; - for (size_t i = 0; i < n; i++) { - f32[i] = ggml_bf16_to_fp32(*(const ggml_bf16_t *) &p[i]); - } - } else { - const struct ggml_type_traits * tr = ggml_get_type_traits(src->type); - if (!tr || !tr->to_float) { - fprintf(stderr, "[GGUF] FATAL: unsupported conv weight type %s for '%s'\n", ggml_type_name(src->type), - name.c_str()); - exit(1); - } - tr->to_float(raw, f32.data(), (int64_t) n); - } - - std::vector f16(n); - ggml_fp32_to_fp16_row(f32.data(), f16.data(), (int) n); - ggml_backend_tensor_set(dst, f16.data(), 0, n * sizeof(ggml_fp16_t)); -} - // Fuse Q, K, V projection weights into a single tensor [ne0, q_ne1 + k_ne1 + v_ne1]. // Works for any quantized type since quantization is per-row (along ne[0]). // The fused tensor data is q rows || k rows || v rows (contiguous). diff --git a/src/pipeline-codec.cpp b/src/pipeline-codec.cpp index 356b8ae..a36aa96 100644 --- a/src/pipeline-codec.cpp +++ b/src/pipeline-codec.cpp @@ -57,7 +57,7 @@ bool pipeline_codec_load(PipelineCodec * pc, const char * gguf_path, BackendPair { WeightCtx wctx; wctx_init(&wctx, 4); - pc->pre_conv_w = gf_load_tensor(&wctx, pc->gguf, "tok_dec.pre_conv.weight"); + pc->pre_conv_w = gf_load_conv(&wctx, pc->gguf, "tok_dec.pre_conv.weight"); pc->pre_conv_b = gf_load_tensor(&wctx, pc->gguf, "tok_dec.pre_conv.bias"); if (!wctx_alloc(&wctx, pc->backend)) { qt_log(QT_LOG_ERROR, "[Pipeline] pre_conv backend allocation failed"); diff --git a/src/seanet-encoder.h b/src/seanet-encoder.h index 5fa83a7..7ad3c64 100644 --- a/src/seanet-encoder.h +++ b/src/seanet-encoder.h @@ -107,7 +107,7 @@ static bool qwen_seanet_encoder_load(QwenSEANetEncoder * s, const GGUFModel & gf WeightCtx wctx; wctx_init(&wctx, n_tensors); - s->init_w = gf_load_tensor(&wctx, gf, "tok_enc.conv.0.weight"); + s->init_w = gf_load_conv(&wctx, gf, "tok_enc.conv.0.weight"); s->init_b = gf_load_tensor(&wctx, gf, "tok_enc.conv.0.bias"); // Stage indexing follows the Python ModuleList layout : @@ -125,23 +125,23 @@ static bool qwen_seanet_encoder_load(QwenSEANetEncoder * s, const GGUFModel & gf char name[80]; snprintf(name, sizeof(name), "tok_enc.res.%d.blk.1.weight", RES_PY_IDX[i]); - stg.resnet.c0_w = gf_load_tensor(&wctx, gf, name); + stg.resnet.c0_w = gf_load_conv(&wctx, gf, name); snprintf(name, sizeof(name), "tok_enc.res.%d.blk.1.bias", RES_PY_IDX[i]); stg.resnet.c0_b = gf_load_tensor(&wctx, gf, name); snprintf(name, sizeof(name), "tok_enc.res.%d.blk.3.weight", RES_PY_IDX[i]); - stg.resnet.c1_w = gf_load_tensor(&wctx, gf, name); + stg.resnet.c1_w = gf_load_conv(&wctx, gf, name); snprintf(name, sizeof(name), "tok_enc.res.%d.blk.3.bias", RES_PY_IDX[i]); stg.resnet.c1_b = gf_load_tensor(&wctx, gf, name); snprintf(name, sizeof(name), "tok_enc.conv.%d.weight", DOWN_PY_IDX[i]); - stg.down_w = gf_load_tensor(&wctx, gf, name); + stg.down_w = gf_load_conv(&wctx, gf, name); snprintf(name, sizeof(name), "tok_enc.conv.%d.bias", DOWN_PY_IDX[i]); stg.down_b = gf_load_tensor(&wctx, gf, name); dim = stg.out_ch; } - s->last_w = gf_load_tensor(&wctx, gf, "tok_enc.conv.14.weight"); + s->last_w = gf_load_conv(&wctx, gf, "tok_enc.conv.14.weight"); s->last_b = gf_load_tensor(&wctx, gf, "tok_enc.conv.14.bias"); if (!wctx_alloc(&wctx, backend)) { diff --git a/tools/quantize.cpp b/tools/quantize.cpp index 85d6692..a557694 100644 --- a/tools/quantize.cpp +++ b/tools/quantize.cpp @@ -106,8 +106,9 @@ static bool is_embed(const char * name) { // false here keep their source dtype (F32) regardless of the requested // type. Conv weights pass through the main loop and fall back to F16 when // the row width does not divide the variant block size (kernel K=7,3,1,...). -// gf_load_conv_f16 then memcpys F16 source straight to the F16 backend -// tensor (ARM im2col strict requirement, see src/gguf-weights.h). +// gf_load_conv then loads every conv kernel as F16 on the backend +// regardless of the source GGUF dtype, see the TODO upstream block above +// the function definition in src/gguf-weights.h. // // Sensitive tensors that MUST stay in full precision : // tok_enc.vq_*.{i}.codebook RVQ codebook tables, encoder side @@ -374,8 +375,8 @@ int main(int argc, char ** argv) { // Conv kernels (K=7,3,1,...) cannot fit a block-quant row : fall back // to F16. F16 has no block size, 10-bit mantissa beats BF16 (7) and - // Q* effective on these weights, and gf_load_conv_f16 memcpys F16 - // source straight to the F16 backend tensor at load time. + // Q* effective on these weights, and gf_load_conv on the C++ side + // loads every conv kernel as F16 regardless of source dtype. if (can_convert && !aligned) { target = GGML_TYPE_F16; aligned = true;