enne2/qwentts.cpp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

loader, quantize: align GGUF on llama.cpp, conv kernels widened to F16 at load

Browse Source 
GGUF norm matches llama.cpp policy: F32 master stays F32, BF16
variant keeps source BF16, K-quants fall back to F16 when kernel
rows do not align. No conv override in pick_type.

Conv kernels widen to F16 at load through gf_load_conv (12 sites).
qwen_load_ctw_f32 accepts BF16 source.

TODO upstream GGML: ggml_conv_1d and ggml_conv_1d_dw force F16 on
their im2col output, while conv_2d picks the kernel dtype. This
crashes F32 and BF16 kernels on CPU (im2col only handles F16) and
BF16 on Vulkan (mul_mat refuses BF16 on the operand the kernel
ends up on). Aligning conv_1d on conv_2d removes the workaround.
This commit is contained in:
Pascal
2026-05-11 12:42:54 +02:00
parent 59fda26827
commit 530eed6ac5
8 changed files with 113 additions and 69 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/causal-trans-conv.h
+11 -6
View File
@@ -36,12 +36,14 @@ static struct ggml_tensor * qwen_load_ctw_f32(WeightCtx * wctx, const GGUFModel
fprintf(stderr, "[CausalTransConv] FATAL: tensor '%s' not found\n", name.c_str());
exit(1);
}
// Source dtype is F32 in the F32 master, F16 in the quantized variants
// since 3D conv weights cannot be Q8_0 / Q4_K_M and ggml falls back to
// F16 in the quantizer. Both paths cast to F32 here ; the K*OC*IC
// Source dtype follows the GGUF norm (pure llama.cpp policy). The F32
// master keeps tensors in F32, the BF16 variant keeps them in their
// source BF16, and the K-quant variants land them in F16 through the
// aligned fallback (kernel rows of width K=2 do not divide a K-quant
// block size). All three are widened to F32 here ; the K*OC*IC
// permutation always lands in a freshly allocated F32 buffer anyway.
if (src->type != GGML_TYPE_F32 && src->type != GGML_TYPE_F16) {
fprintf(stderr, "[CausalTransConv] FATAL: '%s' expected F32 or F16, got type %d\n", name.c_str(),
if (src->type != GGML_TYPE_F32 && src->type != GGML_TYPE_F16 && src->type != GGML_TYPE_BF16) {
fprintf(stderr, "[CausalTransConv] FATAL: '%s' expected F32, F16 or BF16, got type %d\n", name.c_str(),
(int) src->type);
exit(1);
}
@@ -60,7 +62,10 @@ static struct ggml_tensor * qwen_load_ctw_f32(WeightCtx * wctx, const GGUFModel
if (src->type == GGML_TYPE_F32) {
return ((const float *) raw)[idx];
}
return ggml_fp16_to_fp32(((const ggml_fp16_t *) raw)[idx]);
if (src->type == GGML_TYPE_F16) {
return ggml_fp16_to_fp32(((const ggml_fp16_t *) raw)[idx]);
}
return ggml_bf16_to_fp32(((const ggml_bf16_t *) raw)[idx]);
};
for (int ic = 0; ic < IC; ic++) {
src/convnext-block.h
+1 -1
View File
@@ -77,7 +77,7 @@ static bool qwen_upsample_stage_load(QwenUpsampleStage * stage, const GGUFModel
QwenConvNeXtBlock & cn = stage->convnext[i];
snprintf(name, sizeof(name), "tok_dec.upsample.%d.dwconv.weight", i);
cn.dwconv_w = gf_load_tensor(&wctx, gf, name);
cn.dwconv_w = gf_load_conv(&wctx, gf, name);
snprintf(name, sizeof(name), "tok_dec.upsample.%d.dwconv.bias", i);
cn.dwconv_b = gf_load_tensor(&wctx, gf, name);
snprintf(name, sizeof(name), "tok_dec.upsample.%d.norm.weight", i);
src/dac-decoder-v2.h
+4 -4
View File
@@ -150,7 +150,7 @@ static bool qwen_dac_decoder_load(QwenDACDecoder * d, const GGUFModel & gf, ggml
WeightCtx wctx;
wctx_init(&wctx, n_tensors);
d->conv_pre_w = gf_load_tensor(&wctx, gf, "tok_dec.dec.0.conv.weight");
d->conv_pre_w = gf_load_conv(&wctx, gf, "tok_dec.dec.0.conv.weight");
d->conv_pre_b = gf_load_tensor(&wctx, gf, "tok_dec.dec.0.conv.bias");
for (int i = 0; i < QWEN_DAC_NUM_BLOCKS; i++) {
@@ -182,17 +182,17 @@ static bool qwen_dac_decoder_load(QwenDACDecoder * d, const GGUFModel & gf, ggml
qwen_dac_load_snakebeta(&wctx, gf, &ru.act1, std::string(rp) + ".act1.alpha",
std::string(rp) + ".act1.beta");
ru.c1w = gf_load_tensor(&wctx, gf, std::string(rp) + ".conv1.weight");
ru.c1w = gf_load_conv(&wctx, gf, std::string(rp) + ".conv1.weight");
ru.c1b = gf_load_tensor(&wctx, gf, std::string(rp) + ".conv1.bias");
qwen_dac_load_snakebeta(&wctx, gf, &ru.act2, std::string(rp) + ".act2.alpha",
std::string(rp) + ".act2.beta");
ru.c2w = gf_load_tensor(&wctx, gf, std::string(rp) + ".conv2.weight");
ru.c2w = gf_load_conv(&wctx, gf, std::string(rp) + ".conv2.weight");
ru.c2b = gf_load_tensor(&wctx, gf, std::string(rp) + ".conv2.bias");
}
}
qwen_dac_load_snakebeta(&wctx, gf, &d->snake_post, "tok_dec.dec.5.snake.alpha", "tok_dec.dec.5.snake.beta");
d->conv_post_w = gf_load_tensor(&wctx, gf, "tok_dec.dec.6.conv.weight");
d->conv_post_w = gf_load_conv(&wctx, gf, "tok_dec.dec.6.conv.weight");
d->conv_post_b = gf_load_tensor(&wctx, gf, "tok_dec.dec.6.conv.bias");
if (!wctx_alloc(&wctx, backend)) {
src/encoder-downsample.h
+1 -1
View File
@@ -30,7 +30,7 @@ static bool qwen_encoder_downsample_load(QwenEncoderDownsample * d, const GGUFMo
WeightCtx wctx;
wctx_init(&wctx, 4);
d->weight = gf_load_tensor(&wctx, gf, "tok_enc.downsample.weight");
d->weight = gf_load_conv(&wctx, gf, "tok_enc.downsample.weight");
if (!wctx_alloc(&wctx, backend)) {
fprintf(stderr, "[EncDownsample] FATAL: backend allocation failed\n");
return false;
src/gguf-weights.h
+85 -47
View File
@@ -266,6 +266,91 @@ static struct ggml_tensor * gf_load_tensor_f32(WeightCtx * wctx, const GGUFModel
return tensor;
}
// Load a Conv1d / Conv1dDW kernel weight, forcing F16 storage on the
// backend regardless of the source GGUF dtype.
//
// TODO upstream GGML : ggml_conv_1d and ggml_conv_1d_dw in
// ggml/src/ggml.c hardcode dst_type = GGML_TYPE_F16 in their internal
// ggml_im2col call (currently ggml.c lines around 4508 and 4542).
// ggml_conv_2d at the equivalent site uses the adaptive pattern
// dst_type = a->type (around line 4595). Two backend bugs follow.
//
// 1) CPU im2col dispatches on dst->type. The im2col_f16 path
// asserts src0->type == GGML_TYPE_F16, so a F32 or BF16 conv
// kernel crashes on CPU.
// 2) ggml_conv_1d lowers to ggml_mul_mat(im2col, reshape(weight))
// where the weight ends up as src1, not src0. The Vulkan fast
// path ggml_vk_get_dequantize_mul_mat_vec asserts b_type in
// {F32, F16, Q8_1}, so a BF16 conv kernel crashes on Vulkan
// even when im2col itself succeeds.
//
// Vulkan does ship a pipeline_im2col_f32 and CUDA handles both F32
// and F16 cleanly, so the fix upstream is to align ggml_conv_1d and
// ggml_conv_1d_dw on ggml_conv_2d's adaptive a->type pattern. Until
// that lands, the only safe assumption across CPU, CUDA, and Vulkan
// is F16 kernels everywhere, so we mirror ggml_conv_1d's hardcoded
// choice here and load every conv kernel as F16 regardless of source
// dtype. F16 source is a direct passthrough, F32 and BF16 widen
// through a F32 staging buffer.
static struct ggml_tensor * gf_load_conv(WeightCtx * wctx, const GGUFModel & gf, const std::string & name) {
int64_t idx = gguf_find_tensor(gf.gguf, name.c_str());
if (idx < 0) {
fprintf(stderr, "[GGUF] FATAL: tensor '%s' not found\n", name.c_str());
exit(1);
}
struct ggml_tensor * src = ggml_get_tensor(gf.meta, name.c_str());
int n_dims = ggml_n_dims(src);
int64_t ne[4] = { 1, 1, 1, 1 };
for (int i = 0; i < n_dims; i++) {
ne[i] = src->ne[i];
}
// F16 source : direct passthrough, no conversion.
if (src->type == GGML_TYPE_F16) {
return gf_load_tensor(wctx, gf, name);
}
if (src->type != GGML_TYPE_F32 && src->type != GGML_TYPE_BF16) {
fprintf(stderr, "[GGUF] FATAL: gf_load_conv unsupported source type %s for '%s'\n", ggml_type_name(src->type),
name.c_str());
exit(1);
}
// Allocate F16 backend tensor in the WeightCtx graph.
struct ggml_tensor * tensor = ggml_new_tensor(wctx->ctx, GGML_TYPE_F16, n_dims, ne);
ggml_set_name(tensor, name.c_str());
size_t n = (size_t) ggml_nelements(src);
size_t raw_off = gguf_get_tensor_offset(gf.gguf, idx);
const void * raw = gf.mapping + gf.data_offset + raw_off;
// The staging vector owns float[] buffers to keep memory alive
// until wctx_alloc copies it to the backend. n F16 elements
// occupy n * 2 bytes, which fits in (n + 1) / 2 floats. The
// pending entry references the same buffer reinterpreted as
// ggml_fp16_t and carries the exact F16 byte count.
size_t n_floats = (n + 1) / 2;
auto buf = std::make_unique<float[]>(n_floats);
ggml_fp16_t * data = (ggml_fp16_t *) buf.get();
if (src->type == GGML_TYPE_F32) {
ggml_fp32_to_fp16_row((const float *) raw, data, (int) n);
} else {
// BF16 source : widen to F32 first, then narrow to F16 in
// one pass to preserve mantissa bits the BF16-to-F16 direct
// cast would otherwise leave undefined.
std::vector<float> f32(n);
const uint16_t * p = (const uint16_t *) raw;
for (size_t i = 0; i < n; i++) {
f32[i] = ggml_bf16_to_fp32(*(const ggml_bf16_t *) &p[i]);
}
ggml_fp32_to_fp16_row(f32.data(), data, (int) n);
}
wctx->pending.push_back({ tensor, (const void *) data, n * sizeof(ggml_fp16_t), 0 });
wctx->staging.push_back(std::move(buf));
return tensor;
}
// Get raw pointer to tensor data in the mmapped file.
// Useful for CPU-side operations (e.g. bf16 embed lookup for lyrics).
// Returns NULL if not found.
@@ -290,53 +375,6 @@ static enum ggml_type gf_get_type(const GGUFModel & gf, const std::string & name
return src->type;
}
// Load a Conv1d weight onto an F16 backend tensor regardless of the source
// dtype. Mandatory on ARM aarch64 : the CPU im2col op asserts src0 is F16,
// while x86 silently accepts BF16 / F32. F16 source memcpy passes through ;
// F32 / BF16 widen ; Q8_0 / Q4_K / Q5_K / Q6_K dequantize via type traits.
// The destination tensor must be allocated as GGML_TYPE_F16.
static void gf_load_conv_f16(struct ggml_tensor * dst, const GGUFModel & gf, const std::string & name) {
struct ggml_tensor * src = ggml_get_tensor(gf.meta, name.c_str());
if (!src) {
fprintf(stderr, "[GGUF] FATAL: tensor '%s' not in meta context\n", name.c_str());
exit(1);
}
GGML_ASSERT(dst->type == GGML_TYPE_F16);
GGML_ASSERT(ggml_nelements(dst) == ggml_nelements(src));
const void * raw = gf_get_data(gf, name.c_str());
size_t n = (size_t) ggml_nelements(src);
// F16 source : direct memcpy, no conversion needed.
if (src->type == GGML_TYPE_F16) {
ggml_backend_tensor_set(dst, raw, 0, ggml_nbytes(dst));
return;
}
// All other types widen / dequantize to F32, then cast down to F16.
std::vector<float> f32(n);
if (src->type == GGML_TYPE_F32) {
memcpy(f32.data(), raw, n * sizeof(float));
} else if (src->type == GGML_TYPE_BF16) {
const uint16_t * p = (const uint16_t *) raw;
for (size_t i = 0; i < n; i++) {
f32[i] = ggml_bf16_to_fp32(*(const ggml_bf16_t *) &p[i]);
}
} else {
const struct ggml_type_traits * tr = ggml_get_type_traits(src->type);
if (!tr || !tr->to_float) {
fprintf(stderr, "[GGUF] FATAL: unsupported conv weight type %s for '%s'\n", ggml_type_name(src->type),
name.c_str());
exit(1);
}
tr->to_float(raw, f32.data(), (int64_t) n);
}
std::vector<ggml_fp16_t> f16(n);
ggml_fp32_to_fp16_row(f32.data(), f16.data(), (int) n);
ggml_backend_tensor_set(dst, f16.data(), 0, n * sizeof(ggml_fp16_t));
}
// Fuse Q, K, V projection weights into a single tensor [ne0, q_ne1 + k_ne1 + v_ne1].
// Works for any quantized type since quantization is per-row (along ne[0]).
// The fused tensor data is q rows || k rows || v rows (contiguous).
src/pipeline-codec.cpp
+1 -1
View File
@@ -57,7 +57,7 @@ bool pipeline_codec_load(PipelineCodec * pc, const char * gguf_path, BackendPair
{
WeightCtx wctx;
wctx_init(&wctx, 4);
pc->pre_conv_w = gf_load_tensor(&wctx, pc->gguf, "tok_dec.pre_conv.weight");
pc->pre_conv_w = gf_load_conv(&wctx, pc->gguf, "tok_dec.pre_conv.weight");
pc->pre_conv_b = gf_load_tensor(&wctx, pc->gguf, "tok_dec.pre_conv.bias");
if (!wctx_alloc(&wctx, pc->backend)) {
qt_log(QT_LOG_ERROR, "[Pipeline] pre_conv backend allocation failed");
src/seanet-encoder.h
+5 -5
View File
@@ -107,7 +107,7 @@ static bool qwen_seanet_encoder_load(QwenSEANetEncoder * s, const GGUFModel & gf
WeightCtx wctx;
wctx_init(&wctx, n_tensors);
s->init_w = gf_load_tensor(&wctx, gf, "tok_enc.conv.0.weight");
s->init_w = gf_load_conv(&wctx, gf, "tok_enc.conv.0.weight");
s->init_b = gf_load_tensor(&wctx, gf, "tok_enc.conv.0.bias");
// Stage indexing follows the Python ModuleList layout :
@@ -125,23 +125,23 @@ static bool qwen_seanet_encoder_load(QwenSEANetEncoder * s, const GGUFModel & gf
char name[80];
snprintf(name, sizeof(name), "tok_enc.res.%d.blk.1.weight", RES_PY_IDX[i]);
stg.resnet.c0_w = gf_load_tensor(&wctx, gf, name);
stg.resnet.c0_w = gf_load_conv(&wctx, gf, name);
snprintf(name, sizeof(name), "tok_enc.res.%d.blk.1.bias", RES_PY_IDX[i]);
stg.resnet.c0_b = gf_load_tensor(&wctx, gf, name);
snprintf(name, sizeof(name), "tok_enc.res.%d.blk.3.weight", RES_PY_IDX[i]);
stg.resnet.c1_w = gf_load_tensor(&wctx, gf, name);
stg.resnet.c1_w = gf_load_conv(&wctx, gf, name);
snprintf(name, sizeof(name), "tok_enc.res.%d.blk.3.bias", RES_PY_IDX[i]);
stg.resnet.c1_b = gf_load_tensor(&wctx, gf, name);
snprintf(name, sizeof(name), "tok_enc.conv.%d.weight", DOWN_PY_IDX[i]);
stg.down_w = gf_load_tensor(&wctx, gf, name);
stg.down_w = gf_load_conv(&wctx, gf, name);
snprintf(name, sizeof(name), "tok_enc.conv.%d.bias", DOWN_PY_IDX[i]);
stg.down_b = gf_load_tensor(&wctx, gf, name);
dim = stg.out_ch;
}
s->last_w = gf_load_tensor(&wctx, gf, "tok_enc.conv.14.weight");
s->last_w = gf_load_conv(&wctx, gf, "tok_enc.conv.14.weight");
s->last_b = gf_load_tensor(&wctx, gf, "tok_enc.conv.14.bias");
if (!wctx_alloc(&wctx, backend)) {