fix(models): detect nvfp4 experts behind a mixed-precision compressed-tensors format (#343)

python/freetoken/models/config.py

@@ -21,7 +21,8 @@ def vision_load_enabled() -> bool:
 def detect_expert_quant(hf_config: Any) -> str:
     """Routed-expert quantization from a checkpoint's ``quantization_config``: ``"nvfp4"`` for
     a ModelOpt FP4 build (``quant_algo: NVFP4``) OR an llm-compressor NVFP4 export
-    (``quant_method: compressed-tensors`` + ``format: nvfp4-pack-quantized``, e.g.
+    (``quant_method: compressed-tensors`` + ``format: nvfp4-pack-quantized``, or
+    ``format: mixed-precision`` with an nvfp4 config group, e.g.
     RedHatAI/GLM-5.3-Flash-NVFP4), else the lowercased algo string (``"none"`` when
     unquantized). Models with mixed-precision configs (e.g. qwen3_5_moe) need their
     own detector."""
@@ -34,9 +35,19 @@ def detect_expert_quant(hf_config: Any) -> str:
         return "none"
     if "fp4" in str(algo).lower():
         return "nvfp4"
+    fmt = str(get("format") or "").lower()
     # exact "nvfp4" (not the "fp4" substring) so MXFP4 exports don't misroute
-    if "nvfp4" in str(get("format") or "").lower():
+    if "nvfp4" in fmt:
         return "nvfp4"
+    # llm-compressor writes "mixed-precision" at the top when the groups differ (GLM-5.3-Flash: nvfp4 routed experts, fp8 MTP experts); the real format then sits in each group
+    if fmt == "mixed-precision":
+        groups = get("config_groups") or {}
+        groups = [g or {} for g in (groups.values() if isinstance(groups, dict) else [])]
+        # groups that target the experts decide; only a generic ["Linear"] group falls back to all of them
+        expert_groups = [g for g in groups if any("experts" in str(t) for t in (g.get("targets") or []))]
+        for g in expert_groups or groups:
+            if "nvfp4" in str(g.get("format") or "").lower():
+                return "nvfp4"
     return str(algo).lower()
 
 

tests/models/test_glm5_next_config.py

@@ -121,6 +121,24 @@ _CT_NVFP4_QUANT = {
     },
 }
 
+_CT_MIXED_QUANT = {
+    # From RedHatAI/GLM-5.3-Flash-NVFP4 as published: nvfp4 routed experts plus fp8 experts on the MTP layer, so the top-level format is "mixed-precision".
+    "quant_method": "compressed-tensors",
+    "format": "mixed-precision",
+    "config_groups": {
+        "group_0": {
+            "targets": ["re:.*\\.layers\\.(?:[3-9]|[1-3][0-9]|4[0-4])\\.mlp\\.experts\\..*(gate|up|down)_proj$"],
+            "weights": {"num_bits": 4, "type": "float", "group_size": 16, "strategy": "tensor_group"},
+            "format": "nvfp4-pack-quantized",
+        },
+        "group_1": {
+            "targets": ["re:.*\\.layers\\.45\\.mlp\\.experts\\.\\d+\\.(gate_proj|up_proj|down_proj)$"],
+            "weights": {"num_bits": 8, "type": "float", "strategy": "block"},
+            "format": "float-quantized",
+        },
+    },
+}
+
 _NVFP4_QUANT = {
     # From LibertAIDAI/GLM-5.3-Flash-NVFP4 (ModelOpt weight-only NVFP4).
     "quant_algo": "NVFP4",
@@ -246,6 +264,37 @@ def test_compressed_tensors_nvfp4_detected():
     assert cfg.expert_quant == "nvfp4"
 
 
+def test_compressed_tensors_mixed_precision_detected():
+    """The published RedHatAI export says ``format: mixed-precision`` at the top and
+    ``nvfp4-pack-quantized`` only inside the routed-expert group; expert_quant must
+    still resolve to nvfp4, not to the raw quant_method."""
+    cfg = parse_config(_hf_config(_CT_MIXED_QUANT))
+    assert cfg.expert_quant == "nvfp4"
+
+
+def test_compressed_tensors_mixed_precision_reads_the_expert_group():
+    """A mixed export with nvfp4 dense layers but fp8 experts must not report nvfp4
+    experts: the group that targets the experts decides."""
+    quant = {
+        "quant_method": "compressed-tensors",
+        "format": "mixed-precision",
+        "config_groups": {
+            "group_0": {
+                "targets": ["re:.*self_attn.*_proj$"],
+                "weights": {"num_bits": 4, "type": "float", "group_size": 16, "strategy": "tensor_group"},
+                "format": "nvfp4-pack-quantized",
+            },
+            "group_1": {
+                "targets": ["re:.*mlp\\.experts\\..*(gate|up|down)_proj$"],
+                "weights": {"num_bits": 8, "type": "float", "strategy": "block"},
+                "format": "float-quantized",
+            },
+        },
+    }
+    cfg = parse_config(_hf_config(quant))
+    assert cfg.expert_quant == "compressed-tensors"
+
+
 def test_expert_source_spec_selection():
     """quant_method picks the bank source spec: compressed-tensors maps
     weight_packed/weight_global_scale onto the canonical kinds with a reciprocal