vllm.config.engram

Classes:

• EngramConfig –

  Configuration for Engram embedding storage and sharding.

Functions:

• model_has_engram_layers –

  Whether the model carries n-gram embedding layers.

EngramConfig

Configuration for Engram embedding storage and sharding.

Methods:

• compute_hash –

  Hash settings that affect embedding execution and graph structure.

• get_parallel_size –

  Derive the embedding group size from the parallel configuration.

• resolve_dp_shared_memory –

  Share host tables by default wherever the configuration permits.

• verify_model_config –

  Reject Engram configuration for models without n-gram embeddings.

• verify_parallel_config –

  Reject unsupported embedding parallel topologies.

Attributes:

• cpu_offload (bool) –

  Store embedding weights in pinned CPU memory for UVA lookup.

• disk_offload (bool) –

  Keep Qwen4Exp PLE tables in the checkpoint files and read each step's

• dp_shared_memory (bool | None) –

  Share CPU-offloaded embedding weights between co-located

• embedding_across_dp (bool) –

  Shard embeddings across TP and all DP ranks when enabled.

• use_thp (bool) –

  Back private CPU-offloaded tables with transparent huge pages (best

Source code in vllm/config/engram.py

@config
class EngramConfig:
    """Configuration for Engram embedding storage and sharding."""

    cpu_offload: bool = True
    """Store embedding weights in pinned CPU memory for UVA lookup."""

    disk_offload: bool = False
    """Keep Qwen4Exp PLE tables in the checkpoint files and read each step's
    rows on the host instead of allocating the table. Takes precedence over
    cpu_offload."""

    embedding_across_dp: bool = False
    """Shard embeddings across TP and all DP ranks when enabled.
    Otherwise, each DP rank has a separate TP-sharded embedding replica."""

    dp_shared_memory: bool | None = None
    """Share CPU-offloaded embedding weights between co-located
    DP replicas. Each node stores one copy of every TP shard, reducing host
    memory without per-step Engram DP collectives. Uses host-NUMA cuMem shared
    by FABRIC handle where supported, else /dev/shm (which then needs capacity
    and a shared IPC namespace). Defaults to enabled whenever the other settings
    allow it, falling back to per-replica tables when DP replicas are not
    co-located on one node or the host cannot hold them."""

    use_thp: bool = False
    """Back private CPU-offloaded tables with transparent huge pages (best
    effort, falls back to ordinary pinned pages). Prefaulting the tables at
    startup takes longer. Requires cpu_offload without dp_shared_memory.
    Always on for private tables on ROCm."""

    @model_validator(mode="after")
    def _validate_shared_memory(self) -> Self:
        if self.dp_shared_memory and not self.cpu_offload:
            raise ValueError("dp_shared_memory requires cpu_offload=True")
        if self.use_thp and (not self.cpu_offload or self.dp_shared_memory):
            raise ValueError(
                "use_thp requires cpu_offload=True and dp_shared_memory=False"
            )
        return self

    def verify_model_config(self, model_config: "ModelConfig | None") -> None:
        """Reject Engram configuration for models without n-gram embeddings."""
        field = (
            _NGRAM_LAYER_FIELDS.get(model_config.architecture)
            if model_config is not None
            else None
        )
        if (
            model_config is None
            or field is None
            or not getattr(model_config.hf_text_config, field, None)
        ):
            raise ValueError(
                "EngramConfig requires a model with supported Engram "
                "embeddings and non-empty n-gram layer ids."
            )
        if self.disk_offload:
            from vllm.platforms import current_platform

            if field != "ple_layer_ids" or not current_platform.is_cuda():
                raise ValueError("disk_offload requires a Qwen4Exp model on CUDA")

    def resolve_dp_shared_memory(self, parallel_config: "ParallelConfig") -> None:
        """Share host tables by default wherever the configuration permits."""
        if self.dp_shared_memory is None:
            self.dp_shared_memory = (
                self.cpu_offload
                and not self.use_thp
                and parallel_config.data_parallel_size > 1
                and not parallel_config.enable_elastic_ep
            )

    def verify_parallel_config(self, parallel_config: "ParallelConfig") -> None:
        """Reject unsupported embedding parallel topologies."""
        if self.dp_shared_memory:
            if parallel_config.data_parallel_size <= 1:
                raise ValueError("dp_shared_memory requires data_parallel_size > 1.")
            if parallel_config.enable_elastic_ep:
                raise ValueError("dp_shared_memory is not supported with elastic EP.")
        if (
            self.embedding_across_dp
            and parallel_config.data_parallel_size > 1
            and parallel_config.enable_elastic_ep
        ):
            raise ValueError(
                "Engram embedding_across_dp is not supported with elastic EP yet."
            )
        if self.disk_offload and parallel_config.enable_elastic_ep:
            raise ValueError("disk_offload is not supported with elastic EP.")

    def get_parallel_size(self, parallel_config: "ParallelConfig") -> int:
        """Derive the embedding group size from the parallel configuration."""
        size = parallel_config.tensor_parallel_size
        if self.embedding_across_dp and parallel_config.data_parallel_size > 1:
            size *= parallel_config.data_parallel_size
        return size

    def compute_hash(self) -> str:
        """Hash settings that affect embedding execution and graph structure."""
        return hash_factors(get_hash_factors(self, set()))

cpu_offload = True class-attribute instance-attribute

Store embedding weights in pinned CPU memory for UVA lookup.

disk_offload = False class-attribute instance-attribute

Keep Qwen4Exp PLE tables in the checkpoint files and read each step's rows on the host instead of allocating the table. Takes precedence over cpu_offload.

dp_shared_memory = None class-attribute instance-attribute

Share CPU-offloaded embedding weights between co-located DP replicas. Each node stores one copy of every TP shard, reducing host memory without per-step Engram DP collectives. Uses host-NUMA cuMem shared by FABRIC handle where supported, else /dev/shm (which then needs capacity and a shared IPC namespace). Defaults to enabled whenever the other settings allow it, falling back to per-replica tables when DP replicas are not co-located on one node or the host cannot hold them.

embedding_across_dp = False class-attribute instance-attribute

Shard embeddings across TP and all DP ranks when enabled. Otherwise, each DP rank has a separate TP-sharded embedding replica.

use_thp = False class-attribute instance-attribute

Back private CPU-offloaded tables with transparent huge pages (best effort, falls back to ordinary pinned pages). Prefaulting the tables at startup takes longer. Requires cpu_offload without dp_shared_memory. Always on for private tables on ROCm.

compute_hash()

Hash settings that affect embedding execution and graph structure.

Source code in vllm/config/engram.py

def compute_hash(self) -> str:
    """Hash settings that affect embedding execution and graph structure."""
    return hash_factors(get_hash_factors(self, set()))

get_parallel_size(parallel_config)

Derive the embedding group size from the parallel configuration.

Source code in vllm/config/engram.py

def get_parallel_size(self, parallel_config: "ParallelConfig") -> int:
    """Derive the embedding group size from the parallel configuration."""
    size = parallel_config.tensor_parallel_size
    if self.embedding_across_dp and parallel_config.data_parallel_size > 1:
        size *= parallel_config.data_parallel_size
    return size

resolve_dp_shared_memory(parallel_config)

Share host tables by default wherever the configuration permits.

Source code in vllm/config/engram.py

def resolve_dp_shared_memory(self, parallel_config: "ParallelConfig") -> None:
    """Share host tables by default wherever the configuration permits."""
    if self.dp_shared_memory is None:
        self.dp_shared_memory = (
            self.cpu_offload
            and not self.use_thp
            and parallel_config.data_parallel_size > 1
            and not parallel_config.enable_elastic_ep
        )

verify_model_config(model_config)

Reject Engram configuration for models without n-gram embeddings.

Source code in vllm/config/engram.py

def verify_model_config(self, model_config: "ModelConfig | None") -> None:
    """Reject Engram configuration for models without n-gram embeddings."""
    field = (
        _NGRAM_LAYER_FIELDS.get(model_config.architecture)
        if model_config is not None
        else None
    )
    if (
        model_config is None
        or field is None
        or not getattr(model_config.hf_text_config, field, None)
    ):
        raise ValueError(
            "EngramConfig requires a model with supported Engram "
            "embeddings and non-empty n-gram layer ids."
        )
    if self.disk_offload:
        from vllm.platforms import current_platform

        if field != "ple_layer_ids" or not current_platform.is_cuda():
            raise ValueError("disk_offload requires a Qwen4Exp model on CUDA")

verify_parallel_config(parallel_config)

Reject unsupported embedding parallel topologies.

Source code in vllm/config/engram.py

def verify_parallel_config(self, parallel_config: "ParallelConfig") -> None:
    """Reject unsupported embedding parallel topologies."""
    if self.dp_shared_memory:
        if parallel_config.data_parallel_size <= 1:
            raise ValueError("dp_shared_memory requires data_parallel_size > 1.")
        if parallel_config.enable_elastic_ep:
            raise ValueError("dp_shared_memory is not supported with elastic EP.")
    if (
        self.embedding_across_dp
        and parallel_config.data_parallel_size > 1
        and parallel_config.enable_elastic_ep
    ):
        raise ValueError(
            "Engram embedding_across_dp is not supported with elastic EP yet."
        )
    if self.disk_offload and parallel_config.enable_elastic_ep:
        raise ValueError("disk_offload is not supported with elastic EP.")

model_has_engram_layers(model_config)

Whether the model carries n-gram embedding layers.

Source code in vllm/config/engram.py

def model_has_engram_layers(model_config: "ModelConfig | None") -> bool:
    """Whether the model carries n-gram embedding layers."""
    if model_config is None:
        return False
    field = _NGRAM_LAYER_FIELDS.get(model_config.architecture)
    if field is None:
        return False
    return bool(getattr(model_config.hf_text_config, field, None))