Models

A model in mteb covers two concepts: metadata and implementation. - Metadata contains information about the model such as maximum input length, valid frameworks, license, and degree of openness. - Implementation is a reproducible workflow, which allows others to run the same model again, using the same prompts, hyperparameters, aggregation strategies, etc.

An overview of the model and its metadata within mteb

Utilities

mteb.get_model_metas(model_names=None, languages=None, open_weights=None, frameworks=None, n_parameters_range=(None, None), use_instructions=None, zero_shot_on=None, model_types=None)

Load all models' metadata that fit the specified criteria.

Parameters:

Name	Type	Description	Default
model_names	Iterable[str] | None	A list of model names to filter by. If None, all models are included.	None
languages	Iterable[str] | None	A list of languages to filter by. If None, all languages are included.	None
open_weights	bool | None	Whether to filter by models with open weights. If None this filter is ignored.	None
frameworks	Iterable[str] | None	A list of frameworks to filter by. If None, all frameworks are included.	None
n_parameters_range	tuple[int | None, int | None]	A tuple of lower and upper bounds of the number of parameters to filter by. If (None, None), this filter is ignored.	(None, None)
use_instructions	bool | None	Whether to filter by models that use instructions. If None, all models are included.	None
zero_shot_on	list[AbsTask] | None	A list of tasks on which the model is zero-shot. If None this filter is ignored.	None
model_types	Iterable[str] | None	A list of model types to filter by. If None, all model types are included.	None

Returns:

Type	Description
list[ModelMeta]	A list of model metadata objects that fit the specified criteria.

Source code in mteb/models/get_model_meta.py

def get_model_metas(
    model_names: Iterable[str] | None = None,
    languages: Iterable[str] | None = None,
    open_weights: bool | None = None,
    frameworks: Iterable[str] | None = None,
    n_parameters_range: tuple[int | None, int | None] = (None, None),
    use_instructions: bool | None = None,
    zero_shot_on: list[AbsTask] | None = None,
    model_types: Iterable[str] | None = None,
) -> list[ModelMeta]:
    """Load all models' metadata that fit the specified criteria.

    Args:
        model_names: A list of model names to filter by. If None, all models are included.
        languages: A list of languages to filter by. If None, all languages are included.
        open_weights: Whether to filter by models with open weights. If None this filter is ignored.
        frameworks: A list of frameworks to filter by. If None, all frameworks are included.
        n_parameters_range: A tuple of lower and upper bounds of the number of parameters to filter by.
            If (None, None), this filter is ignored.
        use_instructions: Whether to filter by models that use instructions. If None, all models are included.
        zero_shot_on: A list of tasks on which the model is zero-shot. If None this filter is ignored.
        model_types: A list of model types to filter by. If None, all model types are included.

    Returns:
        A list of model metadata objects that fit the specified criteria.
    """
    res = []
    model_names = set(model_names) if model_names is not None else None
    languages = set(languages) if languages is not None else None
    frameworks = set(frameworks) if frameworks is not None else None
    model_types_set = set(model_types) if model_types is not None else None
    for model_meta in MODEL_REGISTRY.values():
        if (model_names is not None) and (model_meta.name not in model_names):
            continue
        if languages is not None:
            if (model_meta.languages is None) or not (
                languages <= set(model_meta.languages)
            ):
                continue
        if (open_weights is not None) and (model_meta.open_weights != open_weights):
            continue
        if (frameworks is not None) and not (frameworks <= set(model_meta.framework)):
            continue
        if (use_instructions is not None) and (
            model_meta.use_instructions != use_instructions
        ):
            continue
        if model_types_set is not None and not model_types_set.intersection(
            model_meta.model_type
        ):
            continue

        lower, upper = n_parameters_range
        n_parameters = model_meta.n_parameters

        if upper is not None:
            if (n_parameters is None) or (n_parameters > upper):
                continue
            if lower is not None and n_parameters < lower:
                continue

        if zero_shot_on is not None:
            if not model_meta.is_zero_shot_on(zero_shot_on):
                continue
        res.append(model_meta)
    return res

mteb.get_model_meta(model_name, revision=None, fetch_from_hf=True, fill_missing=False)

A function to fetch a model metadata object by name.

Parameters:

Name	Type	Description	Default
model_name	str	Name of the model to fetch	required
revision	str | None	Revision of the model to fetch	None
fetch_from_hf	bool	Whether to fetch the model from HuggingFace Hub if not found in the registry	True
fill_missing	bool	Fill missing attributes from the metadata including number of parameters and memory usage.	False

Returns:

Type	Description
ModelMeta	A model metadata object

Source code in mteb/models/get_model_meta.py

def get_model_meta(
    model_name: str,
    revision: str | None = None,
    fetch_from_hf: bool = True,
    fill_missing: bool = False,
) -> ModelMeta:
    """A function to fetch a model metadata object by name.

    Args:
        model_name: Name of the model to fetch
        revision: Revision of the model to fetch
        fetch_from_hf: Whether to fetch the model from HuggingFace Hub if not found in the registry
        fill_missing: Fill missing attributes from the metadata including number of parameters and memory usage.

    Returns:
        A model metadata object
    """
    if model_name in _MODEL_RENAMES:
        new_name = _MODEL_RENAMES[model_name]
        msg = f"The model '{model_name}' has been renamed to '{new_name}'. To prevent this warning use the new name."
        warnings.warn(msg, DeprecationWarning, stacklevel=2)
        model_name = new_name

    if model_name in MODEL_REGISTRY:
        model_meta = MODEL_REGISTRY[model_name]

        if revision and (not model_meta.revision == revision):
            raise ValueError(
                f"Model revision {revision} not found for model {model_name}. Expected {model_meta.revision}."
            )

        if fill_missing and fetch_from_hf:
            original_meta_dict = model_meta.model_dump()
            new_meta = ModelMeta.from_hub(model_name, fill_missing=fill_missing)
            new_meta_dict = new_meta.model_dump(exclude_none=True)

            updates = {
                k: v
                for k, v in new_meta_dict.items()
                if original_meta_dict.get(k) is None
            }

            if updates:
                return model_meta.model_copy(update=updates)
        return model_meta

    if fetch_from_hf:
        logger.info(
            f"Model not found in model registry. Attempting to extract metadata by loading the model ({model_name}) using HuggingFace."
        )
        meta = ModelMeta.from_hub(model_name, revision)
        return meta

    not_found_msg = f"Model '{model_name}' not found in MTEB registry"
    not_found_msg += " nor on the Huggingface Hub." if fetch_from_hf else "."

    close_matches = difflib.get_close_matches(model_name, MODEL_REGISTRY.keys())
    model_names_no_org = {mdl: mdl.split("/")[-1] for mdl in MODEL_REGISTRY.keys()}
    if model_name in model_names_no_org:
        close_matches = [model_names_no_org[model_name]] + close_matches

    suggestion = ""
    if close_matches:
        if len(close_matches) > 1:
            suggestion = f" Did you mean: '{close_matches[0]}' or {close_matches[1]}?"
        else:
            suggestion = f" Did you mean: '{close_matches[0]}'?"

    raise KeyError(not_found_msg + suggestion)

mteb.get_model(model_name, revision=None, device=None, **kwargs)

A function to fetch and load model object by name.

Note

This function loads the model into memory. If you only want to fetch the metadata, use get_model_meta instead.

Parameters:

Name	Type	Description	Default
model_name	str	Name of the model to fetch	required
revision	str | None	Revision of the model to fetch	None
device	str | None	Device used to load the model	None
**kwargs	Any	Additional keyword arguments to pass to the model loader	{}

Returns:

Type	Description
MTEBModels	A model object

Source code in mteb/models/get_model_meta.py

def get_model(
    model_name: str,
    revision: str | None = None,
    device: str | None = None,
    **kwargs: Any,
) -> MTEBModels:
    """A function to fetch and load model object by name.

    !!! note
        This function loads the model into memory. If you only want to fetch the metadata, use [`get_model_meta`](#mteb.get_model_meta) instead.

    Args:
        model_name: Name of the model to fetch
        revision: Revision of the model to fetch
        device: Device used to load the model
        **kwargs: Additional keyword arguments to pass to the model loader

    Returns:
        A model object
    """
    meta = get_model_meta(model_name, revision)
    model = meta.load_model(device=device, **kwargs)

    if kwargs:
        logger.info(
            f"Model '{model_name}' loaded with additional arguments: {list(kwargs.keys())}"
        )
        meta = meta.model_copy(deep=True)
        meta.loader_kwargs |= kwargs

    model.mteb_model_meta = meta  # type: ignore[misc]
    return model

Metadata

mteb.models.model_meta.ModelMeta

Bases: BaseModel

The model metadata object.

Attributes:

Name	Type	Description
loader	Callable[..., MTEBModels] | None	The function that loads the model. If None it assumes that the model is not implemented.
loader_kwargs	dict[str, Any]	The keyword arguments to pass to the loader function.
name	str | None	The name of the model, ideally the name on huggingface. It should be in the format "organization/model_name".
n_parameters	int | None	The total number of parameters in the model, e.g. 7_000_000 for a 7M parameter model. Can be none in case the number of parameters is unknown.
n_embedding_parameters	int | None	The number of parameters used for the embedding layer. Can be None if the number of embedding parameters is not known (e.g. for proprietary models).
n_active_parameters_override	int | None	The number of active parameters used bu model. Should be used only for Mixture of Experts models.
memory_usage_mb	float | None	The memory usage of the model in MB. Can be None if the memory usage is not known (e.g. for proprietary models). To calculate it use the calculate_memory_usage_mb method.
max_tokens	float | None	The maximum number of tokens the model can handle. Can be None if the maximum number of tokens is not known (e.g. for proprietary models).
embed_dim	int | None	The dimension of the embeddings produced by the model. Currently all models are assumed to produce fixed-size embeddings.
revision	str | None	The revision number of the model. If None, it is assumed that the metadata (including the loader) is valid for all revisions of the model.
release_date	StrDate | None	The date the model's revision was released. If None, then release date will be added based on 1st commit in hf repository of model.
license	Licenses | StrURL | None	The license under which the model is released. Required if open_weights is True.
open_weights	bool | None	Whether the model is open source or proprietary.
public_training_code	str | None	A link to the publicly available training code. If None, it is assumed that the training code is not publicly available.
public_training_data	str | bool | None	A link to the publicly available training data. If None, it is assumed that the training data is not publicly available.
similarity_fn_name	ScoringFunction | None	The distance metric used by the model.
framework	list[FRAMEWORKS]	The framework the model is implemented in, can be a list of frameworks e.g. ["Sentence Transformers", "PyTorch"].
reference	StrURL | None	A URL to the model's page on huggingface or another source.
languages	list[ISOLanguageScript] | None	The languages the model is intended to be specified as a 3-letter language code followed by a script code e.g., "eng-Latn" for English in the Latin script.
use_instructions	bool | None	Whether the model uses instructions E.g. for prompt-based models. This also includes models that require a specific format for input, such as "query: {document}" or "passage: {document}".
citation	str | None	The citation for the model. This is a bibtex string.
training_datasets	set[str] | None	A dictionary of datasets that the model was trained on. Names should be names as their appear in mteb for example {"ArguAna"} if the model is trained on the ArguAna test set. This field is used to determine if a model generalizes zero-shot to a benchmark as well as mark dataset contaminations.
adapted_from	str | None	Name of the model from which this model is adapted. For quantizations, fine-tunes, long doc extensions, etc.
superseded_by	str | None	Name of the model that supersedes this model, e.g., nvidia/NV-Embed-v2 supersedes v1.
model_type	list[MODEL_TYPES]	A list of strings representing the type of model.
modalities	list[Modalities]	A list of strings representing the modalities the model supports. Default is ["text"].
contacts	list[str] | None	The people to contact in case of a problem in the model, preferably a GitHub handle.

Source code in mteb/models/model_meta.py

class ModelMeta(BaseModel):
    """The model metadata object.

    Attributes:
        loader: The function that loads the model. If None it assumes that the model is not implemented.
        loader_kwargs: The keyword arguments to pass to the loader function.
        name: The name of the model, ideally the name on huggingface. It should be in the format "organization/model_name".
        n_parameters: The total number of parameters in the model, e.g. `7_000_000` for a 7M parameter model. Can be none in case the number of parameters is unknown.
        n_embedding_parameters: The number of parameters used for the embedding layer. Can be None if the number of embedding parameters is not known (e.g. for proprietary models).
        n_active_parameters_override: The number of active parameters used bu model. Should be used **only** for Mixture of Experts models.
        memory_usage_mb: The memory usage of the model in MB. Can be None if the memory usage is not known (e.g. for proprietary models). To calculate it use the `calculate_memory_usage_mb` method.
        max_tokens: The maximum number of tokens the model can handle. Can be None if the maximum number of tokens is not known (e.g. for proprietary
            models).
        embed_dim: The dimension of the embeddings produced by the model. Currently all models are assumed to produce fixed-size embeddings.
        revision: The revision number of the model. If None, it is assumed that the metadata (including the loader) is valid for all revisions of the model.
        release_date: The date the model's revision was released. If None, then release date will be added based on 1st commit in hf repository of model.
        license: The license under which the model is released. Required if open_weights is True.
        open_weights: Whether the model is open source or proprietary.
        public_training_code: A link to the publicly available training code. If None, it is assumed that the training code is not publicly available.
        public_training_data: A link to the publicly available training data. If None, it is assumed that the training data is not publicly available.
        similarity_fn_name: The distance metric used by the model.
        framework: The framework the model is implemented in, can be a list of frameworks e.g. `["Sentence Transformers", "PyTorch"]`.
        reference: A URL to the model's page on huggingface or another source.
        languages: The languages the model is intended to be specified as a 3-letter language code followed by a script code e.g., "eng-Latn" for English
            in the Latin script.
        use_instructions: Whether the model uses instructions E.g. for prompt-based models. This also includes models that require a specific format for
            input, such as "query: {document}" or "passage: {document}".
        citation: The citation for the model. This is a bibtex string.
        training_datasets: A dictionary of datasets that the model was trained on. Names should be names as their appear in `mteb` for example
            {"ArguAna"} if the model is trained on the ArguAna test set. This field is used to determine if a model generalizes zero-shot to
            a benchmark as well as mark dataset contaminations.
        adapted_from: Name of the model from which this model is adapted. For quantizations, fine-tunes, long doc extensions, etc.
        superseded_by: Name of the model that supersedes this model, e.g., nvidia/NV-Embed-v2 supersedes v1.
        model_type: A list of strings representing the type of model.
        modalities: A list of strings representing the modalities the model supports. Default is ["text"].
        contacts: The people to contact in case of a problem in the model, preferably a GitHub handle.
    """

    model_config = ConfigDict(extra="forbid")

    # loaders
    loader: Callable[..., MTEBModels] | None
    loader_kwargs: dict[str, Any] = field(default_factory=dict)
    name: str | None
    revision: str | None
    release_date: StrDate | None
    languages: list[ISOLanguageScript] | None
    n_parameters: int | None
    n_active_parameters_override: int | None = None
    n_embedding_parameters: int | None = None
    memory_usage_mb: float | None
    max_tokens: float | None
    embed_dim: int | None
    license: Licenses | StrURL | None
    open_weights: bool | None
    public_training_code: str | None
    public_training_data: str | bool | None
    framework: list[FRAMEWORKS]
    reference: StrURL | None = None
    similarity_fn_name: ScoringFunction | None
    use_instructions: bool | None
    training_datasets: set[str] | None
    adapted_from: str | None = None
    superseded_by: str | None = None
    modalities: list[Modalities] = ["text"]
    model_type: list[MODEL_TYPES] = ["dense"]
    citation: str | None = None
    contacts: list[str] | None = None

    @model_validator(mode="before")
    @classmethod
    def _handle_legacy_is_cross_encoder(cls, data: Any) -> Any:
        """Handle legacy is_cross_encoder field by converting it to model_type.

        This validator handles backward compatibility for the deprecated is_cross_encoder field.
        If is_cross_encoder=True is provided, it adds "cross_encoder" to model_type.
        """
        if isinstance(data, dict) and "is_cross_encoder" in data:
            is_cross_encoder_value = data.pop("is_cross_encoder")

            if is_cross_encoder_value is not None:
                warnings.warn(
                    "is_cross_encoder is deprecated and will be removed in a future version. "
                    "Use model_type=['cross-encoder'] instead.",
                    DeprecationWarning,
                    stacklevel=2,
                )

                model_type = data.get("model_type", ["dense"])

                if is_cross_encoder_value:
                    if "cross-encoder" not in model_type:
                        data["model_type"] = ["cross-encoder"]
                else:
                    if "cross-encoder" in model_type:
                        model_type = [t for t in model_type if t != "cross-encoder"]
                        data["model_type"] = model_type if model_type else ["dense"]

        return data

    @property
    def is_cross_encoder(self) -> bool:
        """Returns True if the model is a cross-encoder.

        Derived from model_type field. A model is considered a cross-encoder if "cross-encoder" is in its model_type list.
        """
        return "cross-encoder" in self.model_type

    @property
    def n_active_parameters(self):
        """Number of active parameters. Assumed to be `n_parameters - n_embedding_parameters`. Can be overwritten using `n_active_parameters_override` e.g. for MoE models."""
        if self.n_active_parameters_override is not None:
            return self.n_active_parameters_override

        if self.n_parameters is not None and self.n_embedding_parameters is not None:
            return self.n_parameters - self.n_embedding_parameters
        return None

    @field_validator("similarity_fn_name", mode="before")
    @classmethod
    def _validate_similarity_fn_name(cls, value: str) -> ScoringFunction | None:
        """Converts the similarity function name to the corresponding enum value.

        Sentence_transformers uses Literal['cosine', 'dot', 'euclidean', 'manhattan'],
        and pylate uses Literal['MaxSim']

        Args:
            value: The similarity function name as a string.

        Returns:
            The corresponding ScoringFunction enum value.
        """
        if type(value) is ScoringFunction or value is None:
            return value
        mapping = {
            "cosine": ScoringFunction.COSINE,
            "dot": ScoringFunction.DOT_PRODUCT,
            "MaxSim": ScoringFunction.MAX_SIM,
        }
        if value in mapping:
            return mapping[value]
        raise ValueError(f"Invalid similarity function name: {value}")

    def to_dict(self):
        """Returns a dictionary representation of the model metadata."""
        dict_repr = self.model_dump()
        loader = dict_repr.pop("loader", None)
        dict_repr["training_datasets"] = (
            list(dict_repr["training_datasets"])
            if isinstance(dict_repr["training_datasets"], set)
            else dict_repr["training_datasets"]
        )
        dict_repr["loader"] = _get_loader_name(loader)
        dict_repr["is_cross_encoder"] = self.is_cross_encoder
        return dict_repr

    @field_validator("languages")
    @classmethod
    def _languages_are_valid(
        cls, languages: list[ISOLanguageScript] | None
    ) -> list[ISOLanguageScript] | None:
        if languages is None:
            return None

        for code in languages:
            check_language_code(code)
        return languages

    @field_validator("name")
    @classmethod
    def _check_name(cls, v: str | None) -> str | None:
        if v is None:
            return v
        if "/" not in v:
            raise ValueError(
                "Model name must be in the format 'organization/model_name'"
            )
        return v

    def load_model(self, device: str | None = None, **kwargs: Any) -> MTEBModels:
        """Loads the model using the specified loader function."""
        if self.loader is None:
            raise NotImplementedError(
                "No model implementation is available for this model."
            )
        if self.name is None:
            raise ValueError("name is not set for ModelMeta. Cannot load model.")

        # Allow overwrites
        _kwargs = self.loader_kwargs.copy()
        _kwargs.update(kwargs)
        if device is not None:
            _kwargs["device"] = device

        model: MTEBModels = self.loader(self.name, revision=self.revision, **_kwargs)
        model.mteb_model_meta = self  # type: ignore[misc]
        return model

    def model_name_as_path(self) -> str:
        """Returns the model name in a format that can be used as a file path.

        Replaces "/" with "__" and spaces with "_".
        """
        if self.name is None:
            raise ValueError("Model name is not set")
        return self.name.replace("/", "__").replace(" ", "_")

    @classmethod
    def _detect_cross_encoder_or_dense(
        cls,
        model_name: str,
        revision: str | None,
        config: dict[str, Any] | None,
        sentence_transformers_loader: Callable[..., MTEBModels],
        cross_encoder_loader: Callable[..., MTEBModels],
    ) -> tuple[Callable[..., MTEBModels] | None, MODEL_TYPES]:
        """Detect if model is CrossEncoder or default to dense."""
        if not config:
            logger.warning(
                f"Could not load config.json for {model_name}. "
                "Defaulting to SentenceTransformer loader."
            )
            return sentence_transformers_loader, "dense"

        architectures = config.get("architectures", [])

        is_cross_encoder = any(
            arch.endswith("ForSequenceClassification") for arch in architectures
        )
        if is_cross_encoder:
            return cross_encoder_loader, "cross-encoder"

        if cls._is_causal_lm_reranker(architectures, config, model_name):
            return cross_encoder_loader, "cross-encoder"

        logger.info(
            f"Model {model_name} does not have modules.json or recognized architecture. "
            "Defaulting to SentenceTransformer loader."
        )
        return sentence_transformers_loader, "dense"

    @staticmethod
    def _is_causal_lm_reranker(
        architectures: list[str], config: dict[str, Any], model_name: str
    ) -> bool:
        """Check if model is a CausalLM-style reranker."""
        is_causal_lm = any(arch.endswith("ForCausalLM") for arch in architectures)

        if not is_causal_lm:
            return False

        num_labels = config.get("num_labels", 0)
        model_name_lower = model_name.lower()

        return (
            num_labels > 0
            or "rerank" in model_name_lower
            or "cross-encoder" in model_name_lower
        )

    @classmethod
    def _detect_model_type_and_loader(
        cls,
        model_name: str,
        revision: str | None = None,
        config: dict[str, Any] | None = None,
    ) -> tuple[Callable[..., MTEBModels] | None, MODEL_TYPES]:
        """Detect the model type and appropriate loader based on HuggingFace Hub configuration files.

        This follows the Sentence Transformers architecture detection logic:
        1. Check for modules.json - If present, model is a SentenceTransformer (dense encoder)
        2. If no modules.json, check config.json for architecture:
            - ForSequenceClassification → CrossEncoder
            - CausalLM with reranking indicators → CrossEncoder
        3. Default to dense (SentenceTransformer) if no clear indicators are found

        Detection for CausalLM-style rerankers:
        - Model has ForCausalLM architecture AND
        - Has num_labels > 0 in config, OR
        - Model name contains "rerank" or "cross-encoder"

        Args:
            model_name: The HuggingFace model name
            revision: The model revision
            config: The loaded config.json from the HuggingFace model repository. If not provided, it will be fetched from the hub.


        Returns:
            A tuple of (loader_function, model_type) where:
            - loader_function: A callable that returns MTEBModels, or None if model doesn't exist
            - model_type: One of "dense", "cross-encoder", or "late-interaction"
        """
        from mteb.models import CrossEncoderWrapper, sentence_transformers_loader

        try:
            modules_config = _get_json_from_hub(
                model_name, "modules.json", "model", revision=revision
            )

            if (
                modules_config
            ):  # SentenceTransformer/SparseEncoder (Not support for now)
                return sentence_transformers_loader, "dense"
            else:
                return cls._detect_cross_encoder_or_dense(
                    model_name,
                    revision,
                    config,
                    sentence_transformers_loader,
                    cross_encoder_loader=CrossEncoderWrapper,
                )

        except Exception as e:
            logger.warning(
                f"Error detecting model type for {model_name}: {e}. "
                "Defaulting to SentenceTransformer loader."
            )

        return sentence_transformers_loader, "dense"

    @classmethod
    def create_empty(cls, overwrites: dict[str, Any] | None = None) -> Self:
        """Creates an empty ModelMeta with all fields set to None or empty."""
        empty_model = cls(
            loader=None,
            name=None,
            revision=None,
            release_date=None,
            languages=None,
            n_parameters=None,
            n_embedding_parameters=None,
            memory_usage_mb=None,
            max_tokens=None,
            embed_dim=None,
            license=None,
            open_weights=None,
            public_training_code=None,
            public_training_data=None,
            framework=[],
            reference=None,
            similarity_fn_name=None,
            use_instructions=None,
            training_datasets=None,
            adapted_from=None,
            superseded_by=None,
            citation=None,
            contacts=None,
        )
        if overwrites:
            empty_model = empty_model.model_copy(update=overwrites)

        if empty_model.name is None:
            empty_model.name = "no_model_name/available"
        if empty_model.revision is None:
            empty_model.revision = "no_revision_available"

        return empty_model

    def merge(self, overwrite: Self) -> Self:
        """Merges another this ModelMeta with another ModelMeta.

        Args:
            overwrite: The ModelMeta to merge into this one. Non-None fields in this ModelMeta will overwrite the corresponding fields in this
                ModelMeta. the `framework` and `model_type` fields with combined.

        Returns:
            A new ModelMeta with the merged fields.
        """
        merged_data = self.model_dump()
        overwrite_data = overwrite.model_dump()

        for key, value in overwrite_data.items():
            if (
                key == "name"
                and value == "no_model_name/available"
                and self.name != "no_model_name/available"
            ):
                continue  # skip overwriting name if overwrite has no name available
            if (
                key == "revision"
                and value == "no_revision_available"
                and self.revision != "no_revision_available"
            ):
                continue  # skip overwriting revision if overwrite has no revision available
            if key in ["framework", "model_type"]:
                # Combine lists and remove duplicates
                merged_list = set(merged_data.get(key, [])) | set(value or [])
                merged_data[key] = list(merged_list)
            if value is not None:
                merged_data[key] = value

        return self.model_copy(update=merged_data)

    @classmethod
    def _from_sentence_transformer_model(cls, model: SentenceTransformer) -> Self:
        """Generates a ModelMeta from only a SentenceTransformer model, without fetching any additional metadata from HuggingFace Hub."""
        from mteb.models import sentence_transformers_loader

        name: str | None = (
            model.model_card_data.model_name
            if model.model_card_data.model_name
            else model.model_card_data.base_model
        )
        n_embedding_parameters = (
            cls._get_n_embedding_parameters_from_sentence_transformers(model)
        )
        return cls.create_empty(
            overwrites=dict(
                name=name,
                revision=model.model_card_data.base_model_revision,
                loader=sentence_transformers_loader,
                max_tokens=model.max_seq_length,
                embed_dim=model.get_sentence_embedding_dimension(),
                similarity_fn_name=ScoringFunction.from_str(model.similarity_fn_name),
                framework=["Sentence Transformers", "PyTorch"],
                n_embedding_parameters=n_embedding_parameters,
            )
        )

    @staticmethod
    def _get_n_embedding_parameters_from_sentence_transformers(
        model: SentenceTransformer | CrossEncoder,
    ) -> int | None:
        """Calculates the number of embedding parameters in a SentenceTransformer model

        This is based on the heuristic: `vocab_size * embedding_dim` where vocab_size and embedding_dim are extracted from the model's first
        Transformer module.
        """
        logger.info(
            "Calculating number of embedding parameters for SentenceTransformer model."
        )

        emb = None
        if isinstance(model, CrossEncoder) and hasattr(
            model.model, "get_input_embeddings"
        ):
            emb = model.model.get_input_embeddings()
            return int(np.prod(emb.weight.shape))
        elif isinstance(model, SentenceTransformer):
            vocab = None
            try:
                vocab = len(model.tokenizer.vocab)
            except Exception as e:
                msg = f"Could not determine vocab size for model {model.model_card_data.model_name} and therefore cannot calculate number of embedding parameters. \nError: \n{e}"
                logger.warning(msg)
            embedding_dimensions = model.get_sentence_embedding_dimension()
            if embedding_dimensions is not None and vocab is not None:
                return vocab * embedding_dimensions

        logger.warning(
            f"Model does not have a recognized architecture for calculating embedding parameters (model={model.model_card_data.model_name})."
        )
        return None

    @classmethod
    def _from_cross_encoder_model(cls, model: CrossEncoder) -> Self:
        """Generates a ModelMeta from only a CrossEncoder model, without fetching any additional metadata from HuggingFace Hub."""
        from mteb.models import CrossEncoderWrapper

        return cls.create_empty(
            overwrites=dict(
                loader=CrossEncoderWrapper,
                name=model.model.name_or_path,
                revision=model.config._commit_hash,
                framework=["Sentence Transformers", "PyTorch"],
                model_type=["cross-encoder"],
                n_embedding_parameters=cls._get_n_embedding_parameters_from_sentence_transformers(
                    model
                ),
            )
        )

    @classmethod
    def _from_hub(
        cls,
        model_name: str,
        revision: str | None = None,
    ) -> Self:
        """Generates a ModelMeta from a HuggingFace model name.

        Args:
            model_name: The HuggingFace model name.
            revision: Revision of the model
            fill_missing: Fill missing attributes from the metadata including number of parameters and memory usage.

        Returns:
            The generated ModelMeta.
        """
        loader: Callable[..., MTEBModels] | None
        model_type: MODEL_TYPES

        reference = "https://huggingface.co/" + model_name

        if not _repo_exists(model_name):
            warnings.warn(
                f"Could not find model {model_name} on HuggingFace Hub repository ({reference}). Metadata will be limited."
            )
            return cls.create_empty(
                overwrites=dict(
                    name=model_name,
                    revision=revision,
                )
            )
        config = _get_json_from_hub(
            model_name, "config.json", "model", revision=revision
        )
        loader, model_type = cls._detect_model_type_and_loader(
            model_name, revision, config=config
        )
        card = ModelCard.load(model_name)
        card_data = card.data
        card_data = cast("ModelCardData", card_data)
        try:
            model_config = AutoConfig.from_pretrained(model_name)
        except Exception as e:
            # some models can't load AutoConfig (e.g. `average_word_embeddings_levy_dependency`)
            model_config = None
            logger.warning(
                f"Can't get model configuration for {model_name}. Error: {e}"
            )

        frameworks = cls._get_frameworks_from_hf_tags(model_name) if model_name else []

        if revision is None:
            revisions = _get_repo_commits(model_name, "model")
            revision = revisions[0].commit_id if revisions else None

        model_license = card_data.license if card_data.license != "other" else None
        n_parameters = cls._calculate_num_parameters_from_hub(model_name)
        n_embedding_parameters = cls._estimate_embedding_parameters_from_hub(
            model_name, revision=revision, config=config
        )
        memory_usage_mb = cls._calculate_memory_usage_mb(model_name, n_parameters)

        embedding_dim = getattr(model_config, "hidden_size", None)
        max_tokens = getattr(model_config, "max_position_embeddings", None)

        sbert_config = _get_json_from_hub(
            model_name, "sentence_bert_config.json", "model", revision=revision
        )
        if sbert_config:
            if max_tokens is None:
                max_tokens = sbert_config.get("max_seq_length", None)
        # have model type, similarity function fields
        config_sbert = _get_json_from_hub(
            model_name, "config_sentence_transformers.json", "model", revision=revision
        )
        similarity_fn_name = (
            ScoringFunction.from_str(config_sbert["similarity_fn_name"])
            if config_sbert is not None
            and config_sbert.get("similarity_fn_name") is not None
            else ScoringFunction.COSINE
        )

        return cls.create_empty(
            overwrites=dict(
                loader=loader,
                name=model_name,
                model_type=[model_type],
                revision=revision,
                reference=reference,
                release_date=cls.fetch_release_date(model_name),
                license=model_license,
                framework=frameworks,
                n_parameters=n_parameters,
                n_embedding_parameters=n_embedding_parameters,
                memory_usage_mb=memory_usage_mb,
                max_tokens=max_tokens,
                embed_dim=embedding_dim,
                similarity_fn_name=similarity_fn_name,
            )
        )

    @classmethod
    def from_sentence_transformer_model(
        cls,
        model: SentenceTransformer,
        revision: str | None = None,
        fill_missing: bool = False,
        compute_metadata: bool | None = None,
        fetch_from_hf: bool = False,
    ) -> Self:
        """Generates a ModelMeta from a SentenceTransformer model.

        Args:
            model: SentenceTransformer model.
            revision: Revision of the model
            fill_missing: Fill missing attributes from the metadata including number of parameters and memory usage.
            compute_metadata: Deprecated. Use fill_missing instead.
            fetch_from_hf: Whether to fetch additional metadata from HuggingFace Hub based on the model name. If False, only metadata that can be
                extracted from the SentenceTransformer model will be used.

        Returns:
            The generated ModelMeta.
        """
        if compute_metadata is not None:
            warnings.warn(
                "The compute_metadata parameter is deprecated and will be removed in a future version. "
                f"Use fetch_from_hf instead. Setting `fetch_from_hf={compute_metadata}`.",
                DeprecationWarning,
                stacklevel=2,
            )
            fetch_from_hf = compute_metadata

        if fill_missing is not None:
            warnings.warn(
                "The fill_missing parameter is deprecated and will be removed in a future version. "
                f"Use fetch_from_hf instead. Setting `fetch_from_hf={fill_missing}`.",
                DeprecationWarning,
                stacklevel=2,
            )
            fetch_from_hf = fill_missing

        meta = cls._from_sentence_transformer_model(model)
        if fetch_from_hf:
            if meta.name is None:
                logger.warning(
                    "Model name is not set in metadata extracted from SentenceTransformer model. Cannot fetch additional metadata from HuggingFace Hub."
                )
            else:
                name = cast("str", meta.name)
                meta_hub = cls._from_hub(name, revision)
                # prioritize metadata from the model card but fill missing fields from the hub
                meta = meta_hub.merge(meta)

        return meta

    @classmethod
    def from_hub(
        cls,
        model: str,
        revision: str | None = None,
        fill_missing: bool | None = None,
        compute_metadata: bool | None = None,
    ) -> Self:
        """Generates a ModelMeta for model from HuggingFace hub.

        Args:
            model: Name of the model from HuggingFace hub. For example, `intfloat/multilingual-e5-large`
            revision: Revision of the model
            fill_missing: Deprecated. The fill missing did not add any functionality for this function, but was added for compatibility with
                'from_sentence_transformer_model' and `from_cross_encoder`. It will be removed in a future version.
            compute_metadata: Deprecated. Was superseded by fill_missing.

        Returns:
            The generated ModelMeta.
        """
        if compute_metadata is not None:
            warnings.warn(
                "The compute_metadata parameter is deprecated and will be removed in a future version. It will be ignored.",
                DeprecationWarning,
                stacklevel=2,
            )

        if fill_missing is not None:
            warnings.warn(
                "The fill_missing parameter is deprecated and will be removed in a future version. It will be ignored.",
                DeprecationWarning,
                stacklevel=2,
            )

        meta = cls._from_hub(
            model,
            revision,
        )

        return meta

    @classmethod
    def from_cross_encoder(
        cls,
        model: CrossEncoder,
        revision: str | None = None,
        fill_missing: bool | None = None,
        compute_metadata: bool | None = None,
        fetch_from_hf: bool = False,
    ) -> Self:
        """Generates a ModelMeta from a CrossEncoder.

        Args:
            model: The CrossEncoder model
            revision: Revision of the model
            fill_missing: Fill missing attributes from the metadata including number of parameters and memory usage.
            compute_metadata: Deprecated. Use fill_missing instead.
            fetch_from_hf: Whether to fetch additional metadata from HuggingFace Hub based on the model name. If False, only metadata that can be
                extracted from the CrossEncoder model will be used.

        Returns:
            The generated ModelMeta
        """
        if compute_metadata is not None:
            warnings.warn(
                "The compute_metadata parameter is deprecated and will be removed in a future version. "
                f"Use fetch_from_hf instead. Setting `fetch_from_hf={compute_metadata}`.",
                DeprecationWarning,
                stacklevel=2,
            )
            fetch_from_hf = compute_metadata
        if fill_missing is not None:
            warnings.warn(
                "The fill_missing parameter is deprecated and will be removed in a future version. "
                f"Use fill_missing instead. Setting `fill_missing={fill_missing}`.",
                DeprecationWarning,
                stacklevel=2,
            )
            fetch_from_hf = fill_missing

        meta = cls._from_cross_encoder_model(model)
        if fetch_from_hf:
            name = cast("str", meta.name)
            meta_hub = cls._from_hub(name, revision)
            # prioritize metadata from the model card but fill missing fields from the hub
            meta = meta_hub.merge(meta)

        return meta

    def is_zero_shot_on(self, tasks: Sequence[AbsTask] | Sequence[str]) -> bool | None:
        """Indicates whether the given model can be considered zero-shot or not on the given tasks.

        Returns:
             None if no training data is specified on the model.
        """
        # If no tasks were specified, we're obviously zero-shot
        if not tasks:
            return True
        training_datasets = self.get_training_datasets()
        # If no tasks were specified, we're obviously zero-shot
        if training_datasets is None:
            return None

        if isinstance(tasks[0], str):
            benchmark_datasets = set(tasks)
        else:
            tasks = cast("Sequence[AbsTask]", tasks)
            benchmark_datasets = set()
            for task in tasks:
                benchmark_datasets.add(task.metadata.name)
        intersection = training_datasets & benchmark_datasets
        return len(intersection) == 0

    def get_training_datasets(self) -> set[str] | None:
        """Returns all training datasets of the model including similar tasks."""
        import mteb

        if self.training_datasets is None:
            return None

        training_datasets = self.training_datasets.copy()
        if self.adapted_from is not None:
            try:
                adapted_from_model = mteb.get_model_meta(
                    self.adapted_from, fetch_from_hf=False
                )
                adapted_training_datasets = adapted_from_model.get_training_datasets()
                if adapted_training_datasets is not None:
                    training_datasets |= adapted_training_datasets
            except (ValueError, KeyError) as e:
                msg = f"Could not get source model: {e} in MTEB"
                logger.warning(msg)
                warnings.warn(msg)

        return_dataset = training_datasets.copy()
        visited: set[str] = set()

        for dataset in training_datasets:
            similar_tasks = _collect_similar_tasks(dataset, visited)
            return_dataset |= similar_tasks

        return return_dataset

    def zero_shot_percentage(
        self, tasks: Sequence[AbsTask] | Sequence[str]
    ) -> int | None:
        """Indicates how out-of-domain the selected tasks are for the given model.

        Args:
            tasks: A sequence of tasks or dataset names to evaluate against.

        Returns:
            An integer percentage (0-100) indicating how out-of-domain the tasks are for the model.
            Returns None if no training data is specified on the model or if no tasks are provided.
        """
        training_datasets = self.get_training_datasets()
        if (training_datasets is None) or (not tasks):
            return None
        if isinstance(tasks[0], str):
            benchmark_datasets = set(tasks)
        else:
            tasks = cast("Sequence[AbsTask]", tasks)
            benchmark_datasets = {task.metadata.name for task in tasks}
        overlap = training_datasets & benchmark_datasets
        perc_overlap = 100 * (len(overlap) / len(benchmark_datasets))
        return int(100 - perc_overlap)

    @staticmethod
    def _calculate_num_parameters_from_hub(model_name: str | None = None) -> int | None:
        if not model_name:
            return None
        try:
            safetensors_metadata = get_safetensors_metadata(model_name)
            if len(safetensors_metadata.parameter_count) >= 0:
                return sum(safetensors_metadata.parameter_count.values())
        except (
            NotASafetensorsRepoError,
            SafetensorsParsingError,
            GatedRepoError,
            RepositoryNotFoundError,
        ) as e:
            logger.warning(
                f"Can't calculate number of parameters for {model_name}. Got error {e}"
            )
        return None

    def calculate_num_parameters_from_hub(self) -> int | None:
        """Calculates the number of parameters in the model.

        Returns:
            Number of parameters in the model.
        """
        return self._calculate_num_parameters_from_hub(self.name)

    @staticmethod
    def _estimate_embedding_parameters_from_hub(
        model_name: str | None = None,
        revision: str | None = None,
        config: dict[str, Any] | None = None,
    ) -> int | None:
        """Calculate the number of embedding parameters from the model config (vocab_size * hidden_size).  Note that this is an heuristic that works for many models, but might be incorrect.

        Returns:
            Number of embedding parameters in the model.
        """
        if not model_name:
            return None

        if not config:
            logger.warning(
                f"Could not calculate embedding parameters for {model_name} as config.json could not be loaded"
            )
            return None

        vocab_size = config.get("vocab_size")
        if vocab_size is None and "text_config" in config:
            vocab_size = config["text_config"].get("vocab_size")

        if vocab_size is None:
            logger.warning(
                f"Could not calculate embedding parameters for {model_name} as vocab_size is missing from config"
            )
            return None

        hidden_size = config.get("hidden_size") or config.get("hidden_dim")
        if hidden_size is None and "text_config" in config:
            hidden_size = config["text_config"].get("hidden_size") or config[
                "text_config"
            ].get("hidden_dim")

        if hidden_size is None:
            logger.warning(
                f"Could not calculate embedding parameters for {model_name} as hidden_size/hidden_dim is missing from config"
            )
            return None
        return vocab_size * hidden_size

    @staticmethod
    def _calculate_memory_usage_mb(
        model_name: str, n_parameters: int | None
    ) -> int | None:
        MB = 1024**2  # noqa: N806

        try:
            safetensors_metadata = get_safetensors_metadata(model_name)
            if len(safetensors_metadata.parameter_count) >= 0:
                dtype_size_map = {
                    "F64": 8,  # 64-bit float
                    "F32": 4,  # 32-bit float (FP32)
                    "F16": 2,  # 16-bit float (FP16)
                    "BF16": 2,  # BFloat16
                    "I64": 8,  # 64-bit integer
                    "I32": 4,  # 32-bit integer
                    "I16": 2,  # 16-bit integer
                    "I8": 1,  # 8-bit integer
                    "U8": 1,  # Unsigned 8-bit integer
                    "BOOL": 1,  # Boolean (assuming 1 byte per value)
                }
                total_memory_bytes = sum(
                    parameters * dtype_size_map.get(dtype, 4)
                    for dtype, parameters in safetensors_metadata.parameter_count.items()
                )
                return round(total_memory_bytes / MB)  # Convert to MB
        except (
            NotASafetensorsRepoError,
            SafetensorsParsingError,
            GatedRepoError,
            RepositoryNotFoundError,
        ) as e:
            logger.warning(
                f"Can't calculate memory usage for {model_name}. Got error {e}"
            )

        if n_parameters is None:
            return None
        # Model memory in bytes. For FP32 each parameter is 4 bytes.
        model_memory_bytes = n_parameters * 4

        # Convert to MB
        model_memory_mb = model_memory_bytes / MB
        return round(model_memory_mb)

    def calculate_memory_usage_mb(self) -> int | None:
        """Calculates the memory usage of the model in MB.

        Returns:
            The memory usage of the model in MB, or None if it cannot be determined.
        """
        if "API" in self.framework or self.name is None:
            return None

        return self._calculate_memory_usage_mb(self.name, self.n_parameters)

    @staticmethod
    def fetch_release_date(model_name: str) -> StrDate | None:
        """Fetches the release date from HuggingFace Hub based on the first commit.

        Returns:
            The release date in YYYY-MM-DD format, or None if it cannot be determined.
        """
        commits = _get_repo_commits(repo_id=model_name, repo_type="model")
        if commits:
            initial_commit = commits[-1]
            release_date = initial_commit.created_at.strftime("%Y-%m-%d")
            return release_date
        return None

    @staticmethod
    def _get_frameworks_from_hf_tags(model_name: str) -> list[FRAMEWORKS]:
        """Extract frameworks supported by the model from HuggingFace model tags.

        Args:
            model_name: HuggingFace model name

        Returns:
            List of framework names found in tags. Defaults to empty list if no frameworks found.
        """
        try:
            info = model_info(model_name)
            if not info.tags:
                return []
        except Exception as e:
            logger.warning(
                f"Failed to fetch frameworks from HuggingFace tags for {model_name}: {e}"
            )
            return []

        # Mapping from HuggingFace tags to MTEB framework names
        tag_to_framework: dict[str, FRAMEWORKS] = {
            "sentence-transformers": "Sentence Transformers",
            "transformers": "Transformers",
            "onnx": "ONNX",
            "safetensors": "safetensors",
            "gguf": "GGUF",
        }

        # Assume PyTorch support by default
        # TODO: could be detected from repo as well: https://github.com/embeddings-benchmark/mteb/issues/4104
        frameworks: list[FRAMEWORKS] = ["PyTorch"]

        for framework_tag in tag_to_framework.keys():
            if framework_tag in info.tags:
                frameworks.append(tag_to_framework[framework_tag])

        return frameworks

    def to_python(self) -> str:
        """Returns a string representation of the model."""
        return _pydantic_instance_to_code(self)

is_cross_encoder property

Returns True if the model is a cross-encoder.

Derived from model_type field. A model is considered a cross-encoder if "cross-encoder" is in its model_type list.

n_active_parameters property

Number of active parameters. Assumed to be n_parameters - n_embedding_parameters. Can be overwritten using n_active_parameters_override e.g. for MoE models.

calculate_memory_usage_mb()

Calculates the memory usage of the model in MB.

Returns:

Type	Description
int | None	The memory usage of the model in MB, or None if it cannot be determined.

Source code in mteb/models/model_meta.py

def calculate_memory_usage_mb(self) -> int | None:
    """Calculates the memory usage of the model in MB.

    Returns:
        The memory usage of the model in MB, or None if it cannot be determined.
    """
    if "API" in self.framework or self.name is None:
        return None

    return self._calculate_memory_usage_mb(self.name, self.n_parameters)

calculate_num_parameters_from_hub()

Calculates the number of parameters in the model.

Returns:

Type	Description
int | None	Number of parameters in the model.

Source code in mteb/models/model_meta.py

def calculate_num_parameters_from_hub(self) -> int | None:
    """Calculates the number of parameters in the model.

    Returns:
        Number of parameters in the model.
    """
    return self._calculate_num_parameters_from_hub(self.name)

create_empty(overwrites=None) classmethod

Creates an empty ModelMeta with all fields set to None or empty.

Source code in mteb/models/model_meta.py

@classmethod
def create_empty(cls, overwrites: dict[str, Any] | None = None) -> Self:
    """Creates an empty ModelMeta with all fields set to None or empty."""
    empty_model = cls(
        loader=None,
        name=None,
        revision=None,
        release_date=None,
        languages=None,
        n_parameters=None,
        n_embedding_parameters=None,
        memory_usage_mb=None,
        max_tokens=None,
        embed_dim=None,
        license=None,
        open_weights=None,
        public_training_code=None,
        public_training_data=None,
        framework=[],
        reference=None,
        similarity_fn_name=None,
        use_instructions=None,
        training_datasets=None,
        adapted_from=None,
        superseded_by=None,
        citation=None,
        contacts=None,
    )
    if overwrites:
        empty_model = empty_model.model_copy(update=overwrites)

    if empty_model.name is None:
        empty_model.name = "no_model_name/available"
    if empty_model.revision is None:
        empty_model.revision = "no_revision_available"

    return empty_model

fetch_release_date(model_name) staticmethod

Fetches the release date from HuggingFace Hub based on the first commit.

Returns:

Type	Description
StrDate | None	The release date in YYYY-MM-DD format, or None if it cannot be determined.

Source code in mteb/models/model_meta.py

@staticmethod
def fetch_release_date(model_name: str) -> StrDate | None:
    """Fetches the release date from HuggingFace Hub based on the first commit.

    Returns:
        The release date in YYYY-MM-DD format, or None if it cannot be determined.
    """
    commits = _get_repo_commits(repo_id=model_name, repo_type="model")
    if commits:
        initial_commit = commits[-1]
        release_date = initial_commit.created_at.strftime("%Y-%m-%d")
        return release_date
    return None

from_cross_encoder(model, revision=None, fill_missing=None, compute_metadata=None, fetch_from_hf=False) classmethod

Generates a ModelMeta from a CrossEncoder.

Parameters:

Name	Type	Description	Default
model	CrossEncoder	The CrossEncoder model	required
revision	str | None	Revision of the model	None
fill_missing	bool | None	Fill missing attributes from the metadata including number of parameters and memory usage.	None
compute_metadata	bool | None	Deprecated. Use fill_missing instead.	None
fetch_from_hf	bool	Whether to fetch additional metadata from HuggingFace Hub based on the model name. If False, only metadata that can be extracted from the CrossEncoder model will be used.	False

Returns:

Type	Description
Self	The generated ModelMeta

Source code in mteb/models/model_meta.py

@classmethod
def from_cross_encoder(
    cls,
    model: CrossEncoder,
    revision: str | None = None,
    fill_missing: bool | None = None,
    compute_metadata: bool | None = None,
    fetch_from_hf: bool = False,
) -> Self:
    """Generates a ModelMeta from a CrossEncoder.

    Args:
        model: The CrossEncoder model
        revision: Revision of the model
        fill_missing: Fill missing attributes from the metadata including number of parameters and memory usage.
        compute_metadata: Deprecated. Use fill_missing instead.
        fetch_from_hf: Whether to fetch additional metadata from HuggingFace Hub based on the model name. If False, only metadata that can be
            extracted from the CrossEncoder model will be used.

    Returns:
        The generated ModelMeta
    """
    if compute_metadata is not None:
        warnings.warn(
            "The compute_metadata parameter is deprecated and will be removed in a future version. "
            f"Use fetch_from_hf instead. Setting `fetch_from_hf={compute_metadata}`.",
            DeprecationWarning,
            stacklevel=2,
        )
        fetch_from_hf = compute_metadata
    if fill_missing is not None:
        warnings.warn(
            "The fill_missing parameter is deprecated and will be removed in a future version. "
            f"Use fill_missing instead. Setting `fill_missing={fill_missing}`.",
            DeprecationWarning,
            stacklevel=2,
        )
        fetch_from_hf = fill_missing

    meta = cls._from_cross_encoder_model(model)
    if fetch_from_hf:
        name = cast("str", meta.name)
        meta_hub = cls._from_hub(name, revision)
        # prioritize metadata from the model card but fill missing fields from the hub
        meta = meta_hub.merge(meta)

    return meta

from_hub(model, revision=None, fill_missing=None, compute_metadata=None) classmethod

Generates a ModelMeta for model from HuggingFace hub.

Parameters:

Name	Type	Description	Default
model	str	Name of the model from HuggingFace hub. For example, intfloat/multilingual-e5-large	required
revision	str | None	Revision of the model	None
fill_missing	bool | None	Deprecated. The fill missing did not add any functionality for this function, but was added for compatibility with 'from_sentence_transformer_model' and from_cross_encoder. It will be removed in a future version.	None
compute_metadata	bool | None	Deprecated. Was superseded by fill_missing.	None

Returns:

Type	Description
Self	The generated ModelMeta.

Source code in mteb/models/model_meta.py

@classmethod
def from_hub(
    cls,
    model: str,
    revision: str | None = None,
    fill_missing: bool | None = None,
    compute_metadata: bool | None = None,
) -> Self:
    """Generates a ModelMeta for model from HuggingFace hub.

    Args:
        model: Name of the model from HuggingFace hub. For example, `intfloat/multilingual-e5-large`
        revision: Revision of the model
        fill_missing: Deprecated. The fill missing did not add any functionality for this function, but was added for compatibility with
            'from_sentence_transformer_model' and `from_cross_encoder`. It will be removed in a future version.
        compute_metadata: Deprecated. Was superseded by fill_missing.

    Returns:
        The generated ModelMeta.
    """
    if compute_metadata is not None:
        warnings.warn(
            "The compute_metadata parameter is deprecated and will be removed in a future version. It will be ignored.",
            DeprecationWarning,
            stacklevel=2,
        )

    if fill_missing is not None:
        warnings.warn(
            "The fill_missing parameter is deprecated and will be removed in a future version. It will be ignored.",
            DeprecationWarning,
            stacklevel=2,
        )

    meta = cls._from_hub(
        model,
        revision,
    )

    return meta

from_sentence_transformer_model(model, revision=None, fill_missing=False, compute_metadata=None, fetch_from_hf=False) classmethod

Generates a ModelMeta from a SentenceTransformer model.

Parameters:

Name	Type	Description	Default
model	SentenceTransformer	SentenceTransformer model.	required
revision	str | None	Revision of the model	None
fill_missing	bool	Fill missing attributes from the metadata including number of parameters and memory usage.	False
compute_metadata	bool | None	Deprecated. Use fill_missing instead.	None
fetch_from_hf	bool	Whether to fetch additional metadata from HuggingFace Hub based on the model name. If False, only metadata that can be extracted from the SentenceTransformer model will be used.	False

Returns:

Type	Description
Self	The generated ModelMeta.

Source code in mteb/models/model_meta.py

@classmethod
def from_sentence_transformer_model(
    cls,
    model: SentenceTransformer,
    revision: str | None = None,
    fill_missing: bool = False,
    compute_metadata: bool | None = None,
    fetch_from_hf: bool = False,
) -> Self:
    """Generates a ModelMeta from a SentenceTransformer model.

    Args:
        model: SentenceTransformer model.
        revision: Revision of the model
        fill_missing: Fill missing attributes from the metadata including number of parameters and memory usage.
        compute_metadata: Deprecated. Use fill_missing instead.
        fetch_from_hf: Whether to fetch additional metadata from HuggingFace Hub based on the model name. If False, only metadata that can be
            extracted from the SentenceTransformer model will be used.

    Returns:
        The generated ModelMeta.
    """
    if compute_metadata is not None:
        warnings.warn(
            "The compute_metadata parameter is deprecated and will be removed in a future version. "
            f"Use fetch_from_hf instead. Setting `fetch_from_hf={compute_metadata}`.",
            DeprecationWarning,
            stacklevel=2,
        )
        fetch_from_hf = compute_metadata

    if fill_missing is not None:
        warnings.warn(
            "The fill_missing parameter is deprecated and will be removed in a future version. "
            f"Use fetch_from_hf instead. Setting `fetch_from_hf={fill_missing}`.",
            DeprecationWarning,
            stacklevel=2,
        )
        fetch_from_hf = fill_missing

    meta = cls._from_sentence_transformer_model(model)
    if fetch_from_hf:
        if meta.name is None:
            logger.warning(
                "Model name is not set in metadata extracted from SentenceTransformer model. Cannot fetch additional metadata from HuggingFace Hub."
            )
        else:
            name = cast("str", meta.name)
            meta_hub = cls._from_hub(name, revision)
            # prioritize metadata from the model card but fill missing fields from the hub
            meta = meta_hub.merge(meta)

    return meta

get_training_datasets()

Returns all training datasets of the model including similar tasks.

Source code in mteb/models/model_meta.py

def get_training_datasets(self) -> set[str] | None:
    """Returns all training datasets of the model including similar tasks."""
    import mteb

    if self.training_datasets is None:
        return None

    training_datasets = self.training_datasets.copy()
    if self.adapted_from is not None:
        try:
            adapted_from_model = mteb.get_model_meta(
                self.adapted_from, fetch_from_hf=False
            )
            adapted_training_datasets = adapted_from_model.get_training_datasets()
            if adapted_training_datasets is not None:
                training_datasets |= adapted_training_datasets
        except (ValueError, KeyError) as e:
            msg = f"Could not get source model: {e} in MTEB"
            logger.warning(msg)
            warnings.warn(msg)

    return_dataset = training_datasets.copy()
    visited: set[str] = set()

    for dataset in training_datasets:
        similar_tasks = _collect_similar_tasks(dataset, visited)
        return_dataset |= similar_tasks

    return return_dataset

is_zero_shot_on(tasks)

Indicates whether the given model can be considered zero-shot or not on the given tasks.

Returns:

Type	Description
bool | None	None if no training data is specified on the model.

Source code in mteb/models/model_meta.py

def is_zero_shot_on(self, tasks: Sequence[AbsTask] | Sequence[str]) -> bool | None:
    """Indicates whether the given model can be considered zero-shot or not on the given tasks.

    Returns:
         None if no training data is specified on the model.
    """
    # If no tasks were specified, we're obviously zero-shot
    if not tasks:
        return True
    training_datasets = self.get_training_datasets()
    # If no tasks were specified, we're obviously zero-shot
    if training_datasets is None:
        return None

    if isinstance(tasks[0], str):
        benchmark_datasets = set(tasks)
    else:
        tasks = cast("Sequence[AbsTask]", tasks)
        benchmark_datasets = set()
        for task in tasks:
            benchmark_datasets.add(task.metadata.name)
    intersection = training_datasets & benchmark_datasets
    return len(intersection) == 0

load_model(device=None, **kwargs)

Loads the model using the specified loader function.

Source code in mteb/models/model_meta.py

def load_model(self, device: str | None = None, **kwargs: Any) -> MTEBModels:
    """Loads the model using the specified loader function."""
    if self.loader is None:
        raise NotImplementedError(
            "No model implementation is available for this model."
        )
    if self.name is None:
        raise ValueError("name is not set for ModelMeta. Cannot load model.")

    # Allow overwrites
    _kwargs = self.loader_kwargs.copy()
    _kwargs.update(kwargs)
    if device is not None:
        _kwargs["device"] = device

    model: MTEBModels = self.loader(self.name, revision=self.revision, **_kwargs)
    model.mteb_model_meta = self  # type: ignore[misc]
    return model

merge(overwrite)

Merges another this ModelMeta with another ModelMeta.

Parameters:

Name	Type	Description	Default
overwrite	Self	The ModelMeta to merge into this one. Non-None fields in this ModelMeta will overwrite the corresponding fields in this ModelMeta. the framework and model_type fields with combined.	required

Returns:

Type	Description
Self	A new ModelMeta with the merged fields.

Source code in mteb/models/model_meta.py

def merge(self, overwrite: Self) -> Self:
    """Merges another this ModelMeta with another ModelMeta.

    Args:
        overwrite: The ModelMeta to merge into this one. Non-None fields in this ModelMeta will overwrite the corresponding fields in this
            ModelMeta. the `framework` and `model_type` fields with combined.

    Returns:
        A new ModelMeta with the merged fields.
    """
    merged_data = self.model_dump()
    overwrite_data = overwrite.model_dump()

    for key, value in overwrite_data.items():
        if (
            key == "name"
            and value == "no_model_name/available"
            and self.name != "no_model_name/available"
        ):
            continue  # skip overwriting name if overwrite has no name available
        if (
            key == "revision"
            and value == "no_revision_available"
            and self.revision != "no_revision_available"
        ):
            continue  # skip overwriting revision if overwrite has no revision available
        if key in ["framework", "model_type"]:
            # Combine lists and remove duplicates
            merged_list = set(merged_data.get(key, [])) | set(value or [])
            merged_data[key] = list(merged_list)
        if value is not None:
            merged_data[key] = value

    return self.model_copy(update=merged_data)

model_name_as_path()

Returns the model name in a format that can be used as a file path.

Replaces "/" with "__" and spaces with "_".

Source code in mteb/models/model_meta.py

def model_name_as_path(self) -> str:
    """Returns the model name in a format that can be used as a file path.

    Replaces "/" with "__" and spaces with "_".
    """
    if self.name is None:
        raise ValueError("Model name is not set")
    return self.name.replace("/", "__").replace(" ", "_")

to_dict()

Returns a dictionary representation of the model metadata.

Source code in mteb/models/model_meta.py

def to_dict(self):
    """Returns a dictionary representation of the model metadata."""
    dict_repr = self.model_dump()
    loader = dict_repr.pop("loader", None)
    dict_repr["training_datasets"] = (
        list(dict_repr["training_datasets"])
        if isinstance(dict_repr["training_datasets"], set)
        else dict_repr["training_datasets"]
    )
    dict_repr["loader"] = _get_loader_name(loader)
    dict_repr["is_cross_encoder"] = self.is_cross_encoder
    return dict_repr

to_python()

Returns a string representation of the model.

Source code in mteb/models/model_meta.py

def to_python(self) -> str:
    """Returns a string representation of the model."""
    return _pydantic_instance_to_code(self)

zero_shot_percentage(tasks)

Indicates how out-of-domain the selected tasks are for the given model.

Parameters:

Name	Type	Description	Default
tasks	Sequence[AbsTask] | Sequence[str]	A sequence of tasks or dataset names to evaluate against.	required

Returns:

Type	Description
int | None	An integer percentage (0-100) indicating how out-of-domain the tasks are for the model.
int | None	Returns None if no training data is specified on the model or if no tasks are provided.

Source code in mteb/models/model_meta.py

def zero_shot_percentage(
    self, tasks: Sequence[AbsTask] | Sequence[str]
) -> int | None:
    """Indicates how out-of-domain the selected tasks are for the given model.

    Args:
        tasks: A sequence of tasks or dataset names to evaluate against.

    Returns:
        An integer percentage (0-100) indicating how out-of-domain the tasks are for the model.
        Returns None if no training data is specified on the model or if no tasks are provided.
    """
    training_datasets = self.get_training_datasets()
    if (training_datasets is None) or (not tasks):
        return None
    if isinstance(tasks[0], str):
        benchmark_datasets = set(tasks)
    else:
        tasks = cast("Sequence[AbsTask]", tasks)
        benchmark_datasets = {task.metadata.name for task in tasks}
    overlap = training_datasets & benchmark_datasets
    perc_overlap = 100 * (len(overlap) / len(benchmark_datasets))
    return int(100 - perc_overlap)

Model Protocols

mteb.models.EncoderProtocol

Bases: Protocol

The interface for an encoder in MTEB.

Besides the required functions specified below, the encoder can additionally specify the following signatures seen below. In general the interface is kept aligned with sentence-transformers interface. In cases where exceptions occurs these are handled within MTEB.

Source code in mteb/models/models_protocols.py

@runtime_checkable
class EncoderProtocol(Protocol):
    """The interface for an encoder in MTEB.

    Besides the required functions specified below, the encoder can additionally specify the following signatures seen below.
    In general the interface is kept aligned with sentence-transformers interface. In cases where exceptions occurs these are handled within MTEB.
    """

    def __init__(
        self,
        model_name: str,
        revision: str | None,
        device: str | None = None,
        **kwargs: Any,
    ) -> None:
        """The initialization function for the encoder. Used when calling it from the mteb run CLI.

        Args:
            model_name: Name of the model
            revision: revision of the model
            device: Device used to load the model
            kwargs: Any additional kwargs
        """
        ...

    def encode(
        self,
        inputs: DataLoader[BatchedInput],
        *,
        task_metadata: TaskMetadata,
        hf_split: str,
        hf_subset: str,
        prompt_type: PromptType | None = None,
        **kwargs: Unpack[EncodeKwargs],
    ) -> Array:
        """Encodes the given sentences using the encoder.

        Args:
            inputs: Batch of inputs to encode.
            task_metadata: The metadata of the task. Encoders (e.g. SentenceTransformers) use to
                select the appropriate prompts, with priority given to more specific task/prompt combinations over general ones.

                The order of priorities for prompt selection are:
                    1. Composed prompt of task name + prompt type (query or passage)
                    2. Specific task prompt
                    3. Composed prompt of task type + prompt type (query or passage)
                    4. Specific task type prompt
                    5. Specific prompt type (query or passage)
            hf_split: Split of current task, allows to know some additional information about current split.
                E.g. Current language
            hf_subset: Subset of current task. Similar to `hf_split` to get more information
            prompt_type: The name type of prompt. (query or passage)
            **kwargs: Additional arguments to pass to the encoder.

        Returns:
            The encoded input in a numpy array or torch tensor of the shape (Number of sentences) x (Embedding dimension).
        """
        ...

    def similarity(
        self,
        embeddings1: Array,
        embeddings2: Array,
    ) -> Array:
        """Compute the similarity between two collections of embeddings.

        The output will be a matrix with the similarity scores between all embeddings from the first parameter and all
        embeddings from the second parameter. This differs from similarity_pairwise which computes the similarity
        between corresponding pairs of embeddings.

        Read more at: https://www.sbert.net/docs/package_reference/sentence_transformer/SentenceTransformer.html#sentence_transformers.SentenceTransformer.similarity

        Args:
            embeddings1: [num_embeddings_1, embedding_dim] or [embedding_dim]-shaped numpy array or torch tensor.
            embeddings2: [num_embeddings_2, embedding_dim] or [embedding_dim]-shaped numpy array or torch tensor.

        Returns:
            A [num_embeddings_1, num_embeddings_2]-shaped torch tensor with similarity scores.
        """
        ...

    def similarity_pairwise(
        self,
        embeddings1: Array,
        embeddings2: Array,
    ) -> Array:
        """Compute the similarity between two collections of embeddings. The output will be a vector with the similarity scores between each pair of embeddings.

        Read more at: https://www.sbert.net/docs/package_reference/sentence_transformer/SentenceTransformer.html#sentence_transformers.SentenceTransformer.similarity_pairwise

        Args:
            embeddings1: [num_embeddings, embedding_dim] or [embedding_dim]-shaped numpy array or torch tensor.
            embeddings2: [num_embeddings, embedding_dim] or [embedding_dim]-shaped numpy array or torch tensor.

        Returns:
            A [num_embeddings]-shaped torch tensor with pairwise similarity scores.
        """
        ...

    @property
    def mteb_model_meta(self) -> ModelMeta:
        """Metadata of the model"""
        ...

mteb_model_meta property

Metadata of the model

__init__(model_name, revision, device=None, **kwargs)

The initialization function for the encoder. Used when calling it from the mteb run CLI.

Parameters:

Name	Type	Description	Default
model_name	str	Name of the model	required
revision	str | None	revision of the model	required
device	str | None	Device used to load the model	None
kwargs	Any	Any additional kwargs	{}

Source code in mteb/models/models_protocols.py

def __init__(
    self,
    model_name: str,
    revision: str | None,
    device: str | None = None,
    **kwargs: Any,
) -> None:
    """The initialization function for the encoder. Used when calling it from the mteb run CLI.

    Args:
        model_name: Name of the model
        revision: revision of the model
        device: Device used to load the model
        kwargs: Any additional kwargs
    """
    ...

encode(inputs, *, task_metadata, hf_split, hf_subset, prompt_type=None, **kwargs)

Encodes the given sentences using the encoder.

Parameters:

Name	Type	Description	Default
inputs	DataLoader[BatchedInput]	Batch of inputs to encode.	required
task_metadata	TaskMetadata	The metadata of the task. Encoders (e.g. SentenceTransformers) use to select the appropriate prompts, with priority given to more specific task/prompt combinations over general ones. The order of priorities for prompt selection are: 1. Composed prompt of task name + prompt type (query or passage) 2. Specific task prompt 3. Composed prompt of task type + prompt type (query or passage) 4. Specific task type prompt 5. Specific prompt type (query or passage)	required
hf_split	str	Split of current task, allows to know some additional information about current split. E.g. Current language	required
hf_subset	str	Subset of current task. Similar to hf_split to get more information	required
prompt_type	PromptType | None	The name type of prompt. (query or passage)	None
**kwargs	Unpack[EncodeKwargs]	Additional arguments to pass to the encoder.	{}

Returns:

Type	Description
Array	The encoded input in a numpy array or torch tensor of the shape (Number of sentences) x (Embedding dimension).

Source code in mteb/models/models_protocols.py

def encode(
    self,
    inputs: DataLoader[BatchedInput],
    *,
    task_metadata: TaskMetadata,
    hf_split: str,
    hf_subset: str,
    prompt_type: PromptType | None = None,
    **kwargs: Unpack[EncodeKwargs],
) -> Array:
    """Encodes the given sentences using the encoder.

    Args:
        inputs: Batch of inputs to encode.
        task_metadata: The metadata of the task. Encoders (e.g. SentenceTransformers) use to
            select the appropriate prompts, with priority given to more specific task/prompt combinations over general ones.

            The order of priorities for prompt selection are:
                1. Composed prompt of task name + prompt type (query or passage)
                2. Specific task prompt
                3. Composed prompt of task type + prompt type (query or passage)
                4. Specific task type prompt
                5. Specific prompt type (query or passage)
        hf_split: Split of current task, allows to know some additional information about current split.
            E.g. Current language
        hf_subset: Subset of current task. Similar to `hf_split` to get more information
        prompt_type: The name type of prompt. (query or passage)
        **kwargs: Additional arguments to pass to the encoder.

    Returns:
        The encoded input in a numpy array or torch tensor of the shape (Number of sentences) x (Embedding dimension).
    """
    ...

similarity(embeddings1, embeddings2)

Compute the similarity between two collections of embeddings.

The output will be a matrix with the similarity scores between all embeddings from the first parameter and all embeddings from the second parameter. This differs from similarity_pairwise which computes the similarity between corresponding pairs of embeddings.

Read more at: https://www.sbert.net/docs/package_reference/sentence_transformer/SentenceTransformer.html#sentence_transformers.SentenceTransformer.similarity

Parameters:

Name	Type	Description	Default
embeddings1	Array	[num_embeddings_1, embedding_dim] or [embedding_dim]-shaped numpy array or torch tensor.	required
embeddings2	Array	[num_embeddings_2, embedding_dim] or [embedding_dim]-shaped numpy array or torch tensor.	required

Returns:

Type	Description
Array	A [num_embeddings_1, num_embeddings_2]-shaped torch tensor with similarity scores.

Source code in mteb/models/models_protocols.py

def similarity(
    self,
    embeddings1: Array,
    embeddings2: Array,
) -> Array:
    """Compute the similarity between two collections of embeddings.

    The output will be a matrix with the similarity scores between all embeddings from the first parameter and all
    embeddings from the second parameter. This differs from similarity_pairwise which computes the similarity
    between corresponding pairs of embeddings.

    Read more at: https://www.sbert.net/docs/package_reference/sentence_transformer/SentenceTransformer.html#sentence_transformers.SentenceTransformer.similarity

    Args:
        embeddings1: [num_embeddings_1, embedding_dim] or [embedding_dim]-shaped numpy array or torch tensor.
        embeddings2: [num_embeddings_2, embedding_dim] or [embedding_dim]-shaped numpy array or torch tensor.

    Returns:
        A [num_embeddings_1, num_embeddings_2]-shaped torch tensor with similarity scores.
    """
    ...

similarity_pairwise(embeddings1, embeddings2)

Compute the similarity between two collections of embeddings. The output will be a vector with the similarity scores between each pair of embeddings.

Read more at: https://www.sbert.net/docs/package_reference/sentence_transformer/SentenceTransformer.html#sentence_transformers.SentenceTransformer.similarity_pairwise

Parameters:

Name	Type	Description	Default
embeddings1	Array	[num_embeddings, embedding_dim] or [embedding_dim]-shaped numpy array or torch tensor.	required
embeddings2	Array	[num_embeddings, embedding_dim] or [embedding_dim]-shaped numpy array or torch tensor.	required

Returns:

Type	Description
Array	A [num_embeddings]-shaped torch tensor with pairwise similarity scores.

Source code in mteb/models/models_protocols.py

def similarity_pairwise(
    self,
    embeddings1: Array,
    embeddings2: Array,
) -> Array:
    """Compute the similarity between two collections of embeddings. The output will be a vector with the similarity scores between each pair of embeddings.

    Read more at: https://www.sbert.net/docs/package_reference/sentence_transformer/SentenceTransformer.html#sentence_transformers.SentenceTransformer.similarity_pairwise

    Args:
        embeddings1: [num_embeddings, embedding_dim] or [embedding_dim]-shaped numpy array or torch tensor.
        embeddings2: [num_embeddings, embedding_dim] or [embedding_dim]-shaped numpy array or torch tensor.

    Returns:
        A [num_embeddings]-shaped torch tensor with pairwise similarity scores.
    """
    ...

mteb.models.SearchProtocol

Bases: Protocol

Interface for searching models.

Source code in mteb/models/models_protocols.py

@runtime_checkable
class SearchProtocol(Protocol):
    """Interface for searching models."""

    def index(
        self,
        corpus: CorpusDatasetType,
        *,
        task_metadata: TaskMetadata,
        hf_split: str,
        hf_subset: str,
        encode_kwargs: EncodeKwargs,
        num_proc: int | None,
    ) -> None:
        """Index the corpus for retrieval.

        Args:
            corpus: Corpus dataset to index.
            task_metadata: Metadata of the task, used to determine how to index the corpus.
            hf_split: Split of current task, allows to know some additional information about current split.
            hf_subset: Subset of current task. Similar to `hf_split` to get more information
            encode_kwargs: Additional arguments to pass to the encoder during indexing.
            num_proc: Number of processes to use for dataloading.
        """
        ...

    def search(
        self,
        queries: QueryDatasetType,
        *,
        task_metadata: TaskMetadata,
        hf_split: str,
        hf_subset: str,
        top_k: int,
        encode_kwargs: EncodeKwargs,
        top_ranked: TopRankedDocumentsType | None = None,
        num_proc: int | None,
    ) -> RetrievalOutputType:
        """Search the corpus using the given queries.

        Args:
            queries: Queries to find
            task_metadata: Task metadata
            hf_split: split of the dataset
            hf_subset: subset of the dataset
            top_ranked: Top-ranked documents for each query, mapping query IDs to a list of document IDs.
                Passed only from Reranking tasks.
            top_k: Number of top documents to return for each query.
            encode_kwargs: Additional arguments to pass to the encoder during indexing.
            num_proc: Number of processes to use for dataloading.

        Returns:
            Dictionary with query IDs as keys with dict as values, where each value is a mapping of document IDs to their relevance scores.
        """
        ...

    @property
    def mteb_model_meta(self) -> ModelMeta:
        """Metadata of the model"""
        ...

mteb_model_meta property

Metadata of the model

index(corpus, *, task_metadata, hf_split, hf_subset, encode_kwargs, num_proc)

Index the corpus for retrieval.

Parameters:

Name	Type	Description	Default
corpus	CorpusDatasetType	Corpus dataset to index.	required
task_metadata	TaskMetadata	Metadata of the task, used to determine how to index the corpus.	required
hf_split	str	Split of current task, allows to know some additional information about current split.	required
hf_subset	str	Subset of current task. Similar to hf_split to get more information	required
encode_kwargs	EncodeKwargs	Additional arguments to pass to the encoder during indexing.	required
num_proc	int | None	Number of processes to use for dataloading.	required

Source code in mteb/models/models_protocols.py

def index(
    self,
    corpus: CorpusDatasetType,
    *,
    task_metadata: TaskMetadata,
    hf_split: str,
    hf_subset: str,
    encode_kwargs: EncodeKwargs,
    num_proc: int | None,
) -> None:
    """Index the corpus for retrieval.

    Args:
        corpus: Corpus dataset to index.
        task_metadata: Metadata of the task, used to determine how to index the corpus.
        hf_split: Split of current task, allows to know some additional information about current split.
        hf_subset: Subset of current task. Similar to `hf_split` to get more information
        encode_kwargs: Additional arguments to pass to the encoder during indexing.
        num_proc: Number of processes to use for dataloading.
    """
    ...

search(queries, *, task_metadata, hf_split, hf_subset, top_k, encode_kwargs, top_ranked=None, num_proc)

Search the corpus using the given queries.

Parameters:

Name	Type	Description	Default
queries	QueryDatasetType	Queries to find	required
task_metadata	TaskMetadata	Task metadata	required
hf_split	str	split of the dataset	required
hf_subset	str	subset of the dataset	required
top_ranked	TopRankedDocumentsType | None	Top-ranked documents for each query, mapping query IDs to a list of document IDs. Passed only from Reranking tasks.	None
top_k	int	Number of top documents to return for each query.	required
encode_kwargs	EncodeKwargs	Additional arguments to pass to the encoder during indexing.	required
num_proc	int | None	Number of processes to use for dataloading.	required

Returns:

Type	Description
RetrievalOutputType	Dictionary with query IDs as keys with dict as values, where each value is a mapping of document IDs to their relevance scores.

Source code in mteb/models/models_protocols.py

def search(
    self,
    queries: QueryDatasetType,
    *,
    task_metadata: TaskMetadata,
    hf_split: str,
    hf_subset: str,
    top_k: int,
    encode_kwargs: EncodeKwargs,
    top_ranked: TopRankedDocumentsType | None = None,
    num_proc: int | None,
) -> RetrievalOutputType:
    """Search the corpus using the given queries.

    Args:
        queries: Queries to find
        task_metadata: Task metadata
        hf_split: split of the dataset
        hf_subset: subset of the dataset
        top_ranked: Top-ranked documents for each query, mapping query IDs to a list of document IDs.
            Passed only from Reranking tasks.
        top_k: Number of top documents to return for each query.
        encode_kwargs: Additional arguments to pass to the encoder during indexing.
        num_proc: Number of processes to use for dataloading.

    Returns:
        Dictionary with query IDs as keys with dict as values, where each value is a mapping of document IDs to their relevance scores.
    """
    ...

mteb.models.CrossEncoderProtocol

Bases: Protocol

The interface for a CrossEncoder in MTEB.

Besides the required functions specified below, the cross-encoder can additionally specify the following signatures seen below. In general the interface is kept aligned with sentence-transformers interface. In cases where exceptions occurs these are handled within MTEB.

Source code in mteb/models/models_protocols.py

@runtime_checkable
class CrossEncoderProtocol(Protocol):
    """The interface for a CrossEncoder in MTEB.

    Besides the required functions specified below, the cross-encoder can additionally specify the following signatures seen below.
    In general the interface is kept aligned with sentence-transformers interface. In cases where exceptions occurs these are handled within MTEB.
    """

    def __init__(
        self,
        model_name: str,
        revision: str | None,
        device: str | None = None,
        **kwargs: Any,
    ) -> None:
        """The initialization function for the encoder. Used when calling it from the mteb run CLI.

        Args:
            model_name: Name of the model
            revision: revision of the model
            device: Device used to load the model
            kwargs: Any additional kwargs
        """
        ...

    def predict(
        self,
        inputs1: DataLoader[BatchedInput],
        inputs2: DataLoader[BatchedInput],
        *,
        task_metadata: TaskMetadata,
        hf_split: str,
        hf_subset: str,
        prompt_type: PromptType | None = None,
        **kwargs: Unpack[EncodeKwargs],
    ) -> Array:
        """Predicts relevance scores for pairs of inputs. Note that, unlike the encoder, the cross-encoder can compare across inputs.

        Args:
            inputs1: First Dataloader of inputs to encode. For reranking tasks, these are queries (for text only tasks `QueryDatasetType`).
            inputs2: Second Dataloader of inputs to encode. For reranking, these are documents (for text only tasks `RetrievalOutputType`).
            task_metadata: Metadata of the current task.
            hf_split: Split of current task, allows to know some additional information about current split.
                E.g. Current language
            hf_subset: Subset of current task. Similar to `hf_split` to get more information
            prompt_type: The name type of prompt. (query or passage)
            **kwargs: Additional arguments to pass to the cross-encoder.

        Returns:
            The predicted relevance scores for each inputs pair.
        """
        ...

    @property
    def mteb_model_meta(self) -> ModelMeta:
        """Metadata of the model"""
        ...

mteb_model_meta property

Metadata of the model

__init__(model_name, revision, device=None, **kwargs)

The initialization function for the encoder. Used when calling it from the mteb run CLI.

Parameters:

Name	Type	Description	Default
model_name	str	Name of the model	required
revision	str | None	revision of the model	required
device	str | None	Device used to load the model	None
kwargs	Any	Any additional kwargs	{}

Source code in mteb/models/models_protocols.py

def __init__(
    self,
    model_name: str,
    revision: str | None,
    device: str | None = None,
    **kwargs: Any,
) -> None:
    """The initialization function for the encoder. Used when calling it from the mteb run CLI.

    Args:
        model_name: Name of the model
        revision: revision of the model
        device: Device used to load the model
        kwargs: Any additional kwargs
    """
    ...

predict(inputs1, inputs2, *, task_metadata, hf_split, hf_subset, prompt_type=None, **kwargs)

Predicts relevance scores for pairs of inputs. Note that, unlike the encoder, the cross-encoder can compare across inputs.

Parameters:

Name	Type	Description	Default
inputs1	DataLoader[BatchedInput]	First Dataloader of inputs to encode. For reranking tasks, these are queries (for text only tasks QueryDatasetType).	required
inputs2	DataLoader[BatchedInput]	Second Dataloader of inputs to encode. For reranking, these are documents (for text only tasks RetrievalOutputType).	required
task_metadata	TaskMetadata	Metadata of the current task.	required
hf_split	str	Split of current task, allows to know some additional information about current split. E.g. Current language	required
hf_subset	str	Subset of current task. Similar to hf_split to get more information	required
prompt_type	PromptType | None	The name type of prompt. (query or passage)	None
**kwargs	Unpack[EncodeKwargs]	Additional arguments to pass to the cross-encoder.	{}

Returns:

Type	Description
Array	The predicted relevance scores for each inputs pair.

Source code in mteb/models/models_protocols.py

def predict(
    self,
    inputs1: DataLoader[BatchedInput],
    inputs2: DataLoader[BatchedInput],
    *,
    task_metadata: TaskMetadata,
    hf_split: str,
    hf_subset: str,
    prompt_type: PromptType | None = None,
    **kwargs: Unpack[EncodeKwargs],
) -> Array:
    """Predicts relevance scores for pairs of inputs. Note that, unlike the encoder, the cross-encoder can compare across inputs.

    Args:
        inputs1: First Dataloader of inputs to encode. For reranking tasks, these are queries (for text only tasks `QueryDatasetType`).
        inputs2: Second Dataloader of inputs to encode. For reranking, these are documents (for text only tasks `RetrievalOutputType`).
        task_metadata: Metadata of the current task.
        hf_split: Split of current task, allows to know some additional information about current split.
            E.g. Current language
        hf_subset: Subset of current task. Similar to `hf_split` to get more information
        prompt_type: The name type of prompt. (query or passage)
        **kwargs: Additional arguments to pass to the cross-encoder.

    Returns:
        The predicted relevance scores for each inputs pair.
    """
    ...

mteb.models.MTEBModels = EncoderProtocol | CrossEncoderProtocol | SearchProtocol module-attribute

Type alias for all MTEB model types as many models implement multiple protocols and many tasks can be solved by multiple model types.

mteb.models.IndexEncoderSearchProtocol

Bases: Protocol

Protocol for search backends used in encoder-based retrieval.

Source code in mteb/models/search_encoder_index/search_backend_protocol.py

class IndexEncoderSearchProtocol(Protocol):
    """Protocol for search backends used in encoder-based retrieval."""

    def add_documents(
        self,
        embeddings: Array,
        idxs: list[str],
    ) -> None:
        """Add documents to the search backend.

        Args:
            embeddings: Embeddings of the documents to add.
            idxs: IDs of the documents to add.
        """

    def search(
        self,
        embeddings: Array,
        top_k: int,
        similarity_fn: Callable[[Array, Array], Array],
        top_ranked: TopRankedDocumentsType | None = None,
        query_idx_to_id: dict[int, str] | None = None,
    ) -> tuple[list[list[float]], list[list[int]]]:
        """Search through added corpus embeddings or rerank top-ranked documents.

        Supports both full-corpus and reranking search modes:
            - Full-corpus mode: `top_ranked=None`, uses added corpus embeddings.
            - Reranking mode:  `top_ranked` contains mapping {query_id: [doc_ids]}.

        Args:
            embeddings: Query embeddings, shape (num_queries, dim).
            top_k: Number of top results to return.
            similarity_fn: Function to compute similarity between query and corpus.
            top_ranked: Mapping of query_id -> list of candidate doc_ids. Used for reranking.
            query_idx_to_id: Mapping of query index -> query_id. Used for reranking.

        Returns:
            A tuple (top_k_values, top_k_indices), for each query:
                - top_k_values: List of top-k similarity scores.
                - top_k_indices: List of indices of the top-k documents in the added corpus.
        """

    def clear(self) -> None:
        """Clear all stored documents and embeddings from the backend."""

add_documents(embeddings, idxs)

Add documents to the search backend.

Parameters:

Name	Type	Description	Default
embeddings	Array	Embeddings of the documents to add.	required
idxs	list[str]	IDs of the documents to add.	required

Source code in mteb/models/search_encoder_index/search_backend_protocol.py

def add_documents(
    self,
    embeddings: Array,
    idxs: list[str],
) -> None:
    """Add documents to the search backend.

    Args:
        embeddings: Embeddings of the documents to add.
        idxs: IDs of the documents to add.
    """

clear()

Clear all stored documents and embeddings from the backend.

Source code in mteb/models/search_encoder_index/search_backend_protocol.py

def clear(self) -> None:
    """Clear all stored documents and embeddings from the backend."""

search(embeddings, top_k, similarity_fn, top_ranked=None, query_idx_to_id=None)

Search through added corpus embeddings or rerank top-ranked documents.

Supports both full-corpus and reranking search modes

• Full-corpus mode: top_ranked=None, uses added corpus embeddings.
• Reranking mode: top_ranked contains mapping {query_id: [doc_ids]}.

Parameters:

Name	Type	Description	Default
embeddings	Array	Query embeddings, shape (num_queries, dim).	required
top_k	int	Number of top results to return.	required
similarity_fn	Callable[[Array, Array], Array]	Function to compute similarity between query and corpus.	required
top_ranked	TopRankedDocumentsType | None	Mapping of query_id -> list of candidate doc_ids. Used for reranking.	None
query_idx_to_id	dict[int, str] | None	Mapping of query index -> query_id. Used for reranking.	None

Returns:

Type	Description
tuple[list[list[float]], list[list[int]]]	A tuple (top_k_values, top_k_indices), for each query: - top_k_values: List of top-k similarity scores. - top_k_indices: List of indices of the top-k documents in the added corpus.

Source code in mteb/models/search_encoder_index/search_backend_protocol.py

def search(
    self,
    embeddings: Array,
    top_k: int,
    similarity_fn: Callable[[Array, Array], Array],
    top_ranked: TopRankedDocumentsType | None = None,
    query_idx_to_id: dict[int, str] | None = None,
) -> tuple[list[list[float]], list[list[int]]]:
    """Search through added corpus embeddings or rerank top-ranked documents.

    Supports both full-corpus and reranking search modes:
        - Full-corpus mode: `top_ranked=None`, uses added corpus embeddings.
        - Reranking mode:  `top_ranked` contains mapping {query_id: [doc_ids]}.

    Args:
        embeddings: Query embeddings, shape (num_queries, dim).
        top_k: Number of top results to return.
        similarity_fn: Function to compute similarity between query and corpus.
        top_ranked: Mapping of query_id -> list of candidate doc_ids. Used for reranking.
        query_idx_to_id: Mapping of query index -> query_id. Used for reranking.

    Returns:
        A tuple (top_k_values, top_k_indices), for each query:
            - top_k_values: List of top-k similarity scores.
            - top_k_indices: List of indices of the top-k documents in the added corpus.
    """

mteb.models.CacheBackendProtocol

Bases: Protocol

Protocol for a vector cache map (used to store text/image embeddings).

Implementations may back the cache with different storage backends.

The cache maps an input item (text or image) to its vector embedding, identified by a deterministic hash.

Source code in mteb/models/cache_wrappers/cache_backend_protocol.py

@runtime_checkable
class CacheBackendProtocol(Protocol):
    """Protocol for a vector cache map (used to store text/image embeddings).

    Implementations may back the cache with different storage backends.

    The cache maps an input item (text or image) to its vector embedding,
    identified by a deterministic hash.
    """

    def __init__(self, directory: Path | None = None, **kwargs: Any) -> None:
        """Initialize the cache backend.

        Args:
            directory: Directory path to store cache files.
            **kwargs: Additional backend-specific arguments.
        """

    def add(self, item: list[dict[str, Any]], vectors: Array) -> None:
        """Add a vector to the cache.

        Args:
            item: Input item containing 'text' or 'image'.
            vectors: Embedding vector of shape (dim,) or (1, dim).
        """

    def get_vector(self, item: dict[str, Any]) -> Array | None:
        """Retrieve the cached vector for the given item.

        Args:
            item: Input item.

        Returns:
            Cached vector as np.ndarray, or None if not found.
        """

    def save(self) -> None:
        """Persist cache data to disk (index + metadata)."""

    def load(self) -> None:
        """Load cache from disk (index + metadata)."""

    def close(self) -> None:
        """Release resources or flush data."""

    def __contains__(self, item: dict[str, Any]) -> bool:
        """Check whether the cache contains an item."""

__contains__(item)

Check whether the cache contains an item.

Source code in mteb/models/cache_wrappers/cache_backend_protocol.py

def __contains__(self, item: dict[str, Any]) -> bool:
    """Check whether the cache contains an item."""

__init__(directory=None, **kwargs)

Initialize the cache backend.

Parameters:

Name	Type	Description	Default
directory	Path | None	Directory path to store cache files.	None
**kwargs	Any	Additional backend-specific arguments.	{}

Source code in mteb/models/cache_wrappers/cache_backend_protocol.py

def __init__(self, directory: Path | None = None, **kwargs: Any) -> None:
    """Initialize the cache backend.

    Args:
        directory: Directory path to store cache files.
        **kwargs: Additional backend-specific arguments.
    """

add(item, vectors)

Add a vector to the cache.

Parameters:

Name	Type	Description	Default
item	list[dict[str, Any]]	Input item containing 'text' or 'image'.	required
vectors	Array	Embedding vector of shape (dim,) or (1, dim).	required

Source code in mteb/models/cache_wrappers/cache_backend_protocol.py

def add(self, item: list[dict[str, Any]], vectors: Array) -> None:
    """Add a vector to the cache.

    Args:
        item: Input item containing 'text' or 'image'.
        vectors: Embedding vector of shape (dim,) or (1, dim).
    """

close()

Release resources or flush data.

Source code in mteb/models/cache_wrappers/cache_backend_protocol.py

def close(self) -> None:
    """Release resources or flush data."""

get_vector(item)

Retrieve the cached vector for the given item.

Parameters:

Name	Type	Description	Default
item	dict[str, Any]	Input item.	required

Returns:

Type	Description
Array | None	Cached vector as np.ndarray, or None if not found.

Source code in mteb/models/cache_wrappers/cache_backend_protocol.py

def get_vector(self, item: dict[str, Any]) -> Array | None:
    """Retrieve the cached vector for the given item.

    Args:
        item: Input item.

    Returns:
        Cached vector as np.ndarray, or None if not found.
    """

load()

Load cache from disk (index + metadata).

Source code in mteb/models/cache_wrappers/cache_backend_protocol.py

def load(self) -> None:
    """Load cache from disk (index + metadata)."""

save()

Persist cache data to disk (index + metadata).

Source code in mteb/models/cache_wrappers/cache_backend_protocol.py

def save(self) -> None:
    """Persist cache data to disk (index + metadata)."""