Models

A model in mteb covers two concepts: metadata and implementation. - Metadata contains information about the model such as maximum input length, valid frameworks, license, and degree of openness. - Implementation is a reproducible workflow, which allows others to run the same model again, using the same prompts, hyperparameters, aggregation strategies, etc.

An overview of the model and its metadata within mteb

Utilities

mteb.get_model_metas(model_names=None, languages=None, open_weights=None, frameworks=None, n_parameters_range=(None, None), use_instructions=None, zero_shot_on=None, model_types=None)

Load all models' metadata that fit the specified criteria.

Parameters:

Name	Type	Description	Default
model_names	Iterable[str] | None	A list of model names to filter by. If None, all models are included.	None
languages	Iterable[str] | None	A list of languages to filter by. If None, all languages are included.	None
open_weights	bool | None	Whether to filter by models with open weights. If None this filter is ignored.	None
frameworks	Iterable[str] | None	A list of frameworks to filter by. If None, all frameworks are included.	None
n_parameters_range	tuple[int | None, int | None]	A tuple of lower and upper bounds of the number of parameters to filter by. If (None, None), this filter is ignored.	(None, None)
use_instructions	bool | None	Whether to filter by models that use instructions. If None, all models are included.	None
zero_shot_on	list[AbsTask] | None	A list of tasks on which the model is zero-shot. If None this filter is ignored.	None
model_types	Iterable[str] | None	A list of model types to filter by. If None, all model types are included.	None

Returns:

Type	Description
list[ModelMeta]	A list of model metadata objects that fit the specified criteria.

Source code in mteb/models/get_model_meta.py

def get_model_metas(
    model_names: Iterable[str] | None = None,
    languages: Iterable[str] | None = None,
    open_weights: bool | None = None,
    frameworks: Iterable[str] | None = None,
    n_parameters_range: tuple[int | None, int | None] = (None, None),
    use_instructions: bool | None = None,
    zero_shot_on: list[AbsTask] | None = None,
    model_types: Iterable[str] | None = None,
) -> list[ModelMeta]:
    """Load all models' metadata that fit the specified criteria.

    Args:
        model_names: A list of model names to filter by. If None, all models are included.
        languages: A list of languages to filter by. If None, all languages are included.
        open_weights: Whether to filter by models with open weights. If None this filter is ignored.
        frameworks: A list of frameworks to filter by. If None, all frameworks are included.
        n_parameters_range: A tuple of lower and upper bounds of the number of parameters to filter by.
            If (None, None), this filter is ignored.
        use_instructions: Whether to filter by models that use instructions. If None, all models are included.
        zero_shot_on: A list of tasks on which the model is zero-shot. If None this filter is ignored.
        model_types: A list of model types to filter by. If None, all model types are included.

    Returns:
        A list of model metadata objects that fit the specified criteria.
    """
    res = []
    model_names = set(model_names) if model_names is not None else None
    languages = set(languages) if languages is not None else None
    frameworks = set(frameworks) if frameworks is not None else None
    model_types_set = set(model_types) if model_types is not None else None
    for model_meta in MODEL_REGISTRY.values():
        if (model_names is not None) and (model_meta.name not in model_names):
            continue
        if languages is not None:
            if (model_meta.languages is None) or not (
                languages <= set(model_meta.languages)
            ):
                continue
        if (open_weights is not None) and (model_meta.open_weights != open_weights):
            continue
        if (frameworks is not None) and not (frameworks <= set(model_meta.framework)):
            continue
        if (use_instructions is not None) and (
            model_meta.use_instructions != use_instructions
        ):
            continue
        if model_types_set is not None and not model_types_set.intersection(
            model_meta.model_type
        ):
            continue

        lower, upper = n_parameters_range
        n_parameters = model_meta.n_parameters

        if upper is not None:
            if (n_parameters is None) or (n_parameters > upper):
                continue
            if lower is not None and n_parameters < lower:
                continue

        if zero_shot_on is not None:
            if not model_meta.is_zero_shot_on(zero_shot_on):
                continue
        res.append(model_meta)
    return res

mteb.get_model_meta(model_name, revision=None, fetch_from_hf=True)

A function to fetch a model metadata object by name.

Parameters:

Name	Type	Description	Default
model_name	str	Name of the model to fetch	required
revision	str | None	Revision of the model to fetch	None
fetch_from_hf	bool	Whether to fetch the model from HuggingFace Hub if not found in the registry	True

Returns:

Type	Description
ModelMeta	A model metadata object

Source code in mteb/models/get_model_meta.py

def get_model_meta(
    model_name: str, revision: str | None = None, fetch_from_hf: bool = True
) -> ModelMeta:
    """A function to fetch a model metadata object by name.

    Args:
        model_name: Name of the model to fetch
        revision: Revision of the model to fetch
        fetch_from_hf: Whether to fetch the model from HuggingFace Hub if not found in the registry

    Returns:
        A model metadata object
    """
    if model_name in MODEL_REGISTRY:
        model_meta = MODEL_REGISTRY[model_name]

        if revision and (not model_meta.revision == revision):
            raise ValueError(
                f"Model revision {revision} not found for model {model_name}. Expected {model_meta.revision}."
            )
        return model_meta
    if fetch_from_hf:
        logger.info(
            "Model not found in model registry. Attempting to extract metadata by loading the model ({model_name}) using HuggingFace."
        )
        meta = ModelMeta.from_hub(model_name, revision)
        return meta

    not_found_msg = f"Model '{model_name}' not found in MTEB registry"
    not_found_msg += " nor on the Huggingface Hub." if fetch_from_hf else "."

    close_matches = difflib.get_close_matches(model_name, MODEL_REGISTRY.keys())
    model_names_no_org = {mdl: mdl.split("/")[-1] for mdl in MODEL_REGISTRY.keys()}
    if model_name in model_names_no_org:
        close_matches = [model_names_no_org[model_name]] + close_matches

    suggestion = ""
    if close_matches:
        if len(close_matches) > 1:
            suggestion = f" Did you mean: '{close_matches[0]}' or {close_matches[1]}?"
        else:
            suggestion = f" Did you mean: '{close_matches[0]}'?"

    raise KeyError(not_found_msg + suggestion)

mteb.get_model(model_name, revision=None, device=None, **kwargs)

A function to fetch and load model object by name.

Note

This function loads the model into memory. If you only want to fetch the metadata, use get_model_meta instead.

Parameters:

Name	Type	Description	Default
model_name	str	Name of the model to fetch	required
revision	str | None	Revision of the model to fetch	None
device	str | None	Device used to load the model	None
**kwargs	Any	Additional keyword arguments to pass to the model loader	{}

Returns:

Type	Description
MTEBModels	A model object

Source code in mteb/models/get_model_meta.py

def get_model(
    model_name: str,
    revision: str | None = None,
    device: str | None = None,
    **kwargs: Any,
) -> MTEBModels:
    """A function to fetch and load model object by name.

    !!! note
        This function loads the model into memory. If you only want to fetch the metadata, use [`get_model_meta`](#mteb.get_model_meta) instead.

    Args:
        model_name: Name of the model to fetch
        revision: Revision of the model to fetch
        device: Device used to load the model
        **kwargs: Additional keyword arguments to pass to the model loader

    Returns:
        A model object
    """
    meta = get_model_meta(model_name, revision)
    model = meta.load_model(device=device, **kwargs)

    if kwargs:
        logger.info(
            f"Model '{model_name}' loaded with additional arguments: {list(kwargs.keys())}"
        )
        meta = meta.model_copy(deep=True)
        meta.loader_kwargs |= kwargs

    model.mteb_model_meta = meta  # type: ignore[misc]
    return model

Metadata

mteb.models.model_meta.ModelMeta

Bases: BaseModel

The model metadata object.

Attributes:

Name	Type	Description
loader	Callable[..., MTEBModels] | None	The function that loads the model. If None it assumes that the model is not implemented.
loader_kwargs	dict[str, Any]	The keyword arguments to pass to the loader function.
name	str | None	The name of the model, ideally the name on huggingface. It should be in the format "organization/model_name".
n_parameters	int | None	The number of parameters in the model, e.g. 7_000_000 for a 7M parameter model. Can be None if the number of parameters is not known (e.g. for proprietary models) or if the loader returns a SentenceTransformer model from which it can be derived.
memory_usage_mb	float | None	The memory usage of the model in MB. Can be None if the memory usage is not known (e.g. for proprietary models). To calculate it use the calculate_memory_usage_mb method.
max_tokens	float | None	The maximum number of tokens the model can handle. Can be None if the maximum number of tokens is not known (e.g. for proprietary models).
embed_dim	int | None	The dimension of the embeddings produced by the model. Currently all models are assumed to produce fixed-size embeddings.
revision	str | None	The revision number of the model. If None, it is assumed that the metadata (including the loader) is valid for all revisions of the model.
release_date	StrDate | None	The date the model's revision was released. If None, then release date will be added based on 1st commit in hf repository of model.
license	Licenses | StrURL | None	The license under which the model is released. Required if open_weights is True.
open_weights	bool | None	Whether the model is open source or proprietary.
public_training_code	str | None	A link to the publicly available training code. If None, it is assumed that the training code is not publicly available.
public_training_data	str | bool | None	A link to the publicly available training data. If None, it is assumed that the training data is not publicly available.
similarity_fn_name	ScoringFunction | None	The distance metric used by the model.
framework	list[FRAMEWORKS]	The framework the model is implemented in, can be a list of frameworks e.g. ["Sentence Transformers", "PyTorch"].
reference	StrURL | None	A URL to the model's page on huggingface or another source.
languages	list[ISOLanguageScript] | None	The languages the model is intended to be specified as a 3-letter language code followed by a script code e.g., "eng-Latn" for English in the Latin script.
use_instructions	bool | None	Whether the model uses instructions E.g. for prompt-based models. This also includes models that require a specific format for input, such as "query: {document}" or "passage: {document}".
citation	str | None	The citation for the model. This is a bibtex string.
training_datasets	set[str] | None	A dictionary of datasets that the model was trained on. Names should be names as their appear in mteb for example {"ArguAna"} if the model is trained on the ArguAna test set. This field is used to determine if a model generalizes zero-shot to a benchmark as well as mark dataset contaminations.
adapted_from	str | None	Name of the model from which this model is adapted. For quantizations, fine-tunes, long doc extensions, etc.
superseded_by	str | None	Name of the model that supersedes this model, e.g., nvidia/NV-Embed-v2 supersedes v1.
model_type	list[MODEL_TYPES]	A list of strings representing the type of model.
modalities	list[Modalities]	A list of strings representing the modalities the model supports. Default is ["text"].
contacts	list[str] | None	The people to contact in case of a problem in the model, preferably a GitHub handle.

Source code in mteb/models/model_meta.py

class ModelMeta(BaseModel):
    """The model metadata object.

    Attributes:
        loader: The function that loads the model. If None it assumes that the model is not implemented.
        loader_kwargs: The keyword arguments to pass to the loader function.
        name: The name of the model, ideally the name on huggingface. It should be in the format "organization/model_name".
        n_parameters: The number of parameters in the model, e.g. 7_000_000 for a 7M parameter model. Can be None if the number of parameters is not known (e.g. for proprietary models) or
            if the loader returns a SentenceTransformer model from which it can be derived.
        memory_usage_mb: The memory usage of the model in MB. Can be None if the memory usage is not known (e.g. for proprietary models). To calculate it use the `calculate_memory_usage_mb` method.
        max_tokens: The maximum number of tokens the model can handle. Can be None if the maximum number of tokens is not known (e.g. for proprietary
            models).
        embed_dim: The dimension of the embeddings produced by the model. Currently all models are assumed to produce fixed-size embeddings.
        revision: The revision number of the model. If None, it is assumed that the metadata (including the loader) is valid for all revisions of the model.
        release_date: The date the model's revision was released. If None, then release date will be added based on 1st commit in hf repository of model.
        license: The license under which the model is released. Required if open_weights is True.
        open_weights: Whether the model is open source or proprietary.
        public_training_code: A link to the publicly available training code. If None, it is assumed that the training code is not publicly available.
        public_training_data: A link to the publicly available training data. If None, it is assumed that the training data is not publicly available.
        similarity_fn_name: The distance metric used by the model.
        framework: The framework the model is implemented in, can be a list of frameworks e.g. `["Sentence Transformers", "PyTorch"]`.
        reference: A URL to the model's page on huggingface or another source.
        languages: The languages the model is intended to be specified as a 3-letter language code followed by a script code e.g., "eng-Latn" for English
            in the Latin script.
        use_instructions: Whether the model uses instructions E.g. for prompt-based models. This also includes models that require a specific format for
            input, such as "query: {document}" or "passage: {document}".
        citation: The citation for the model. This is a bibtex string.
        training_datasets: A dictionary of datasets that the model was trained on. Names should be names as their appear in `mteb` for example
            {"ArguAna"} if the model is trained on the ArguAna test set. This field is used to determine if a model generalizes zero-shot to
            a benchmark as well as mark dataset contaminations.
        adapted_from: Name of the model from which this model is adapted. For quantizations, fine-tunes, long doc extensions, etc.
        superseded_by: Name of the model that supersedes this model, e.g., nvidia/NV-Embed-v2 supersedes v1.
        model_type: A list of strings representing the type of model.
        modalities: A list of strings representing the modalities the model supports. Default is ["text"].
        contacts: The people to contact in case of a problem in the model, preferably a GitHub handle.
    """

    model_config = ConfigDict(extra="forbid")

    # loaders
    loader: Callable[..., MTEBModels] | None
    loader_kwargs: dict[str, Any] = field(default_factory=dict)
    name: str | None
    revision: str | None
    release_date: StrDate | None
    languages: list[ISOLanguageScript] | None
    n_parameters: int | None
    memory_usage_mb: float | None
    max_tokens: float | None
    embed_dim: int | None
    license: Licenses | StrURL | None
    open_weights: bool | None
    public_training_code: str | None
    public_training_data: str | bool | None
    framework: list[FRAMEWORKS]
    reference: StrURL | None = None
    similarity_fn_name: ScoringFunction | None
    use_instructions: bool | None
    training_datasets: set[str] | None
    adapted_from: str | None = None
    superseded_by: str | None = None
    modalities: list[Modalities] = ["text"]
    model_type: list[MODEL_TYPES] = ["dense"]
    citation: str | None = None
    contacts: list[str] | None = None

    @model_validator(mode="before")
    @classmethod
    def handle_legacy_is_cross_encoder(cls, data: Any) -> Any:
        """Handle legacy is_cross_encoder field by converting it to model_type.

        This validator handles backward compatibility for the deprecated is_cross_encoder field.
        If is_cross_encoder=True is provided, it adds "cross_encoder" to model_type.
        """
        if isinstance(data, dict) and "is_cross_encoder" in data:
            is_cross_encoder_value = data.pop("is_cross_encoder")

            if is_cross_encoder_value is not None:
                warnings.warn(
                    "is_cross_encoder is deprecated and will be removed in a future version. "
                    "Use model_type=['cross-encoder'] instead.",
                    DeprecationWarning,
                    stacklevel=2,
                )

                model_type = data.get("model_type", ["dense"])

                if is_cross_encoder_value:
                    if "cross-encoder" not in model_type:
                        data["model_type"] = ["cross-encoder"]
                else:
                    if "cross-encoder" in model_type:
                        model_type = [t for t in model_type if t != "cross-encoder"]
                        data["model_type"] = model_type if model_type else ["dense"]

        return data

    @property
    def is_cross_encoder(self) -> bool:
        """Returns True if the model is a cross-encoder.

        Derived from model_type field. A model is considered a cross-encoder if "cross-encoder" is in its model_type list.
        """
        return "cross-encoder" in self.model_type

    @field_validator("similarity_fn_name", mode="before")
    @classmethod
    def _validate_similarity_fn_name(cls, value: str) -> ScoringFunction | None:
        """Converts the similarity function name to the corresponding enum value.

        Sentence_transformers uses Literal['cosine', 'dot', 'euclidean', 'manhattan'],
        and pylate uses Literal['MaxSim']

        Args:
            value: The similarity function name as a string.

        Returns:
            The corresponding ScoringFunction enum value.
        """
        if type(value) is ScoringFunction or value is None:
            return value
        mapping = {
            "cosine": ScoringFunction.COSINE,
            "dot": ScoringFunction.DOT_PRODUCT,
            "MaxSim": ScoringFunction.MAX_SIM,
        }
        if value in mapping:
            return mapping[value]
        raise ValueError(f"Invalid similarity function name: {value}")

    def to_dict(self):
        """Returns a dictionary representation of the model metadata."""
        dict_repr = self.model_dump()
        loader = dict_repr.pop("loader", None)
        dict_repr["training_datasets"] = (
            list(dict_repr["training_datasets"])
            if isinstance(dict_repr["training_datasets"], set)
            else dict_repr["training_datasets"]
        )
        dict_repr["loader"] = _get_loader_name(loader)
        dict_repr["is_cross_encoder"] = self.is_cross_encoder
        return dict_repr

    @field_validator("languages")
    @classmethod
    def _languages_are_valid(
        cls, languages: list[ISOLanguageScript] | None
    ) -> list[ISOLanguageScript] | None:
        if languages is None:
            return None

        for code in languages:
            check_language_code(code)
        return languages

    @field_validator("name")
    @classmethod
    def _check_name(cls, v: str | None) -> str | None:
        if v is None or v in ("bm25s", "Human"):
            return v
        if "/" not in v:
            raise ValueError(
                "Model name must be in the format 'organization/model_name'"
            )
        return v

    def load_model(self, device: str | None = None, **kwargs: Any) -> MTEBModels:
        """Loads the model using the specified loader function."""
        if self.loader is None:
            raise NotImplementedError(
                "No model implementation is available for this model."
            )
        if self.name is None:
            raise ValueError("name is not set for ModelMeta. Cannot load model.")

        # Allow overwrites
        _kwargs = self.loader_kwargs.copy()
        _kwargs.update(kwargs)
        if device is not None:
            _kwargs["device"] = device

        model: MTEBModels = self.loader(self.name, revision=self.revision, **_kwargs)
        model.mteb_model_meta = self  # type: ignore[misc]
        return model

    def model_name_as_path(self) -> str:
        """Returns the model name in a format that can be used as a file path.

        Replaces "/" with "__" and spaces with "_".
        """
        if self.name is None:
            raise ValueError("Model name is not set")
        return self.name.replace("/", "__").replace(" ", "_")

    @classmethod
    def _from_hub(
        cls,
        model_name: str | None,
        revision: str | None = None,
        compute_metadata: bool = True,
    ) -> Self:
        """Generates a ModelMeta from a HuggingFace model name.

        Args:
            model_name: The HuggingFace model name.
            revision: Revision of the model
            compute_metadata: Add metadata based on model card

        Returns:
            The generated ModelMeta.
        """
        from mteb.models import sentence_transformers_loader

        loader = sentence_transformers_loader
        frameworks: list[FRAMEWORKS] = ["PyTorch"]
        model_license = None
        reference = None
        n_parameters = None
        memory_usage_mb = None
        release_date = None
        embedding_dim = None
        max_tokens = None

        if model_name and compute_metadata and _repo_exists(model_name):
            reference = "https://huggingface.co/" + model_name
            card = ModelCard.load(model_name)
            card_data: ModelCardData = card.data
            try:
                model_config = AutoConfig.from_pretrained(model_name)
            except Exception as e:
                # some models can't load AutoConfig (e.g. `average_word_embeddings_levy_dependency`)
                model_config = None
                logger.warning(f"Can't get configuration for {model_name}. Error: {e}")

            hf_frameworks = (
                cls._get_frameworks_from_hf_tags(model_name) if model_name else []
            )
            frameworks.extend(hf_frameworks)

            if revision is None:
                revisions = _get_repo_commits(model_name, "model")
                revision = revisions[0].commit_id if revisions else None

            release_date = cls.fetch_release_date(model_name)
            model_license = card_data.license
            n_parameters = cls._calculate_num_parameters_from_hub(model_name)
            memory_usage_mb = cls._calculate_memory_usage_mb(model_name, n_parameters)
            if model_config and hasattr(model_config, "hidden_size"):
                embedding_dim = model_config.hidden_size
            if model_config and hasattr(model_config, "max_position_embeddings"):
                max_tokens = model_config.max_position_embeddings

        return cls(
            loader=loader,
            name=model_name or "no_model_name/available",
            revision=revision or "no_revision_available",
            reference=reference,
            release_date=release_date,
            languages=None,
            license=model_license,
            framework=frameworks,
            training_datasets=None,
            similarity_fn_name=None,
            n_parameters=n_parameters,
            memory_usage_mb=memory_usage_mb,
            max_tokens=max_tokens,
            embed_dim=embedding_dim,
            open_weights=True,
            public_training_code=None,
            public_training_data=None,
            use_instructions=None,
            modalities=[],
        )

    @classmethod
    def from_sentence_transformer_model(
        cls,
        model: SentenceTransformer,
        revision: str | None = None,
        compute_metadata: bool = True,
    ) -> Self:
        """Generates a ModelMeta from a SentenceTransformer model.

        Args:
            model: SentenceTransformer model.
            revision: Revision of the model
            compute_metadata: Add metadata based on model card

        Returns:
            The generated ModelMeta.
        """
        name: str | None = (
            model.model_card_data.model_name
            if model.model_card_data.model_name
            else model.model_card_data.base_model
        )
        meta = cls._from_hub(name, revision, compute_metadata)
        meta.revision = model.model_card_data.base_model_revision or meta.revision
        meta.max_tokens = model.max_seq_length
        meta.embed_dim = model.get_sentence_embedding_dimension()
        meta.similarity_fn_name = ScoringFunction.from_str(model.similarity_fn_name)
        meta.modalities = ["text"]
        return meta

    @classmethod
    def from_hub(
        cls,
        model: str,
        revision: str | None = None,
        compute_metadata: bool = True,
    ) -> Self:
        """Generates a ModelMeta for model from HuggingFace hub.

        Args:
            model: Name of the model from HuggingFace hub. For example, `intfloat/multilingual-e5-large`
            revision: Revision of the model
            compute_metadata: Add metadata based on model card

        Returns:
            The generated ModelMeta.
        """
        meta = cls._from_hub(model, revision, compute_metadata)
        meta.modalities = ["text"]

        if model and compute_metadata and _repo_exists(model):
            # have max_seq_length field
            sbert_config = _get_json_from_hub(
                model, "sentence_bert_config.json", "model", revision=revision
            )
            if sbert_config:
                meta.max_tokens = (
                    sbert_config.get("max_seq_length", None) or meta.max_tokens
                )
            # have model type, similarity function fields
            config_sbert = _get_json_from_hub(
                model, "config_sentence_transformers.json", "model", revision=revision
            )
            if (
                config_sbert is not None
                and config_sbert.get("similarity_fn_name") is not None
            ):
                meta.similarity_fn_name = ScoringFunction.from_str(
                    config_sbert["similarity_fn_name"]
                )
            else:
                meta.similarity_fn_name = ScoringFunction.COSINE
        return meta

    @classmethod
    def from_cross_encoder(
        cls,
        model: CrossEncoder,
        revision: str | None = None,
        compute_metadata: bool = True,
    ) -> Self:
        """Generates a ModelMeta from a CrossEncoder.

        Args:
            model: The CrossEncoder model
            revision: Revision of the model
            compute_metadata: Add metadata based on model card

        Returns:
            The generated ModelMeta
        """
        from mteb.models import CrossEncoderWrapper

        meta = cls._from_hub(model.model.name_or_path, revision, compute_metadata)
        meta.revision = model.config._commit_hash or meta.revision
        meta.loader = CrossEncoderWrapper
        meta.embed_dim = None
        meta.modalities = ["text"]
        meta.model_type = ["cross-encoder"]
        return meta

    def is_zero_shot_on(self, tasks: Sequence[AbsTask] | Sequence[str]) -> bool | None:
        """Indicates whether the given model can be considered zero-shot or not on the given tasks.

        Returns:
             None if no training data is specified on the model.
        """
        # If no tasks were specified, we're obviously zero-shot
        if not tasks:
            return True
        training_datasets = self.get_training_datasets()
        # If no tasks were specified, we're obviously zero-shot
        if training_datasets is None:
            return None

        if isinstance(tasks[0], str):
            benchmark_datasets = set(tasks)
        else:
            tasks = cast(Sequence["AbsTask"], tasks)
            benchmark_datasets = set()
            for task in tasks:
                benchmark_datasets.add(task.metadata.name)
        intersection = training_datasets & benchmark_datasets
        return len(intersection) == 0

    def get_training_datasets(self) -> set[str] | None:
        """Returns all training datasets of the model including similar tasks."""
        import mteb

        if self.training_datasets is None:
            return None

        training_datasets = self.training_datasets.copy()
        if self.adapted_from is not None:
            try:
                adapted_from_model = mteb.get_model_meta(
                    self.adapted_from, fetch_from_hf=False
                )
                adapted_training_datasets = adapted_from_model.get_training_datasets()
                if adapted_training_datasets is not None:
                    training_datasets |= adapted_training_datasets
            except (ValueError, KeyError) as e:
                msg = f"Could not get source model: {e} in MTEB"
                logger.warning(msg)
                warnings.warn(msg)

        return_dataset = training_datasets.copy()
        visited: set[str] = set()

        for dataset in training_datasets:
            similar_tasks = _collect_similar_tasks(dataset, visited)
            return_dataset |= similar_tasks

        return return_dataset

    def zero_shot_percentage(
        self, tasks: Sequence[AbsTask] | Sequence[str]
    ) -> int | None:
        """Indicates how out-of-domain the selected tasks are for the given model.

        Args:
            tasks: A sequence of tasks or dataset names to evaluate against.

        Returns:
            An integer percentage (0-100) indicating how out-of-domain the tasks are for the model.
            Returns None if no training data is specified on the model or if no tasks are provided.
        """
        training_datasets = self.get_training_datasets()
        if (training_datasets is None) or (not tasks):
            return None
        if isinstance(tasks[0], str):
            benchmark_datasets = set(tasks)
        else:
            tasks = cast(Sequence["AbsTask"], tasks)
            benchmark_datasets = {task.metadata.name for task in tasks}
        overlap = training_datasets & benchmark_datasets
        perc_overlap = 100 * (len(overlap) / len(benchmark_datasets))
        return int(100 - perc_overlap)

    @staticmethod
    def _calculate_num_parameters_from_hub(model_name: str | None = None) -> int | None:
        if not model_name:
            return None
        try:
            safetensors_metadata = get_safetensors_metadata(model_name)
            if len(safetensors_metadata.parameter_count) >= 0:
                return sum(safetensors_metadata.parameter_count.values())
        except (
            NotASafetensorsRepoError,
            SafetensorsParsingError,
            GatedRepoError,
            RepositoryNotFoundError,
        ) as e:
            logger.warning(
                f"Can't calculate number of parameters for {model_name}. Got error {e}"
            )
        return None

    def calculate_num_parameters_from_hub(self) -> int | None:
        """Calculates the number of parameters in the model.

        Returns:
            Number of parameters in the model.
        """
        return self._calculate_num_parameters_from_hub(self.name)

    @staticmethod
    def _calculate_memory_usage_mb(
        model_name: str, n_parameters: int | None
    ) -> int | None:
        MB = 1024**2  # noqa: N806
        try:
            safetensors_metadata = get_safetensors_metadata(model_name)
            if len(safetensors_metadata.parameter_count) >= 0:
                dtype_size_map = {
                    "F64": 8,  # 64-bit float
                    "F32": 4,  # 32-bit float (FP32)
                    "F16": 2,  # 16-bit float (FP16)
                    "BF16": 2,  # BFloat16
                    "I64": 8,  # 64-bit integer
                    "I32": 4,  # 32-bit integer
                    "I16": 2,  # 16-bit integer
                    "I8": 1,  # 8-bit integer
                    "U8": 1,  # Unsigned 8-bit integer
                    "BOOL": 1,  # Boolean (assuming 1 byte per value)
                }
                total_memory_bytes = sum(
                    parameters * dtype_size_map.get(dtype, 4)
                    for dtype, parameters in safetensors_metadata.parameter_count.items()
                )
                return round(total_memory_bytes / MB)  # Convert to MB
        except (
            NotASafetensorsRepoError,
            SafetensorsParsingError,
            GatedRepoError,
            RepositoryNotFoundError,
        ) as e:
            logger.warning(
                f"Can't calculate memory usage for {model_name}. Got error {e}"
            )

        if n_parameters is None:
            return None
        # Model memory in bytes. For FP32 each parameter is 4 bytes.
        model_memory_bytes = n_parameters * 4

        # Convert to MB
        model_memory_mb = model_memory_bytes / MB
        return round(model_memory_mb)

    def calculate_memory_usage_mb(self) -> int | None:
        """Calculates the memory usage of the model in MB.

        Returns:
            The memory usage of the model in MB, or None if it cannot be determined.
        """
        if "API" in self.framework or self.name is None:
            return None

        return self._calculate_memory_usage_mb(self.name, self.n_parameters)

    @staticmethod
    def fetch_release_date(model_name: str) -> StrDate | None:
        """Fetches the release date from HuggingFace Hub based on the first commit.

        Returns:
            The release date in YYYY-MM-DD format, or None if it cannot be determined.
        """
        commits = _get_repo_commits(repo_id=model_name, repo_type="model")
        if commits:
            initial_commit = commits[-1]
            release_date = initial_commit.created_at.strftime("%Y-%m-%d")
            return release_date
        return None

    @staticmethod
    def _get_frameworks_from_hf_tags(model_name: str) -> list[FRAMEWORKS]:
        """Extract frameworks supported by the model from HuggingFace model tags.

        Args:
            model_name: HuggingFace model name

        Returns:
            List of framework names found in tags. Defaults to empty list if no frameworks found.
        """
        try:
            info = model_info(model_name)
            if not info.tags:
                return []
        except Exception as e:
            logger.warning(
                f"Failed to fetch frameworks from HuggingFace tags for {model_name}: {e}"
            )
            return []

        # Mapping from HuggingFace tags to MTEB framework names
        tag_to_framework: dict[str, FRAMEWORKS] = {
            "sentence-transformers": "Sentence Transformers",
            "transformers": "Transformers",
            "onnx": "ONNX",
            "safetensors": "safetensors",
            "gguf": "GGUF",
        }

        frameworks: list[FRAMEWORKS] = []

        for framework_tag in tag_to_framework.keys():
            if framework_tag in info.tags:
                frameworks.append(tag_to_framework[framework_tag])

        return frameworks

    def to_python(self) -> str:
        """Returns a string representation of the model."""
        return _pydantic_instance_to_code(self)

is_cross_encoder property

Returns True if the model is a cross-encoder.

Derived from model_type field. A model is considered a cross-encoder if "cross-encoder" is in its model_type list.

calculate_memory_usage_mb()

Calculates the memory usage of the model in MB.

Returns:

Type	Description
int | None	The memory usage of the model in MB, or None if it cannot be determined.

Source code in mteb/models/model_meta.py

def calculate_memory_usage_mb(self) -> int | None:
    """Calculates the memory usage of the model in MB.

    Returns:
        The memory usage of the model in MB, or None if it cannot be determined.
    """
    if "API" in self.framework or self.name is None:
        return None

    return self._calculate_memory_usage_mb(self.name, self.n_parameters)

calculate_num_parameters_from_hub()

Calculates the number of parameters in the model.

Returns:

Type	Description
int | None	Number of parameters in the model.

Source code in mteb/models/model_meta.py

def calculate_num_parameters_from_hub(self) -> int | None:
    """Calculates the number of parameters in the model.

    Returns:
        Number of parameters in the model.
    """
    return self._calculate_num_parameters_from_hub(self.name)

fetch_release_date(model_name) staticmethod

Fetches the release date from HuggingFace Hub based on the first commit.

Returns:

Type	Description
StrDate | None	The release date in YYYY-MM-DD format, or None if it cannot be determined.

Source code in mteb/models/model_meta.py

@staticmethod
def fetch_release_date(model_name: str) -> StrDate | None:
    """Fetches the release date from HuggingFace Hub based on the first commit.

    Returns:
        The release date in YYYY-MM-DD format, or None if it cannot be determined.
    """
    commits = _get_repo_commits(repo_id=model_name, repo_type="model")
    if commits:
        initial_commit = commits[-1]
        release_date = initial_commit.created_at.strftime("%Y-%m-%d")
        return release_date
    return None

from_cross_encoder(model, revision=None, compute_metadata=True) classmethod

Generates a ModelMeta from a CrossEncoder.

Parameters:

Name	Type	Description	Default
model	CrossEncoder	The CrossEncoder model	required
revision	str | None	Revision of the model	None
compute_metadata	bool	Add metadata based on model card	True

Returns:

Type	Description
Self	The generated ModelMeta

Source code in mteb/models/model_meta.py

@classmethod
def from_cross_encoder(
    cls,
    model: CrossEncoder,
    revision: str | None = None,
    compute_metadata: bool = True,
) -> Self:
    """Generates a ModelMeta from a CrossEncoder.

    Args:
        model: The CrossEncoder model
        revision: Revision of the model
        compute_metadata: Add metadata based on model card

    Returns:
        The generated ModelMeta
    """
    from mteb.models import CrossEncoderWrapper

    meta = cls._from_hub(model.model.name_or_path, revision, compute_metadata)
    meta.revision = model.config._commit_hash or meta.revision
    meta.loader = CrossEncoderWrapper
    meta.embed_dim = None
    meta.modalities = ["text"]
    meta.model_type = ["cross-encoder"]
    return meta

from_hub(model, revision=None, compute_metadata=True) classmethod

Generates a ModelMeta for model from HuggingFace hub.

Parameters:

Name	Type	Description	Default
model	str	Name of the model from HuggingFace hub. For example, intfloat/multilingual-e5-large	required
revision	str | None	Revision of the model	None
compute_metadata	bool	Add metadata based on model card	True

Returns:

Type	Description
Self	The generated ModelMeta.

Source code in mteb/models/model_meta.py

@classmethod
def from_hub(
    cls,
    model: str,
    revision: str | None = None,
    compute_metadata: bool = True,
) -> Self:
    """Generates a ModelMeta for model from HuggingFace hub.

    Args:
        model: Name of the model from HuggingFace hub. For example, `intfloat/multilingual-e5-large`
        revision: Revision of the model
        compute_metadata: Add metadata based on model card

    Returns:
        The generated ModelMeta.
    """
    meta = cls._from_hub(model, revision, compute_metadata)
    meta.modalities = ["text"]

    if model and compute_metadata and _repo_exists(model):
        # have max_seq_length field
        sbert_config = _get_json_from_hub(
            model, "sentence_bert_config.json", "model", revision=revision
        )
        if sbert_config:
            meta.max_tokens = (
                sbert_config.get("max_seq_length", None) or meta.max_tokens
            )
        # have model type, similarity function fields
        config_sbert = _get_json_from_hub(
            model, "config_sentence_transformers.json", "model", revision=revision
        )
        if (
            config_sbert is not None
            and config_sbert.get("similarity_fn_name") is not None
        ):
            meta.similarity_fn_name = ScoringFunction.from_str(
                config_sbert["similarity_fn_name"]
            )
        else:
            meta.similarity_fn_name = ScoringFunction.COSINE
    return meta

from_sentence_transformer_model(model, revision=None, compute_metadata=True) classmethod

Generates a ModelMeta from a SentenceTransformer model.

Parameters:

Name	Type	Description	Default
model	SentenceTransformer	SentenceTransformer model.	required
revision	str | None	Revision of the model	None
compute_metadata	bool	Add metadata based on model card	True

Returns:

Type	Description
Self	The generated ModelMeta.

Source code in mteb/models/model_meta.py

@classmethod
def from_sentence_transformer_model(
    cls,
    model: SentenceTransformer,
    revision: str | None = None,
    compute_metadata: bool = True,
) -> Self:
    """Generates a ModelMeta from a SentenceTransformer model.

    Args:
        model: SentenceTransformer model.
        revision: Revision of the model
        compute_metadata: Add metadata based on model card

    Returns:
        The generated ModelMeta.
    """
    name: str | None = (
        model.model_card_data.model_name
        if model.model_card_data.model_name
        else model.model_card_data.base_model
    )
    meta = cls._from_hub(name, revision, compute_metadata)
    meta.revision = model.model_card_data.base_model_revision or meta.revision
    meta.max_tokens = model.max_seq_length
    meta.embed_dim = model.get_sentence_embedding_dimension()
    meta.similarity_fn_name = ScoringFunction.from_str(model.similarity_fn_name)
    meta.modalities = ["text"]
    return meta

get_training_datasets()

Returns all training datasets of the model including similar tasks.

Source code in mteb/models/model_meta.py

def get_training_datasets(self) -> set[str] | None:
    """Returns all training datasets of the model including similar tasks."""
    import mteb

    if self.training_datasets is None:
        return None

    training_datasets = self.training_datasets.copy()
    if self.adapted_from is not None:
        try:
            adapted_from_model = mteb.get_model_meta(
                self.adapted_from, fetch_from_hf=False
            )
            adapted_training_datasets = adapted_from_model.get_training_datasets()
            if adapted_training_datasets is not None:
                training_datasets |= adapted_training_datasets
        except (ValueError, KeyError) as e:
            msg = f"Could not get source model: {e} in MTEB"
            logger.warning(msg)
            warnings.warn(msg)

    return_dataset = training_datasets.copy()
    visited: set[str] = set()

    for dataset in training_datasets:
        similar_tasks = _collect_similar_tasks(dataset, visited)
        return_dataset |= similar_tasks

    return return_dataset

handle_legacy_is_cross_encoder(data) classmethod

Handle legacy is_cross_encoder field by converting it to model_type.

This validator handles backward compatibility for the deprecated is_cross_encoder field. If is_cross_encoder=True is provided, it adds "cross_encoder" to model_type.

Source code in mteb/models/model_meta.py

@model_validator(mode="before")
@classmethod
def handle_legacy_is_cross_encoder(cls, data: Any) -> Any:
    """Handle legacy is_cross_encoder field by converting it to model_type.

    This validator handles backward compatibility for the deprecated is_cross_encoder field.
    If is_cross_encoder=True is provided, it adds "cross_encoder" to model_type.
    """
    if isinstance(data, dict) and "is_cross_encoder" in data:
        is_cross_encoder_value = data.pop("is_cross_encoder")

        if is_cross_encoder_value is not None:
            warnings.warn(
                "is_cross_encoder is deprecated and will be removed in a future version. "
                "Use model_type=['cross-encoder'] instead.",
                DeprecationWarning,
                stacklevel=2,
            )

            model_type = data.get("model_type", ["dense"])

            if is_cross_encoder_value:
                if "cross-encoder" not in model_type:
                    data["model_type"] = ["cross-encoder"]
            else:
                if "cross-encoder" in model_type:
                    model_type = [t for t in model_type if t != "cross-encoder"]
                    data["model_type"] = model_type if model_type else ["dense"]

    return data

is_zero_shot_on(tasks)

Indicates whether the given model can be considered zero-shot or not on the given tasks.

Returns:

Type	Description
bool | None	None if no training data is specified on the model.

Source code in mteb/models/model_meta.py

def is_zero_shot_on(self, tasks: Sequence[AbsTask] | Sequence[str]) -> bool | None:
    """Indicates whether the given model can be considered zero-shot or not on the given tasks.

    Returns:
         None if no training data is specified on the model.
    """
    # If no tasks were specified, we're obviously zero-shot
    if not tasks:
        return True
    training_datasets = self.get_training_datasets()
    # If no tasks were specified, we're obviously zero-shot
    if training_datasets is None:
        return None

    if isinstance(tasks[0], str):
        benchmark_datasets = set(tasks)
    else:
        tasks = cast(Sequence["AbsTask"], tasks)
        benchmark_datasets = set()
        for task in tasks:
            benchmark_datasets.add(task.metadata.name)
    intersection = training_datasets & benchmark_datasets
    return len(intersection) == 0

load_model(device=None, **kwargs)

Loads the model using the specified loader function.

Source code in mteb/models/model_meta.py

def load_model(self, device: str | None = None, **kwargs: Any) -> MTEBModels:
    """Loads the model using the specified loader function."""
    if self.loader is None:
        raise NotImplementedError(
            "No model implementation is available for this model."
        )
    if self.name is None:
        raise ValueError("name is not set for ModelMeta. Cannot load model.")

    # Allow overwrites
    _kwargs = self.loader_kwargs.copy()
    _kwargs.update(kwargs)
    if device is not None:
        _kwargs["device"] = device

    model: MTEBModels = self.loader(self.name, revision=self.revision, **_kwargs)
    model.mteb_model_meta = self  # type: ignore[misc]
    return model

model_name_as_path()

Returns the model name in a format that can be used as a file path.

Replaces "/" with "__" and spaces with "_".

Source code in mteb/models/model_meta.py

def model_name_as_path(self) -> str:
    """Returns the model name in a format that can be used as a file path.

    Replaces "/" with "__" and spaces with "_".
    """
    if self.name is None:
        raise ValueError("Model name is not set")
    return self.name.replace("/", "__").replace(" ", "_")

to_dict()

Returns a dictionary representation of the model metadata.

Source code in mteb/models/model_meta.py

def to_dict(self):
    """Returns a dictionary representation of the model metadata."""
    dict_repr = self.model_dump()
    loader = dict_repr.pop("loader", None)
    dict_repr["training_datasets"] = (
        list(dict_repr["training_datasets"])
        if isinstance(dict_repr["training_datasets"], set)
        else dict_repr["training_datasets"]
    )
    dict_repr["loader"] = _get_loader_name(loader)
    dict_repr["is_cross_encoder"] = self.is_cross_encoder
    return dict_repr

to_python()

Returns a string representation of the model.

Source code in mteb/models/model_meta.py

def to_python(self) -> str:
    """Returns a string representation of the model."""
    return _pydantic_instance_to_code(self)

zero_shot_percentage(tasks)

Indicates how out-of-domain the selected tasks are for the given model.

Parameters:

Name	Type	Description	Default
tasks	Sequence[AbsTask] | Sequence[str]	A sequence of tasks or dataset names to evaluate against.	required

Returns:

Type	Description
int | None	An integer percentage (0-100) indicating how out-of-domain the tasks are for the model.
int | None	Returns None if no training data is specified on the model or if no tasks are provided.

Source code in mteb/models/model_meta.py

def zero_shot_percentage(
    self, tasks: Sequence[AbsTask] | Sequence[str]
) -> int | None:
    """Indicates how out-of-domain the selected tasks are for the given model.

    Args:
        tasks: A sequence of tasks or dataset names to evaluate against.

    Returns:
        An integer percentage (0-100) indicating how out-of-domain the tasks are for the model.
        Returns None if no training data is specified on the model or if no tasks are provided.
    """
    training_datasets = self.get_training_datasets()
    if (training_datasets is None) or (not tasks):
        return None
    if isinstance(tasks[0], str):
        benchmark_datasets = set(tasks)
    else:
        tasks = cast(Sequence["AbsTask"], tasks)
        benchmark_datasets = {task.metadata.name for task in tasks}
    overlap = training_datasets & benchmark_datasets
    perc_overlap = 100 * (len(overlap) / len(benchmark_datasets))
    return int(100 - perc_overlap)

Model Protocols

mteb.models.EncoderProtocol

Bases: Protocol

The interface for an encoder in MTEB.

Besides the required functions specified below, the encoder can additionally specify the following signatures seen below. In general the interface is kept aligned with sentence-transformers interface. In cases where exceptions occurs these are handled within MTEB.

Source code in mteb/models/models_protocols.py

@runtime_checkable
class EncoderProtocol(Protocol):
    """The interface for an encoder in MTEB.

    Besides the required functions specified below, the encoder can additionally specify the following signatures seen below.
    In general the interface is kept aligned with sentence-transformers interface. In cases where exceptions occurs these are handled within MTEB.
    """

    def __init__(
        self,
        model_name: str,
        revision: str | None,
        device: str | None = None,
        **kwargs: Any,
    ) -> None:
        """The initialization function for the encoder. Used when calling it from the mteb run CLI.

        Args:
            model_name: Name of the model
            revision: revision of the model
            device: Device used to load the model
            kwargs: Any additional kwargs
        """
        ...

    def encode(
        self,
        inputs: DataLoader[BatchedInput],
        *,
        task_metadata: TaskMetadata,
        hf_split: str,
        hf_subset: str,
        prompt_type: PromptType | None = None,
        **kwargs: Any,
    ) -> Array:
        """Encodes the given sentences using the encoder.

        Args:
            inputs: Batch of inputs to encode.
            task_metadata: The metadata of the task. Encoders (e.g. SentenceTransformers) use to
                select the appropriate prompts, with priority given to more specific task/prompt combinations over general ones.

                The order of priorities for prompt selection are:
                    1. Composed prompt of task name + prompt type (query or passage)
                    2. Specific task prompt
                    3. Composed prompt of task type + prompt type (query or passage)
                    4. Specific task type prompt
                    5. Specific prompt type (query or passage)
            hf_split: Split of current task, allows to know some additional information about current split.
                E.g. Current language
            hf_subset: Subset of current task. Similar to `hf_split` to get more information
            prompt_type: The name type of prompt. (query or passage)
            **kwargs: Additional arguments to pass to the encoder.

        Returns:
            The encoded input in a numpy array or torch tensor of the shape (Number of sentences) x (Embedding dimension).
        """
        ...

    def similarity(
        self,
        embeddings1: Array,
        embeddings2: Array,
    ) -> Array:
        """Compute the similarity between two collections of embeddings.

        The output will be a matrix with the similarity scores between all embeddings from the first parameter and all
        embeddings from the second parameter. This differs from similarity_pairwise which computes the similarity
        between corresponding pairs of embeddings.

        Read more at: https://www.sbert.net/docs/package_reference/sentence_transformer/SentenceTransformer.html#sentence_transformers.SentenceTransformer.similarity

        Args:
            embeddings1: [num_embeddings_1, embedding_dim] or [embedding_dim]-shaped numpy array or torch tensor.
            embeddings2: [num_embeddings_2, embedding_dim] or [embedding_dim]-shaped numpy array or torch tensor.

        Returns:
            A [num_embeddings_1, num_embeddings_2]-shaped torch tensor with similarity scores.
        """
        ...

    def similarity_pairwise(
        self,
        embeddings1: Array,
        embeddings2: Array,
    ) -> Array:
        """Compute the similarity between two collections of embeddings. The output will be a vector with the similarity scores between each pair of embeddings.

        Read more at: https://www.sbert.net/docs/package_reference/sentence_transformer/SentenceTransformer.html#sentence_transformers.SentenceTransformer.similarity_pairwise

        Args:
            embeddings1: [num_embeddings, embedding_dim] or [embedding_dim]-shaped numpy array or torch tensor.
            embeddings2: [num_embeddings, embedding_dim] or [embedding_dim]-shaped numpy array or torch tensor.

        Returns:
            A [num_embeddings]-shaped torch tensor with pairwise similarity scores.
        """
        ...

    @property
    def mteb_model_meta(self) -> "ModelMeta":
        """Metadata of the model"""
        ...

mteb_model_meta property

Metadata of the model

__init__(model_name, revision, device=None, **kwargs)

The initialization function for the encoder. Used when calling it from the mteb run CLI.

Parameters:

Name	Type	Description	Default
model_name	str	Name of the model	required
revision	str | None	revision of the model	required
device	str | None	Device used to load the model	None
kwargs	Any	Any additional kwargs	{}

Source code in mteb/models/models_protocols.py

def __init__(
    self,
    model_name: str,
    revision: str | None,
    device: str | None = None,
    **kwargs: Any,
) -> None:
    """The initialization function for the encoder. Used when calling it from the mteb run CLI.

    Args:
        model_name: Name of the model
        revision: revision of the model
        device: Device used to load the model
        kwargs: Any additional kwargs
    """
    ...

encode(inputs, *, task_metadata, hf_split, hf_subset, prompt_type=None, **kwargs)

Encodes the given sentences using the encoder.

Parameters:

Name	Type	Description	Default
inputs	DataLoader[BatchedInput]	Batch of inputs to encode.	required
task_metadata	TaskMetadata	The metadata of the task. Encoders (e.g. SentenceTransformers) use to select the appropriate prompts, with priority given to more specific task/prompt combinations over general ones. The order of priorities for prompt selection are: 1. Composed prompt of task name + prompt type (query or passage) 2. Specific task prompt 3. Composed prompt of task type + prompt type (query or passage) 4. Specific task type prompt 5. Specific prompt type (query or passage)	required
hf_split	str	Split of current task, allows to know some additional information about current split. E.g. Current language	required
hf_subset	str	Subset of current task. Similar to hf_split to get more information	required
prompt_type	PromptType | None	The name type of prompt. (query or passage)	None
**kwargs	Any	Additional arguments to pass to the encoder.	{}

Returns:

Type	Description
Array	The encoded input in a numpy array or torch tensor of the shape (Number of sentences) x (Embedding dimension).

Source code in mteb/models/models_protocols.py

def encode(
    self,
    inputs: DataLoader[BatchedInput],
    *,
    task_metadata: TaskMetadata,
    hf_split: str,
    hf_subset: str,
    prompt_type: PromptType | None = None,
    **kwargs: Any,
) -> Array:
    """Encodes the given sentences using the encoder.

    Args:
        inputs: Batch of inputs to encode.
        task_metadata: The metadata of the task. Encoders (e.g. SentenceTransformers) use to
            select the appropriate prompts, with priority given to more specific task/prompt combinations over general ones.

            The order of priorities for prompt selection are:
                1. Composed prompt of task name + prompt type (query or passage)
                2. Specific task prompt
                3. Composed prompt of task type + prompt type (query or passage)
                4. Specific task type prompt
                5. Specific prompt type (query or passage)
        hf_split: Split of current task, allows to know some additional information about current split.
            E.g. Current language
        hf_subset: Subset of current task. Similar to `hf_split` to get more information
        prompt_type: The name type of prompt. (query or passage)
        **kwargs: Additional arguments to pass to the encoder.

    Returns:
        The encoded input in a numpy array or torch tensor of the shape (Number of sentences) x (Embedding dimension).
    """
    ...

similarity(embeddings1, embeddings2)

Compute the similarity between two collections of embeddings.

The output will be a matrix with the similarity scores between all embeddings from the first parameter and all embeddings from the second parameter. This differs from similarity_pairwise which computes the similarity between corresponding pairs of embeddings.

Read more at: https://www.sbert.net/docs/package_reference/sentence_transformer/SentenceTransformer.html#sentence_transformers.SentenceTransformer.similarity

Parameters:

Name	Type	Description	Default
embeddings1	Array	[num_embeddings_1, embedding_dim] or [embedding_dim]-shaped numpy array or torch tensor.	required
embeddings2	Array	[num_embeddings_2, embedding_dim] or [embedding_dim]-shaped numpy array or torch tensor.	required

Returns:

Type	Description
Array	A [num_embeddings_1, num_embeddings_2]-shaped torch tensor with similarity scores.

Source code in mteb/models/models_protocols.py

def similarity(
    self,
    embeddings1: Array,
    embeddings2: Array,
) -> Array:
    """Compute the similarity between two collections of embeddings.

    The output will be a matrix with the similarity scores between all embeddings from the first parameter and all
    embeddings from the second parameter. This differs from similarity_pairwise which computes the similarity
    between corresponding pairs of embeddings.

    Read more at: https://www.sbert.net/docs/package_reference/sentence_transformer/SentenceTransformer.html#sentence_transformers.SentenceTransformer.similarity

    Args:
        embeddings1: [num_embeddings_1, embedding_dim] or [embedding_dim]-shaped numpy array or torch tensor.
        embeddings2: [num_embeddings_2, embedding_dim] or [embedding_dim]-shaped numpy array or torch tensor.

    Returns:
        A [num_embeddings_1, num_embeddings_2]-shaped torch tensor with similarity scores.
    """
    ...

similarity_pairwise(embeddings1, embeddings2)

Compute the similarity between two collections of embeddings. The output will be a vector with the similarity scores between each pair of embeddings.

Read more at: https://www.sbert.net/docs/package_reference/sentence_transformer/SentenceTransformer.html#sentence_transformers.SentenceTransformer.similarity_pairwise

Parameters:

Name	Type	Description	Default
embeddings1	Array	[num_embeddings, embedding_dim] or [embedding_dim]-shaped numpy array or torch tensor.	required
embeddings2	Array	[num_embeddings, embedding_dim] or [embedding_dim]-shaped numpy array or torch tensor.	required

Returns:

Type	Description
Array	A [num_embeddings]-shaped torch tensor with pairwise similarity scores.

Source code in mteb/models/models_protocols.py

def similarity_pairwise(
    self,
    embeddings1: Array,
    embeddings2: Array,
) -> Array:
    """Compute the similarity between two collections of embeddings. The output will be a vector with the similarity scores between each pair of embeddings.

    Read more at: https://www.sbert.net/docs/package_reference/sentence_transformer/SentenceTransformer.html#sentence_transformers.SentenceTransformer.similarity_pairwise

    Args:
        embeddings1: [num_embeddings, embedding_dim] or [embedding_dim]-shaped numpy array or torch tensor.
        embeddings2: [num_embeddings, embedding_dim] or [embedding_dim]-shaped numpy array or torch tensor.

    Returns:
        A [num_embeddings]-shaped torch tensor with pairwise similarity scores.
    """
    ...

mteb.models.SearchProtocol

Bases: Protocol

Interface for searching models.

Source code in mteb/models/models_protocols.py

@runtime_checkable
class SearchProtocol(Protocol):
    """Interface for searching models."""

    def index(
        self,
        corpus: CorpusDatasetType,
        *,
        task_metadata: TaskMetadata,
        hf_split: str,
        hf_subset: str,
        encode_kwargs: dict[str, Any],
    ) -> None:
        """Index the corpus for retrieval.

        Args:
            corpus: Corpus dataset to index.
            task_metadata: Metadata of the task, used to determine how to index the corpus.
            hf_split: Split of current task, allows to know some additional information about current split.
            hf_subset: Subset of current task. Similar to `hf_split` to get more information
            encode_kwargs: Additional arguments to pass to the encoder during indexing.
        """
        ...

    def search(
        self,
        queries: QueryDatasetType,
        *,
        task_metadata: TaskMetadata,
        hf_split: str,
        hf_subset: str,
        top_k: int,
        encode_kwargs: dict[str, Any],
        top_ranked: TopRankedDocumentsType | None = None,
    ) -> RetrievalOutputType:
        """Search the corpus using the given queries.

        Args:
            queries: Queries to find
            task_metadata: Task metadata
            hf_split: split of the dataset
            hf_subset: subset of the dataset
            top_ranked: Top-ranked documents for each query, mapping query IDs to a list of document IDs.
                Passed only from Reranking tasks.
            top_k: Number of top documents to return for each query.
            encode_kwargs: Additional arguments to pass to the encoder during indexing.

        Returns:
            Dictionary with query IDs as keys with dict as values, where each value is a mapping of document IDs to their relevance scores.
        """
        ...

    @property
    def mteb_model_meta(self) -> "ModelMeta":
        """Metadata of the model"""
        ...

mteb_model_meta property

Metadata of the model

index(corpus, *, task_metadata, hf_split, hf_subset, encode_kwargs)

Index the corpus for retrieval.

Parameters:

Name	Type	Description	Default
corpus	CorpusDatasetType	Corpus dataset to index.	required
task_metadata	TaskMetadata	Metadata of the task, used to determine how to index the corpus.	required
hf_split	str	Split of current task, allows to know some additional information about current split.	required
hf_subset	str	Subset of current task. Similar to hf_split to get more information	required
encode_kwargs	dict[str, Any]	Additional arguments to pass to the encoder during indexing.	required

Source code in mteb/models/models_protocols.py

def index(
    self,
    corpus: CorpusDatasetType,
    *,
    task_metadata: TaskMetadata,
    hf_split: str,
    hf_subset: str,
    encode_kwargs: dict[str, Any],
) -> None:
    """Index the corpus for retrieval.

    Args:
        corpus: Corpus dataset to index.
        task_metadata: Metadata of the task, used to determine how to index the corpus.
        hf_split: Split of current task, allows to know some additional information about current split.
        hf_subset: Subset of current task. Similar to `hf_split` to get more information
        encode_kwargs: Additional arguments to pass to the encoder during indexing.
    """
    ...

search(queries, *, task_metadata, hf_split, hf_subset, top_k, encode_kwargs, top_ranked=None)

Search the corpus using the given queries.

Parameters:

Name	Type	Description	Default
queries	QueryDatasetType	Queries to find	required
task_metadata	TaskMetadata	Task metadata	required
hf_split	str	split of the dataset	required
hf_subset	str	subset of the dataset	required
top_ranked	TopRankedDocumentsType | None	Top-ranked documents for each query, mapping query IDs to a list of document IDs. Passed only from Reranking tasks.	None
top_k	int	Number of top documents to return for each query.	required
encode_kwargs	dict[str, Any]	Additional arguments to pass to the encoder during indexing.	required

Returns:

Type	Description
RetrievalOutputType	Dictionary with query IDs as keys with dict as values, where each value is a mapping of document IDs to their relevance scores.

Source code in mteb/models/models_protocols.py

def search(
    self,
    queries: QueryDatasetType,
    *,
    task_metadata: TaskMetadata,
    hf_split: str,
    hf_subset: str,
    top_k: int,
    encode_kwargs: dict[str, Any],
    top_ranked: TopRankedDocumentsType | None = None,
) -> RetrievalOutputType:
    """Search the corpus using the given queries.

    Args:
        queries: Queries to find
        task_metadata: Task metadata
        hf_split: split of the dataset
        hf_subset: subset of the dataset
        top_ranked: Top-ranked documents for each query, mapping query IDs to a list of document IDs.
            Passed only from Reranking tasks.
        top_k: Number of top documents to return for each query.
        encode_kwargs: Additional arguments to pass to the encoder during indexing.

    Returns:
        Dictionary with query IDs as keys with dict as values, where each value is a mapping of document IDs to their relevance scores.
    """
    ...

mteb.models.CrossEncoderProtocol

Bases: Protocol

The interface for a CrossEncoder in MTEB.

Besides the required functions specified below, the cross-encoder can additionally specify the following signatures seen below. In general the interface is kept aligned with sentence-transformers interface. In cases where exceptions occurs these are handled within MTEB.

Source code in mteb/models/models_protocols.py

@runtime_checkable
class CrossEncoderProtocol(Protocol):
    """The interface for a CrossEncoder in MTEB.

    Besides the required functions specified below, the cross-encoder can additionally specify the following signatures seen below.
    In general the interface is kept aligned with sentence-transformers interface. In cases where exceptions occurs these are handled within MTEB.
    """

    def __init__(
        self,
        model_name: str,
        revision: str | None,
        device: str | None = None,
        **kwargs: Any,
    ) -> None:
        """The initialization function for the encoder. Used when calling it from the mteb run CLI.

        Args:
            model_name: Name of the model
            revision: revision of the model
            device: Device used to load the model
            kwargs: Any additional kwargs
        """
        ...

    def predict(
        self,
        inputs1: DataLoader[BatchedInput],
        inputs2: DataLoader[BatchedInput],
        *,
        task_metadata: TaskMetadata,
        hf_split: str,
        hf_subset: str,
        prompt_type: PromptType | None = None,
        **kwargs: Any,
    ) -> Array:
        """Predicts relevance scores for pairs of inputs. Note that, unlike the encoder, the cross-encoder can compare across inputs.

        Args:
            inputs1: First Dataloader of inputs to encode. For reranking tasks, these are queries (for text only tasks `QueryDatasetType`).
            inputs2: Second Dataloader of inputs to encode. For reranking, these are documents (for text only tasks `RetrievalOutputType`).
            task_metadata: Metadata of the current task.
            hf_split: Split of current task, allows to know some additional information about current split.
                E.g. Current language
            hf_subset: Subset of current task. Similar to `hf_split` to get more information
            prompt_type: The name type of prompt. (query or passage)
            **kwargs: Additional arguments to pass to the cross-encoder.

        Returns:
            The predicted relevance scores for each inputs pair.
        """
        ...

    @property
    def mteb_model_meta(self) -> "ModelMeta":
        """Metadata of the model"""
        ...

mteb_model_meta property

Metadata of the model

__init__(model_name, revision, device=None, **kwargs)

The initialization function for the encoder. Used when calling it from the mteb run CLI.

Parameters:

Name	Type	Description	Default
model_name	str	Name of the model	required
revision	str | None	revision of the model	required
device	str | None	Device used to load the model	None
kwargs	Any	Any additional kwargs	{}

Source code in mteb/models/models_protocols.py

def __init__(
    self,
    model_name: str,
    revision: str | None,
    device: str | None = None,
    **kwargs: Any,
) -> None:
    """The initialization function for the encoder. Used when calling it from the mteb run CLI.

    Args:
        model_name: Name of the model
        revision: revision of the model
        device: Device used to load the model
        kwargs: Any additional kwargs
    """
    ...

predict(inputs1, inputs2, *, task_metadata, hf_split, hf_subset, prompt_type=None, **kwargs)

Predicts relevance scores for pairs of inputs. Note that, unlike the encoder, the cross-encoder can compare across inputs.

Parameters:

Name	Type	Description	Default
inputs1	DataLoader[BatchedInput]	First Dataloader of inputs to encode. For reranking tasks, these are queries (for text only tasks QueryDatasetType).	required
inputs2	DataLoader[BatchedInput]	Second Dataloader of inputs to encode. For reranking, these are documents (for text only tasks RetrievalOutputType).	required
task_metadata	TaskMetadata	Metadata of the current task.	required
hf_split	str	Split of current task, allows to know some additional information about current split. E.g. Current language	required
hf_subset	str	Subset of current task. Similar to hf_split to get more information	required
prompt_type	PromptType | None	The name type of prompt. (query or passage)	None
**kwargs	Any	Additional arguments to pass to the cross-encoder.	{}

Returns:

Type	Description
Array	The predicted relevance scores for each inputs pair.

Source code in mteb/models/models_protocols.py

def predict(
    self,
    inputs1: DataLoader[BatchedInput],
    inputs2: DataLoader[BatchedInput],
    *,
    task_metadata: TaskMetadata,
    hf_split: str,
    hf_subset: str,
    prompt_type: PromptType | None = None,
    **kwargs: Any,
) -> Array:
    """Predicts relevance scores for pairs of inputs. Note that, unlike the encoder, the cross-encoder can compare across inputs.

    Args:
        inputs1: First Dataloader of inputs to encode. For reranking tasks, these are queries (for text only tasks `QueryDatasetType`).
        inputs2: Second Dataloader of inputs to encode. For reranking, these are documents (for text only tasks `RetrievalOutputType`).
        task_metadata: Metadata of the current task.
        hf_split: Split of current task, allows to know some additional information about current split.
            E.g. Current language
        hf_subset: Subset of current task. Similar to `hf_split` to get more information
        prompt_type: The name type of prompt. (query or passage)
        **kwargs: Additional arguments to pass to the cross-encoder.

    Returns:
        The predicted relevance scores for each inputs pair.
    """
    ...

mteb.models.MTEBModels = EncoderProtocol | CrossEncoderProtocol | SearchProtocol module-attribute

Type alias for all MTEB model types as many models implement multiple protocols and many tasks can be solved by multiple model types.

mteb.models.IndexEncoderSearchProtocol

Bases: Protocol

Protocol for search backends used in encoder-based retrieval.

Source code in mteb/models/search_encoder_index/search_backend_protocol.py

class IndexEncoderSearchProtocol(Protocol):
    """Protocol for search backends used in encoder-based retrieval."""

    def add_documents(
        self,
        embeddings: Array,
        idxs: list[str],
    ) -> None:
        """Add documents to the search backend.

        Args:
            embeddings: Embeddings of the documents to add.
            idxs: IDs of the documents to add.
        """

    def search(
        self,
        embeddings: Array,
        top_k: int,
        similarity_fn: Callable[[Array, Array], Array],
        top_ranked: TopRankedDocumentsType | None = None,
        query_idx_to_id: dict[int, str] | None = None,
    ) -> tuple[list[list[float]], list[list[int]]]:
        """Search through added corpus embeddings or rerank top-ranked documents.

        Supports both full-corpus and reranking search modes:
            - Full-corpus mode: `top_ranked=None`, uses added corpus embeddings.
            - Reranking mode:  `top_ranked` contains mapping {query_id: [doc_ids]}.

        Args:
            embeddings: Query embeddings, shape (num_queries, dim).
            top_k: Number of top results to return.
            similarity_fn: Function to compute similarity between query and corpus.
            top_ranked: Mapping of query_id -> list of candidate doc_ids. Used for reranking.
            query_idx_to_id: Mapping of query index -> query_id. Used for reranking.

        Returns:
            A tuple (top_k_values, top_k_indices), for each query:
                - top_k_values: List of top-k similarity scores.
                - top_k_indices: List of indices of the top-k documents in the added corpus.
        """

    def clear(self) -> None:
        """Clear all stored documents and embeddings from the backend."""

add_documents(embeddings, idxs)

Add documents to the search backend.

Parameters:

Name	Type	Description	Default
embeddings	Array	Embeddings of the documents to add.	required
idxs	list[str]	IDs of the documents to add.	required

Source code in mteb/models/search_encoder_index/search_backend_protocol.py

def add_documents(
    self,
    embeddings: Array,
    idxs: list[str],
) -> None:
    """Add documents to the search backend.

    Args:
        embeddings: Embeddings of the documents to add.
        idxs: IDs of the documents to add.
    """

clear()

Clear all stored documents and embeddings from the backend.

Source code in mteb/models/search_encoder_index/search_backend_protocol.py

def clear(self) -> None:
    """Clear all stored documents and embeddings from the backend."""

search(embeddings, top_k, similarity_fn, top_ranked=None, query_idx_to_id=None)

Search through added corpus embeddings or rerank top-ranked documents.

Supports both full-corpus and reranking search modes

• Full-corpus mode: top_ranked=None, uses added corpus embeddings.
• Reranking mode: top_ranked contains mapping {query_id: [doc_ids]}.

Parameters:

Name	Type	Description	Default
embeddings	Array	Query embeddings, shape (num_queries, dim).	required
top_k	int	Number of top results to return.	required
similarity_fn	Callable[[Array, Array], Array]	Function to compute similarity between query and corpus.	required
top_ranked	TopRankedDocumentsType | None	Mapping of query_id -> list of candidate doc_ids. Used for reranking.	None
query_idx_to_id	dict[int, str] | None	Mapping of query index -> query_id. Used for reranking.	None

Returns:

Type	Description
tuple[list[list[float]], list[list[int]]]	A tuple (top_k_values, top_k_indices), for each query: - top_k_values: List of top-k similarity scores. - top_k_indices: List of indices of the top-k documents in the added corpus.

Source code in mteb/models/search_encoder_index/search_backend_protocol.py

def search(
    self,
    embeddings: Array,
    top_k: int,
    similarity_fn: Callable[[Array, Array], Array],
    top_ranked: TopRankedDocumentsType | None = None,
    query_idx_to_id: dict[int, str] | None = None,
) -> tuple[list[list[float]], list[list[int]]]:
    """Search through added corpus embeddings or rerank top-ranked documents.

    Supports both full-corpus and reranking search modes:
        - Full-corpus mode: `top_ranked=None`, uses added corpus embeddings.
        - Reranking mode:  `top_ranked` contains mapping {query_id: [doc_ids]}.

    Args:
        embeddings: Query embeddings, shape (num_queries, dim).
        top_k: Number of top results to return.
        similarity_fn: Function to compute similarity between query and corpus.
        top_ranked: Mapping of query_id -> list of candidate doc_ids. Used for reranking.
        query_idx_to_id: Mapping of query index -> query_id. Used for reranking.

    Returns:
        A tuple (top_k_values, top_k_indices), for each query:
            - top_k_values: List of top-k similarity scores.
            - top_k_indices: List of indices of the top-k documents in the added corpus.
    """

mteb.models.CacheBackendProtocol

Bases: Protocol

Protocol for a vector cache map (used to store text/image embeddings).

Implementations may back the cache with different storage backends.

The cache maps an input item (text or image) to its vector embedding, identified by a deterministic hash.

Source code in mteb/models/cache_wrappers/cache_backend_protocol.py

@runtime_checkable
class CacheBackendProtocol(Protocol):
    """Protocol for a vector cache map (used to store text/image embeddings).

    Implementations may back the cache with different storage backends.

    The cache maps an input item (text or image) to its vector embedding,
    identified by a deterministic hash.
    """

    def __init__(self, directory: Path | None = None, **kwargs: Any) -> None:
        """Initialize the cache backend.

        Args:
            directory: Directory path to store cache files.
            **kwargs: Additional backend-specific arguments.
        """

    def add(self, item: list[dict[str, Any]], vectors: np.ndarray) -> None:
        """Add a vector to the cache.

        Args:
            item: Input item containing 'text' or 'image'.
            vectors: Embedding vector of shape (dim,) or (1, dim).
        """

    def get_vector(self, item: dict[str, Any]) -> np.ndarray | None:
        """Retrieve the cached vector for the given item.

        Args:
            item: Input item.

        Returns:
            Cached vector as np.ndarray, or None if not found.
        """

    def save(self) -> None:
        """Persist cache data to disk (index + metadata)."""

    def load(self) -> None:
        """Load cache from disk (index + metadata)."""

    def close(self) -> None:
        """Release resources or flush data."""

    def __contains__(self, item: dict[str, Any]) -> bool:
        """Check whether the cache contains an item."""

__contains__(item)

Check whether the cache contains an item.

Source code in mteb/models/cache_wrappers/cache_backend_protocol.py

def __contains__(self, item: dict[str, Any]) -> bool:
    """Check whether the cache contains an item."""

__init__(directory=None, **kwargs)

Initialize the cache backend.

Parameters:

Name	Type	Description	Default
directory	Path | None	Directory path to store cache files.	None
**kwargs	Any	Additional backend-specific arguments.	{}

Source code in mteb/models/cache_wrappers/cache_backend_protocol.py

def __init__(self, directory: Path | None = None, **kwargs: Any) -> None:
    """Initialize the cache backend.

    Args:
        directory: Directory path to store cache files.
        **kwargs: Additional backend-specific arguments.
    """

add(item, vectors)

Add a vector to the cache.

Parameters:

Name	Type	Description	Default
item	list[dict[str, Any]]	Input item containing 'text' or 'image'.	required
vectors	ndarray	Embedding vector of shape (dim,) or (1, dim).	required

Source code in mteb/models/cache_wrappers/cache_backend_protocol.py

def add(self, item: list[dict[str, Any]], vectors: np.ndarray) -> None:
    """Add a vector to the cache.

    Args:
        item: Input item containing 'text' or 'image'.
        vectors: Embedding vector of shape (dim,) or (1, dim).
    """

close()

Release resources or flush data.

Source code in mteb/models/cache_wrappers/cache_backend_protocol.py

def close(self) -> None:
    """Release resources or flush data."""

get_vector(item)

Retrieve the cached vector for the given item.

Parameters:

Name	Type	Description	Default
item	dict[str, Any]	Input item.	required

Returns:

Type	Description
ndarray | None	Cached vector as np.ndarray, or None if not found.

Source code in mteb/models/cache_wrappers/cache_backend_protocol.py

def get_vector(self, item: dict[str, Any]) -> np.ndarray | None:
    """Retrieve the cached vector for the given item.

    Args:
        item: Input item.

    Returns:
        Cached vector as np.ndarray, or None if not found.
    """

load()

Load cache from disk (index + metadata).

Source code in mteb/models/cache_wrappers/cache_backend_protocol.py

def load(self) -> None:
    """Load cache from disk (index + metadata)."""

save()

Persist cache data to disk (index + metadata).

Source code in mteb/models/cache_wrappers/cache_backend_protocol.py

def save(self) -> None:
    """Persist cache data to disk (index + metadata)."""