image-match/python/py/Lib/site-packages/transformers/models/auto/tokenization_auto.py

# Copyright 2018 The HuggingFace Inc. team.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Auto Tokenizer class."""

import importlib
import json
import os
import sys
from collections import OrderedDict
from typing import Any

from transformers.utils.import_utils import is_mistral_common_available

from ...configuration_utils import PreTrainedConfig
from ...dynamic_module_utils import get_class_from_dynamic_module, resolve_trust_remote_code
from ...modeling_gguf_pytorch_utils import load_gguf_checkpoint
from ...tokenization_utils_base import TOKENIZER_CONFIG_FILE
from ...utils import (
    extract_commit_hash,
    is_g2p_en_available,
    is_sentencepiece_available,
    is_tokenizers_available,
    logging,
)
from ...utils.hub import cached_file
from ..encoder_decoder import EncoderDecoderConfig
from .auto_factory import _LazyAutoMapping
from .configuration_auto import (
    CONFIG_MAPPING_NAMES,
    AutoConfig,
    config_class_to_model_type,
    model_type_to_module_name,
    replace_list_option_in_docstrings,
)


if is_tokenizers_available():
    from ...tokenization_utils_tokenizers import TokenizersBackend
else:
    TokenizersBackend = None

if is_sentencepiece_available():
    from ...tokenization_utils_sentencepiece import SentencePieceBackend
else:
    SentencePieceBackend = None

logger = logging.get_logger(__name__)

# V5: Simplified mapping - single tokenizer class per model type (always prefer tokenizers-based)
REGISTERED_TOKENIZER_CLASSES: dict[str, type[Any]] = {}
REGISTERED_FAST_ALIASES: dict[str, type[Any]] = {}

TOKENIZER_MAPPING_NAMES = OrderedDict[str, str | None](
    [
        ("aimv2", "CLIPTokenizer" if is_tokenizers_available() else None),
        ("albert", "AlbertTokenizer" if is_tokenizers_available() else None),
        ("align", "BertTokenizer" if is_tokenizers_available() else None),
        ("audioflamingo3", "Qwen2Tokenizer" if is_tokenizers_available() else None),
        ("aya_vision", "CohereTokenizer" if is_tokenizers_available() else None),
        ("bark", "BertTokenizer" if is_tokenizers_available() else None),
        ("bart", "RobertaTokenizer" if is_tokenizers_available() else None),
        ("barthez", "BarthezTokenizer" if is_tokenizers_available() else None),
        ("bartpho", "BartphoTokenizer"),
        ("bert", "BertTokenizer" if is_tokenizers_available() else None),
        ("bert-generation", "BertGenerationTokenizer" if is_sentencepiece_available() else None),
        ("bert-japanese", "BertJapaneseTokenizer"),
        ("bertweet", "BertweetTokenizer"),
        ("big_bird", "BigBirdTokenizer" if is_tokenizers_available() else None),
        ("bigbird_pegasus", "PegasusTokenizer" if is_tokenizers_available() else None),
        ("biogpt", "BioGptTokenizer"),
        ("blenderbot", "BlenderbotTokenizer" if is_tokenizers_available() else None),
        ("blenderbot-small", "BlenderbotSmallTokenizer"),
        ("blip", "BertTokenizer" if is_tokenizers_available() else None),
        ("blip-2", "GPT2Tokenizer" if is_tokenizers_available() else None),
        ("bridgetower", "RobertaTokenizer"),
        ("bros", "BertTokenizer" if is_tokenizers_available() else None),
        ("byt5", "ByT5Tokenizer"),
        ("camembert", "CamembertTokenizer" if is_tokenizers_available() else None),
        ("canine", "CanineTokenizer"),
        ("chinese_clip", "BertTokenizer" if is_tokenizers_available() else None),
        ("clap", "RobertaTokenizer"),
        ("clip", "CLIPTokenizer" if is_tokenizers_available() else None),
        ("clipseg", "CLIPTokenizer" if is_tokenizers_available() else None),
        ("clvp", "ClvpTokenizer"),
        ("code_llama", "CodeLlamaTokenizer" if is_tokenizers_available() else None),
        ("codegen", "GPT2Tokenizer" if is_tokenizers_available() else None),
        ("cohere", "CohereTokenizer" if is_tokenizers_available() else None),
        ("cohere2", "CohereTokenizer" if is_tokenizers_available() else None),
        ("colqwen2", "Qwen2Tokenizer" if is_tokenizers_available() else None),
        ("convbert", "BertTokenizer" if is_tokenizers_available() else None),
        ("cpm", "CpmTokenizer" if is_tokenizers_available() else None),
        ("cpmant", "CpmAntTokenizer"),
        ("ctrl", "CTRLTokenizer"),
        ("data2vec-audio", "Wav2Vec2CTCTokenizer"),
        ("data2vec-text", "RobertaTokenizer"),
        ("dbrx", "GPT2Tokenizer" if is_tokenizers_available() else None),
        ("deberta", "DebertaTokenizer" if is_tokenizers_available() else None),
        ("deberta-v2", "DebertaV2Tokenizer" if is_tokenizers_available() else None),
        ("dia", "DiaTokenizer"),
        ("distilbert", "BertTokenizer" if is_tokenizers_available() else None),
        ("dpr", "DPRQuestionEncoderTokenizer" if is_tokenizers_available() else None),
        ("electra", "BertTokenizer" if is_tokenizers_available() else None),
        ("emu3", "GPT2Tokenizer" if is_tokenizers_available() else None),
        ("ernie", "BertTokenizer" if is_tokenizers_available() else None),
        ("esm", "EsmTokenizer"),
        ("falcon_mamba", "GPTNeoXTokenizer" if is_tokenizers_available() else None),
        ("fastspeech2_conformer", "FastSpeech2ConformerTokenizer" if is_g2p_en_available() else None),
        ("flaubert", "FlaubertTokenizer"),
        ("flava", "BertTokenizer" if is_tokenizers_available() else None),
        ("flex_olmo", "GPT2Tokenizer" if is_tokenizers_available() else None),
        ("florence2", "BartTokenizer" if is_tokenizers_available() else None),
        ("fnet", "FNetTokenizer" if is_tokenizers_available() else None),
        ("fsmt", "FSMTTokenizer"),
        ("funnel", "FunnelTokenizer" if is_tokenizers_available() else None),
        ("gemma", "GemmaTokenizer" if is_tokenizers_available() else None),
        ("gemma2", "GemmaTokenizer" if is_tokenizers_available() else None),
        ("gemma3", "GemmaTokenizer" if is_tokenizers_available() else None),
        ("gemma3_text", "GemmaTokenizer" if is_tokenizers_available() else None),
        ("gemma3n", "GemmaTokenizer" if is_tokenizers_available() else None),
        ("gemma3n_text", "GemmaTokenizer" if is_tokenizers_available() else None),
        ("git", "BertTokenizer" if is_tokenizers_available() else None),
        ("glm", "TokenizersBackend" if is_tokenizers_available() else None),
        ("glm4", "TokenizersBackend" if is_tokenizers_available() else None),
        ("glm4_moe", "TokenizersBackend" if is_tokenizers_available() else None),
        ("glm4_moe_lite", "TokenizersBackend" if is_tokenizers_available() else None),
        ("glm4v", "TokenizersBackend" if is_tokenizers_available() else None),
        ("glm4v_moe", "TokenizersBackend" if is_tokenizers_available() else None),
        ("glm_image", "TokenizersBackend" if is_tokenizers_available() else None),
        ("glmasr", "TokenizersBackend" if is_tokenizers_available() else None),
        ("got_ocr2", "TokenizersBackend" if is_tokenizers_available() else None),
        ("gpt-sw3", "GPTSw3Tokenizer" if is_sentencepiece_available() else None),
        ("gpt2", "GPT2Tokenizer" if is_tokenizers_available() else None),
        ("gpt_bigcode", "GPT2Tokenizer" if is_tokenizers_available() else None),
        ("gpt_neo", "GPT2Tokenizer" if is_tokenizers_available() else None),
        ("gpt_neox", "GPTNeoXTokenizer" if is_tokenizers_available() else None),
        ("gpt_neox_japanese", "GPTNeoXJapaneseTokenizer"),
        ("gptj", "GPT2Tokenizer" if is_tokenizers_available() else None),
        ("granite", "GPT2Tokenizer"),
        ("granitemoe", "GPT2Tokenizer"),
        ("granitemoehybrid", "GPT2Tokenizer"),
        ("granitemoeshared", "GPT2Tokenizer"),
        ("grounding-dino", "BertTokenizer" if is_tokenizers_available() else None),
        ("groupvit", "CLIPTokenizer" if is_tokenizers_available() else None),
        ("herbert", "HerbertTokenizer" if is_tokenizers_available() else None),
        ("hubert", "Wav2Vec2CTCTokenizer"),
        ("ibert", "RobertaTokenizer"),
        ("idefics", "LlamaTokenizer" if is_tokenizers_available() else None),
        ("idefics2", "LlamaTokenizer" if is_tokenizers_available() else None),
        ("instructblip", "GPT2Tokenizer" if is_tokenizers_available() else None),
        ("instructblipvideo", "GPT2Tokenizer" if is_tokenizers_available() else None),
        ("internvl", "Qwen2Tokenizer" if is_tokenizers_available() else None),
        ("jais2", "GPT2Tokenizer" if is_tokenizers_available() else None),
        ("jina_embeddings_v3", "XLMRobertaTokenizer" if is_tokenizers_available() else None),
        ("kosmos-2", "XLMRobertaTokenizer" if is_tokenizers_available() else None),
        ("lasr_ctc", "LasrTokenizer" if is_tokenizers_available() else None),
        ("lasr_encoder", "LasrTokenizer" if is_tokenizers_available() else None),
        ("layoutlm", "BertTokenizer" if is_tokenizers_available() else None),
        ("layoutlmv2", "LayoutLMv2Tokenizer" if is_tokenizers_available() else None),
        ("layoutlmv3", "LayoutLMv3Tokenizer" if is_tokenizers_available() else None),
        ("layoutxlm", "LayoutXLMTokenizer" if is_tokenizers_available() else None),
        ("led", "LEDTokenizer" if is_tokenizers_available() else None),
        ("lighton_ocr", "Qwen2TokenizerFast" if is_tokenizers_available() else None),
        ("lilt", "RobertaTokenizer" if is_tokenizers_available() else None),
        ("longformer", "RobertaTokenizer" if is_tokenizers_available() else None),
        ("luke", "LukeTokenizer"),
        ("lxmert", "LxmertTokenizer" if is_tokenizers_available() else None),
        ("m2m_100", "M2M100Tokenizer" if is_sentencepiece_available() else None),
        ("mamba", "GPTNeoXTokenizer" if is_tokenizers_available() else None),
        ("mamba2", "GPTNeoXTokenizer" if is_tokenizers_available() else None),
        ("marian", "MarianTokenizer" if is_sentencepiece_available() else None),
        ("markuplm", "MarkupLMTokenizer" if is_tokenizers_available() else None),
        ("mbart", "MBartTokenizer" if is_tokenizers_available() else None),
        ("mbart50", "MBart50Tokenizer" if is_tokenizers_available() else None),
        ("mega", "RobertaTokenizer"),
        ("megatron-bert", "BertTokenizer" if is_tokenizers_available() else None),
        ("metaclip_2", "XLMRobertaTokenizer" if is_tokenizers_available() else None),
        ("mgp-str", "MgpstrTokenizer"),
        (
            "ministral",
            "MistralCommonBackend"
            if is_mistral_common_available()
            else ("TokenizersBackend" if is_tokenizers_available() else None),
        ),
        (
            "ministral3",
            "MistralCommonBackend"
            if is_mistral_common_available()
            else ("TokenizersBackend" if is_tokenizers_available() else None),
        ),
        (
            "mistral",
            "MistralCommonBackend"
            if is_mistral_common_available()
            else ("TokenizersBackend" if is_tokenizers_available() else None),
        ),
        (
            "mistral3",
            "MistralCommonBackend"
            if is_mistral_common_available()
            else ("TokenizersBackend" if is_tokenizers_available() else None),
        ),
        (
            "mixtral",
            "MistralCommonBackend"
            if is_mistral_common_available()
            else ("TokenizersBackend" if is_tokenizers_available() else None),
        ),
        ("mluke", "MLukeTokenizer" if is_sentencepiece_available() else None),
        ("mm-grounding-dino", "BertTokenizer" if is_tokenizers_available() else None),
        ("mobilebert", "MobileBertTokenizer" if is_tokenizers_available() else None),
        ("mpnet", "MPNetTokenizer" if is_tokenizers_available() else None),
        ("mpt", "GPTNeoXTokenizer" if is_tokenizers_available() else None),
        ("mra", "RobertaTokenizer"),
        ("mt5", "T5Tokenizer" if is_tokenizers_available() else None),
        ("musicgen", "T5Tokenizer" if is_tokenizers_available() else None),
        ("musicgen_melody", "T5Tokenizer" if is_tokenizers_available() else None),
        ("mvp", "MvpTokenizer" if is_tokenizers_available() else None),
        ("myt5", "MyT5Tokenizer"),
        ("nezha", "BertTokenizer" if is_tokenizers_available() else None),
        ("nllb", "NllbTokenizer" if is_tokenizers_available() else None),
        ("nllb-moe", "NllbTokenizer" if is_tokenizers_available() else None),
        ("nomic_bert", "BertTokenizer" if is_tokenizers_available() else None),
        ("nougat", "NougatTokenizer" if is_tokenizers_available() else None),
        ("nystromformer", "AlbertTokenizer" if is_tokenizers_available() else None),
        ("olmo", "GPTNeoXTokenizer" if is_tokenizers_available() else None),
        ("olmo2", "GPTNeoXTokenizer" if is_tokenizers_available() else None),
        ("olmo3", "TokenizersBackend" if is_tokenizers_available() else None),
        ("olmo_hybrid", "TokenizersBackend" if is_tokenizers_available() else None),
        ("olmoe", "GPTNeoXTokenizer" if is_tokenizers_available() else None),
        ("omdet-turbo", "CLIPTokenizer" if is_tokenizers_available() else None),
        ("oneformer", "CLIPTokenizer" if is_tokenizers_available() else None),
        ("openai-gpt", "OpenAIGPTTokenizer" if is_tokenizers_available() else None),
        ("opt", "GPT2Tokenizer" if is_tokenizers_available() else None),
        ("ovis2", "Qwen2Tokenizer" if is_tokenizers_available() else None),
        ("owlv2", "CLIPTokenizer" if is_tokenizers_available() else None),
        ("owlvit", "CLIPTokenizer" if is_tokenizers_available() else None),
        ("pegasus", "PegasusTokenizer" if is_tokenizers_available() else None),
        ("pegasus_x", "PegasusTokenizer" if is_tokenizers_available() else None),
        ("perceiver", "PerceiverTokenizer"),
        ("phi", "GPT2Tokenizer" if is_tokenizers_available() else None),
        ("phobert", "PhobertTokenizer"),
        ("pix2struct", "T5Tokenizer" if is_tokenizers_available() else None),
        (
            "pixtral",
            "MistralCommonBackend"
            if is_mistral_common_available()
            else ("TokenizersBackend" if is_tokenizers_available() else None),
        ),
        ("plbart", "PLBartTokenizer" if is_tokenizers_available() else None),
        ("prophetnet", "ProphetNetTokenizer"),
        ("qdqbert", "BertTokenizer" if is_tokenizers_available() else None),
        ("qwen2", "Qwen2Tokenizer" if is_tokenizers_available() else None),
        ("qwen2_5_omni", "Qwen2Tokenizer" if is_tokenizers_available() else None),
        ("qwen2_5_vl", "Qwen2Tokenizer" if is_tokenizers_available() else None),
        ("qwen2_audio", "Qwen2Tokenizer" if is_tokenizers_available() else None),
        ("qwen2_moe", "Qwen2Tokenizer" if is_tokenizers_available() else None),
        ("qwen2_vl", "Qwen2Tokenizer" if is_tokenizers_available() else None),
        ("qwen3", "Qwen2Tokenizer" if is_tokenizers_available() else None),
        ("qwen3_5", "Qwen3_5Tokenizer" if is_tokenizers_available() else None),
        ("qwen3_5_moe", "Qwen3_5Tokenizer" if is_tokenizers_available() else None),
        ("qwen3_moe", "Qwen2Tokenizer" if is_tokenizers_available() else None),
        ("qwen3_next", "Qwen2Tokenizer" if is_tokenizers_available() else None),
        ("qwen3_omni_moe", "Qwen2Tokenizer" if is_tokenizers_available() else None),
        ("qwen3_vl", "Qwen2Tokenizer" if is_tokenizers_available() else None),
        ("qwen3_vl_moe", "Qwen2Tokenizer" if is_tokenizers_available() else None),
        ("rag", "RagTokenizer"),
        ("realm", "BertTokenizer" if is_tokenizers_available() else None),
        ("recurrent_gemma", "GemmaTokenizer" if is_tokenizers_available() else None),
        ("reformer", "ReformerTokenizer" if is_tokenizers_available() else None),
        ("rembert", "RemBertTokenizer" if is_tokenizers_available() else None),
        ("retribert", "BertTokenizer" if is_tokenizers_available() else None),
        ("roberta", "RobertaTokenizer"),
        ("roberta-prelayernorm", "RobertaTokenizer"),
        ("roc_bert", "RoCBertTokenizer"),
        ("roformer", "RoFormerTokenizer" if is_tokenizers_available() else None),
        ("rwkv", "GPTNeoXTokenizer" if is_tokenizers_available() else None),
        ("sam3", "CLIPTokenizer" if is_tokenizers_available() else None),
        ("sam3_video", "CLIPTokenizer" if is_tokenizers_available() else None),
        ("seamless_m4t", "SeamlessM4TTokenizer" if is_tokenizers_available() else None),
        ("seamless_m4t_v2", "SeamlessM4TTokenizer" if is_tokenizers_available() else None),
        ("shieldgemma2", "GemmaTokenizer" if is_tokenizers_available() else None),
        ("siglip", "SiglipTokenizer" if is_sentencepiece_available() else None),
        ("siglip2", "Siglip2Tokenizer" if is_tokenizers_available() else None),
        ("speech_to_text", "Speech2TextTokenizer" if is_sentencepiece_available() else None),
        ("speecht5", "SpeechT5Tokenizer" if is_sentencepiece_available() else None),
        ("splinter", "SplinterTokenizer"),
        ("squeezebert", "BertTokenizer" if is_tokenizers_available() else None),
        ("stablelm", "GPTNeoXTokenizer" if is_tokenizers_available() else None),
        ("starcoder2", "GPT2Tokenizer" if is_tokenizers_available() else None),
        ("switch_transformers", "T5Tokenizer" if is_tokenizers_available() else None),
        ("t5", "T5Tokenizer" if is_tokenizers_available() else None),
        ("t5gemma", "GemmaTokenizer" if is_tokenizers_available() else None),
        ("tapas", "TapasTokenizer"),
        ("trocr", "XLMRobertaTokenizer" if is_tokenizers_available() else None),
        ("tvp", "BertTokenizer" if is_tokenizers_available() else None),
        ("udop", "UdopTokenizer" if is_tokenizers_available() else None),
        ("umt5", "T5Tokenizer" if is_tokenizers_available() else None),
        ("unispeech", "Wav2Vec2CTCTokenizer"),
        ("unispeech-sat", "Wav2Vec2CTCTokenizer"),
        ("vilt", "BertTokenizer" if is_tokenizers_available() else None),
        ("visual_bert", "BertTokenizer" if is_tokenizers_available() else None),
        ("vits", "VitsTokenizer"),
        (
            "voxtral",
            "MistralCommonBackend"
            if is_mistral_common_available()
            else ("TokenizersBackend" if is_tokenizers_available() else None),
        ),
        (
            "voxtral_realtime",
            "MistralCommonBackend"
            if is_mistral_common_available()
            else ("TokenizersBackend" if is_tokenizers_available() else None),
        ),
        ("wav2vec2", "Wav2Vec2CTCTokenizer"),
        ("wav2vec2-bert", "Wav2Vec2CTCTokenizer"),
        ("wav2vec2-conformer", "Wav2Vec2CTCTokenizer"),
        ("wav2vec2_phoneme", "Wav2Vec2PhonemeCTCTokenizer"),
        ("whisper", "WhisperTokenizer" if is_tokenizers_available() else None),
        ("xclip", "CLIPTokenizer" if is_tokenizers_available() else None),
        ("xglm", "XGLMTokenizer" if is_tokenizers_available() else None),
        ("xlm", "XLMTokenizer"),
        ("xlm-roberta", "XLMRobertaTokenizer" if is_tokenizers_available() else None),
        ("xlm-roberta-xl", "XLMRobertaTokenizer" if is_tokenizers_available() else None),
        ("xlnet", "XLNetTokenizer" if is_tokenizers_available() else None),
        ("xlstm", "GPTNeoXTokenizer" if is_tokenizers_available() else None),
        ("xmod", "XLMRobertaTokenizer" if is_tokenizers_available() else None),
        ("yoso", "AlbertTokenizer" if is_tokenizers_available() else None),
    ]
)

# Models with incorrect tokenizer_class in their Hub tokenizer_config.json files.
# These models will be forced to use TokenizersBackend.
MODELS_WITH_INCORRECT_HUB_TOKENIZER_CLASS: set[str] = {
    "arctic",
    "chameleon",
    "chatlm",
    "deepseek_v2",
    "deepseek_v3",
    "deepseek_vl",
    "deepseek_vl_hybrid",
    "deepseek_vl_v2",
    "fuyu",
    "h2ovl_chat",
    "hyperclovax_vlm",
    "internlm2",
    "internvl_chat",
    "jamba",
    "janus",
    "llava",
    "llava_next",
    "minicpmv",
    "minimax_m2",
    "modernbert",
    "molmo",
    "molmo2",
    "nemotron",
    "nvfp4",
    "opencua",
    "openvla",
    "phi3",
    "phi3_v",
    "phimoe",
    "step3p5",
    "vipllava",
    "cohere_asr",
}

for model_type in MODELS_WITH_INCORRECT_HUB_TOKENIZER_CLASS:
    if model_type not in TOKENIZER_MAPPING_NAMES:
        TOKENIZER_MAPPING_NAMES[model_type] = "TokenizersBackend" if is_tokenizers_available() else None

TOKENIZER_MAPPING = _LazyAutoMapping(CONFIG_MAPPING_NAMES, TOKENIZER_MAPPING_NAMES)

CONFIG_TO_TYPE = {v: k for k, v in CONFIG_MAPPING_NAMES.items()}


def load_vocab(vocab_file):
    """Loads a vocabulary file into a dictionary."""
    with open(vocab_file, "r", encoding="utf-8") as reader:
        return json.load(reader)


def load_merges(merges_file):
    """Loads a merges file into a list."""
    merges = []
    with open(merges_file, "r", encoding="utf-8") as reader:
        for line in reader:
            line = line.strip()
            if line and not line.startswith("#"):
                merges.append(tuple(line.split()))
    return merges


def tokenizer_class_from_name(class_name: str) -> type[Any] | None:
    # Bloom tokenizer classes were removed but should map to the fast backend for BC
    if class_name in {"BloomTokenizer", "BloomTokenizerFast"}:
        return TokenizersBackend

    if class_name in REGISTERED_FAST_ALIASES:
        return REGISTERED_FAST_ALIASES[class_name]

    if class_name in REGISTERED_TOKENIZER_CLASSES:
        return REGISTERED_TOKENIZER_CLASSES[class_name]

    if class_name == "TokenizersBackend":
        return TokenizersBackend

    # V5: TOKENIZER_MAPPING_NAMES now maps to single strings, not tuples
    for module_name, tokenizer_class in TOKENIZER_MAPPING_NAMES.items():
        if tokenizer_class == class_name:
            module_name = model_type_to_module_name(module_name)
            if (
                module_name in ["mistral", "mistral3", "mixtral", "ministral", "ministral3", "pixtral", "voxtral"]
                and class_name == "MistralCommonBackend"
            ):
                module = importlib.import_module(".tokenization_mistral_common", "transformers")
            else:
                module = importlib.import_module(f".{module_name}", "transformers.models")
            try:
                result = getattr(module, class_name)
                # BC v5: expose XxxFast alias and tokenization_*_fast submodule for pre-v5 remote code.
                if (submod := getattr(result, "__module__", None)) and submod in sys.modules:
                    base_mod = sys.modules[submod]
                    setattr(base_mod, result.__name__ + "Fast", result)
                    sys.modules.setdefault(submod + "_fast", base_mod)
                return result
            except AttributeError:
                continue

    for tokenizer in TOKENIZER_MAPPING._extra_content.values():
        if getattr(tokenizer, "__name__", None) == class_name:
            return tokenizer

    # We did not find the class, but maybe it's because a dep is missing. In that case, the class will be in the main
    # We did not find the class, but maybe it's because a dep is missing. In that case, the class will be in the main
    # init and we return the proper dummy to get an appropriate error message.
    main_module = importlib.import_module("transformers")
    if hasattr(main_module, class_name):
        return getattr(main_module, class_name)

    # BC v5: If a XxxFast class is not found, retry without 'Fast' for tokenizers saved pre-v5.
    if class_name.endswith("Fast"):
        return tokenizer_class_from_name(class_name[:-4])

    return None


def get_tokenizer_config(
    pretrained_model_name_or_path: str | os.PathLike[str],
    cache_dir: str | os.PathLike[str] | None = None,
    force_download: bool = False,
    proxies: dict[str, str] | None = None,
    token: bool | str | None = None,
    revision: str | None = None,
    local_files_only: bool = False,
    subfolder: str = "",
    **kwargs,
) -> dict[str, Any]:
    """
    Loads the tokenizer configuration from a pretrained model tokenizer configuration.

    Args:
        pretrained_model_name_or_path (`str` or `os.PathLike`):
            This can be either:

            - a string, the *model id* of a pretrained model configuration hosted inside a model repo on
              huggingface.co.
            - a path to a *directory* containing a configuration file saved using the
              [`~PreTrainedTokenizer.save_pretrained`] method, e.g., `./my_model_directory/`.

        cache_dir (`str` or `os.PathLike`, *optional*):
            Path to a directory in which a downloaded pretrained model configuration should be cached if the standard
            cache should not be used.
        force_download (`bool`, *optional*, defaults to `False`):
            Whether or not to force to (re-)download the configuration files and override the cached versions if they
            exist.
        proxies (`dict[str, str]`, *optional*):
            A dictionary of proxy servers to use by protocol or endpoint, e.g., `{'http': 'foo.bar:3128',
            'http://hostname': 'foo.bar:4012'}.` The proxies are used on each request.
        token (`str` or *bool*, *optional*):
            The token to use as HTTP bearer authorization for remote files. If `True`, will use the token generated
            when running `hf auth login` (stored in `~/.huggingface`).
        revision (`str`, *optional*, defaults to `"main"`):
            The specific model version to use. It can be a branch name, a tag name, or a commit id, since we use a
            git-based system for storing models and other artifacts on huggingface.co, so `revision` can be any
            identifier allowed by git.
        local_files_only (`bool`, *optional*, defaults to `False`):
            If `True`, will only try to load the tokenizer configuration from local files.
        subfolder (`str`, *optional*, defaults to `""`):
            In case the tokenizer config is located inside a subfolder of the model repo on huggingface.co, you can
            specify the folder name here.

    <Tip>

    Passing `token=True` is required when you want to use a private model.

    </Tip>

    Returns:
        `dict`: The configuration of the tokenizer.

    Examples:

    ```python
    # Download configuration from huggingface.co and cache.
    tokenizer_config = get_tokenizer_config("google-bert/bert-base-uncased")
    # This model does not have a tokenizer config so the result will be an empty dict.
    tokenizer_config = get_tokenizer_config("FacebookAI/xlm-roberta-base")

    # Save a pretrained tokenizer locally and you can reload its config
    from transformers import AutoTokenizer

    tokenizer = AutoTokenizer.from_pretrained("google-bert/bert-base-cased")
    tokenizer.save_pretrained("tokenizer-test")
    tokenizer_config = get_tokenizer_config("tokenizer-test")
    ```"""
    commit_hash = kwargs.get("_commit_hash")
    resolved_config_file = cached_file(
        pretrained_model_name_or_path,
        TOKENIZER_CONFIG_FILE,
        cache_dir=cache_dir,
        force_download=force_download,
        proxies=proxies,
        token=token,
        revision=revision,
        local_files_only=local_files_only,
        subfolder=subfolder,
        _raise_exceptions_for_gated_repo=False,
        _raise_exceptions_for_missing_entries=False,
        _raise_exceptions_for_connection_errors=False,
        _commit_hash=commit_hash,
    )
    if resolved_config_file is None:
        logger.info("Could not locate the tokenizer configuration file, will try to use the model config instead.")
        return {}
    commit_hash = extract_commit_hash(resolved_config_file, commit_hash)

    with open(resolved_config_file, encoding="utf-8") as reader:
        result = json.load(reader)
    result["_commit_hash"] = commit_hash
    return result


class AutoTokenizer:
    r"""
    This is a generic tokenizer class that will be instantiated as one of the tokenizer classes of the library when
    created with the [`AutoTokenizer.from_pretrained`] class method.

    This class cannot be instantiated directly using `__init__()` (throws an error).
    """

    def __init__(self):
        raise OSError(
            "AutoTokenizer is designed to be instantiated "
            "using the `AutoTokenizer.from_pretrained(pretrained_model_name_or_path)` method."
        )

    @classmethod
    @replace_list_option_in_docstrings(TOKENIZER_MAPPING_NAMES)
    def from_pretrained(
        cls, pretrained_model_name_or_path, *inputs, **kwargs
    ) -> TokenizersBackend | SentencePieceBackend:
        r"""
        Instantiate one of the tokenizer classes of the library from a pretrained model vocabulary.

        The tokenizer class to instantiate is selected based on the `model_type` property of the config object (either
        passed as an argument or loaded from `pretrained_model_name_or_path` if possible), or when it's missing, by
        falling back to using pattern matching on `pretrained_model_name_or_path`:

        List options

        Params:
            pretrained_model_name_or_path (`str` or `os.PathLike`):
                Can be either:

                    - A string, the *model id* of a predefined tokenizer hosted inside a model repo on huggingface.co.
                    - A path to a *directory* containing vocabulary files required by the tokenizer, for instance saved
                      using the [`~PreTrainedTokenizer.save_pretrained`] method, e.g., `./my_model_directory/`.
                    - a path to a single saved vocabulary file if and only if the tokenizer only requires a
                      single vocabulary file (like Bert or XLNet), e.g.: `./my_model_directory/vocab.txt`. (Not
                      applicable to all derived classes)
            inputs (additional positional arguments, *optional*):
                Will be passed along to the Tokenizer `__init__()` method.
            config ([`PreTrainedConfig`], *optional*)
                The configuration object used to determine the tokenizer class to instantiate.
            cache_dir (`str` or `os.PathLike`, *optional*):
                Path to a directory in which a downloaded pretrained model configuration should be cached if the
                standard cache should not be used.
            force_download (`bool`, *optional*, defaults to `False`):
                Whether or not to force the (re-)download the model weights and configuration files and override the
                cached versions if they exist.
            proxies (`dict[str, str]`, *optional*):
                A dictionary of proxy servers to use by protocol or endpoint, e.g., `{'http': 'foo.bar:3128',
                'http://hostname': 'foo.bar:4012'}`. The proxies are used on each request.
            revision (`str`, *optional*, defaults to `"main"`):
                The specific model version to use. It can be a branch name, a tag name, or a commit id, since we use a
                git-based system for storing models and other artifacts on huggingface.co, so `revision` can be any
                identifier allowed by git.
            subfolder (`str`, *optional*):
                In case the relevant files are located inside a subfolder of the model repo on huggingface.co (e.g. for
                facebook/rag-token-base), specify it here.
            tokenizer_type (`str`, *optional*):
                Tokenizer type to be loaded.
            backend (`str`, *optional*, defaults to `"tokenizers"`):
                Backend to use for tokenization. Valid options are:
                - `"tokenizers"`: Use the HuggingFace tokenizers library backend (default)
                - `"sentencepiece"`: Use the SentencePiece backend
            trust_remote_code (`bool`, *optional*, defaults to `False`):
                Whether or not to allow for custom models defined on the Hub in their own modeling files. This option
                should only be set to `True` for repositories you trust and in which you have read the code, as it will
                execute code present on the Hub on your local machine.
            kwargs (additional keyword arguments, *optional*):
                Will be passed to the Tokenizer `__init__()` method. Can be used to set special tokens like
                `bos_token`, `eos_token`, `unk_token`, `sep_token`, `pad_token`, `cls_token`, `mask_token`,
                `additional_special_tokens`. See parameters in the `__init__()` for more details.

        Examples:

        ```python
        >>> from transformers import AutoTokenizer

        >>> # Download vocabulary from huggingface.co and cache.
        >>> tokenizer = AutoTokenizer.from_pretrained("google-bert/bert-base-uncased")

        >>> # Download vocabulary from huggingface.co (user-uploaded) and cache.
        >>> tokenizer = AutoTokenizer.from_pretrained("dbmdz/bert-base-german-cased")

        >>> # If vocabulary files are in a directory (e.g. tokenizer was saved using *save_pretrained('./test/saved_model/')*)
        >>> # tokenizer = AutoTokenizer.from_pretrained("./test/bert_saved_model/")

        >>> # Download vocabulary from huggingface.co and define model-specific arguments
        >>> tokenizer = AutoTokenizer.from_pretrained("FacebookAI/roberta-base", add_prefix_space=True)

        >>> # Explicitly use the tokenizers backend
        >>> tokenizer = AutoTokenizer.from_pretrained("hf-internal-testing/llama-tokenizer", backend="tokenizers")

        >>> # Explicitly use the sentencepiece backend
        >>> tokenizer = AutoTokenizer.from_pretrained("hf-internal-testing/llama-tokenizer", backend="sentencepiece")
        ```"""
        config = kwargs.pop("config", None)
        kwargs["_from_auto"] = True

        # V5: Always use fast tokenizers, ignore use_fast parameter
        _ = kwargs.pop("use_fast", None)
        tokenizer_type = kwargs.pop("tokenizer_type", None)
        trust_remote_code = kwargs.pop("trust_remote_code", None)
        gguf_file = kwargs.get("gguf_file")

        # First, let's see whether the tokenizer_type is passed so that we can leverage it
        if tokenizer_type is not None:
            tokenizer_class_name = TOKENIZER_MAPPING_NAMES.get(tokenizer_type, None)

            if tokenizer_class_name is None:
                raise ValueError(
                    f"Passed `tokenizer_type` {tokenizer_type} does not exist. `tokenizer_type` should be one of "
                    f"{', '.join(c for c in TOKENIZER_MAPPING_NAMES)}."
                )

            tokenizer_class = tokenizer_class_from_name(tokenizer_class_name)

            if tokenizer_class is None:
                raise ValueError(f"Tokenizer class {tokenizer_class_name} is not currently imported.")

            return tokenizer_class.from_pretrained(pretrained_model_name_or_path, *inputs, **kwargs)

        if gguf_file:
            gguf_path = cached_file(pretrained_model_name_or_path, gguf_file, **kwargs)
            config_dict = load_gguf_checkpoint(gguf_path, return_tensors=False)["config"]
            config = AutoConfig.for_model(**config_dict)
        elif config is None:
            try:
                config = AutoConfig.from_pretrained(
                    pretrained_model_name_or_path, trust_remote_code=trust_remote_code, **kwargs
                )
            except (ValueError, OSError):
                config = PreTrainedConfig.from_pretrained(pretrained_model_name_or_path, **kwargs)

        config_model_type = config.model_type

        # Next, let's try to use the tokenizer_config file to get the tokenizer class.
        tokenizer_config = get_tokenizer_config(pretrained_model_name_or_path, **kwargs)
        tokenizer_config_class = tokenizer_config.get("tokenizer_class", None)

        # Check for auto_map early to handle dynamic tokenizers properly
        tokenizer_auto_map = None
        if "auto_map" in tokenizer_config:
            if isinstance(tokenizer_config["auto_map"], (tuple, list)):
                # Legacy format for dynamic tokenizers
                tokenizer_auto_map = tokenizer_config["auto_map"]
            else:
                tokenizer_auto_map = tokenizer_config["auto_map"].get("AutoTokenizer", None)

        # if there is a config, we can check that the tokenizer class != than model class and can thus assume we need to use TokenizersBackend
        # Skip this early exit if auto_map is present (custom tokenizer with trust_remote_code)
        if (
            tokenizer_auto_map is None
            and tokenizer_config_class is not None
            and config_model_type is not None
            and config_model_type != ""
            and TOKENIZER_MAPPING_NAMES.get(config_model_type) is not None
            and (TOKENIZER_MAPPING_NAMES.get(config_model_type).removesuffix("Fast"))
            != (tokenizer_config_class.removesuffix("Fast"))
        ):
            # new model, but we ignore it unless the model type is the same
            if TokenizersBackend is not None:
                try:
                    return TokenizersBackend.from_pretrained(pretrained_model_name_or_path, *inputs, **kwargs)
                except Exception as e:
                    logger.debug(f"Failed to use TokenizersBackend: {e}")

            return tokenizer_class_from_name(tokenizer_config_class).from_pretrained(
                pretrained_model_name_or_path, *inputs, **kwargs
            )

        if "_commit_hash" in tokenizer_config:
            kwargs["_commit_hash"] = tokenizer_config["_commit_hash"]

        if tokenizer_config_class and tokenizer_config_class.endswith("Fast"):
            tokenizer_config_class = tokenizer_config_class[:-4]

        has_remote_code = tokenizer_auto_map is not None
        has_local_code = type(config) in TOKENIZER_MAPPING or (
            tokenizer_config_class is not None
            and (
                tokenizer_class_from_name(tokenizer_config_class) is not None
                or tokenizer_class_from_name(tokenizer_config_class + "Fast") is not None
            )
        )
        explicit_local_code = (
            has_local_code
            and type(config) not in TOKENIZER_MAPPING
            and (
                tokenizer_config_class is not None
                and not (
                    tokenizer_class_from_name(tokenizer_config_class)
                    or tokenizer_class_from_name(tokenizer_config_class + "Fast")
                ).__module__.startswith("transformers.")
            )
        )
        # V5: Skip remote tokenizer for custom models with incorrect hub tokenizer class
        if has_remote_code and config_model_type in MODELS_WITH_INCORRECT_HUB_TOKENIZER_CLASS:
            has_remote_code = False
            tokenizer_auto_map = None

        if has_remote_code:
            # V5: Always prefer fast tokenizer (index 1), fallback to slow (index 0)
            if tokenizer_auto_map[1] is not None:
                class_ref = tokenizer_auto_map[1]
            else:
                class_ref = tokenizer_auto_map[0]
            if "--" in class_ref:
                upstream_repo = class_ref.split("--")[0]
            else:
                upstream_repo = None
            trust_remote_code = resolve_trust_remote_code(
                trust_remote_code, pretrained_model_name_or_path, has_local_code, has_remote_code, upstream_repo
            )

        if has_remote_code and trust_remote_code and not explicit_local_code:
            # BC v5: register *Fast aliases before remote code loads.
            if tokenizer_config_class:
                tokenizer_class_from_name(tokenizer_config_class.removesuffix("Fast"))
            tokenizer_class = get_class_from_dynamic_module(class_ref, pretrained_model_name_or_path, **kwargs)
            _ = kwargs.pop("code_revision", None)
            tokenizer_class.register_for_auto_class()
            return tokenizer_class.from_pretrained(
                pretrained_model_name_or_path, *inputs, trust_remote_code=trust_remote_code, **kwargs
            )
        elif tokenizer_config_class is not None:
            tokenizer_class_candidate = tokenizer_config_class
            tokenizer_class = tokenizer_class_from_name(tokenizer_class_candidate)
            if tokenizer_class is None and not tokenizer_class_candidate.endswith("Fast"):
                tokenizer_class = tokenizer_class_from_name(tokenizer_class_candidate + "Fast")
            if tokenizer_class is not None and tokenizer_class.__name__ == "PythonBackend":
                tokenizer_class = TokenizersBackend
            # Fallback to TokenizersBackend if the class wasn't found
            if tokenizer_class is None:
                tokenizer_class = TokenizersBackend

            return tokenizer_class.from_pretrained(pretrained_model_name_or_path, *inputs, **kwargs)
        elif getattr(config, "tokenizer_class", None):
            _class = config.tokenizer_class
            if "PreTrainedTokenizerFast" not in _class and _class.endswith("Fast"):
                _class = _class[:-4]
            tokenizer_class = tokenizer_class_from_name(_class)
            return tokenizer_class.from_pretrained(pretrained_model_name_or_path, *inputs, **kwargs)

        # Otherwise we have to be creative.
        # if model is an encoder decoder, the encoder tokenizer class is used by default
        if isinstance(config, EncoderDecoderConfig):
            if type(config.decoder) is not type(config.encoder):
                logger.warning(
                    f"The encoder model config class: {config.encoder.__class__} is different from the decoder model "
                    f"config class: {config.decoder.__class__}. It is not recommended to use the "
                    "`AutoTokenizer.from_pretrained()` method in this case. Please use the encoder and decoder "
                    "specific tokenizer classes."
                )
            config = config.encoder

        model_type = config_class_to_model_type(type(config).__name__) or getattr(config, "model_type", None)
        if model_type is not None:
            tokenizer_class = TOKENIZER_MAPPING.get(type(config), TokenizersBackend)
            if tokenizer_class is not None:
                return tokenizer_class.from_pretrained(pretrained_model_name_or_path, *inputs, **kwargs)

        # Fallback: try tokenizer_class from tokenizer_config.json
        tokenizer_config_class = tokenizer_config.get("tokenizer_class", None)
        if tokenizer_config_class is not None:
            if tokenizer_config_class != "TokenizersBackend" and tokenizer_config_class.endswith("Fast"):
                tokenizer_config_class = tokenizer_config_class[:-4]
            tokenizer_class = tokenizer_class_from_name(tokenizer_config_class)
            if tokenizer_class is None and not tokenizer_config_class.endswith("Fast"):
                tokenizer_class = tokenizer_class_from_name(tokenizer_config_class + "Fast")
            if tokenizer_class is not None and tokenizer_class.__name__ == "PythonBackend":
                tokenizer_class = TokenizersBackend
            if tokenizer_class is None:
                tokenizer_class = TokenizersBackend
            return tokenizer_class.from_pretrained(pretrained_model_name_or_path, *inputs, **kwargs)

        raise ValueError(
            f"Unrecognized configuration class {config.__class__} to build an AutoTokenizer.\n"
            f"Model type should be one of {', '.join(c.__name__ for c in TOKENIZER_MAPPING)}."
        )

    @staticmethod
    def register(
        config_class, tokenizer_class=None, slow_tokenizer_class=None, fast_tokenizer_class=None, exist_ok=False
    ):
        """
        Register a new tokenizer in this mapping.

        Args:
            config_class ([`PreTrainedConfig`]):
                The configuration corresponding to the model to register.
            tokenizer_class: The tokenizer class to register (V5 - preferred parameter).
            slow_tokenizer_class: (Deprecated) The slow tokenizer to register.
            fast_tokenizer_class: (Deprecated) The fast tokenizer to register.
        """
        if tokenizer_class is None:
            # Legacy: prefer fast over slow
            if fast_tokenizer_class is not None:
                tokenizer_class = fast_tokenizer_class
            elif slow_tokenizer_class is not None:
                tokenizer_class = slow_tokenizer_class
            else:
                raise ValueError("You need to pass a `tokenizer_class`")

        for candidate in (slow_tokenizer_class, fast_tokenizer_class, tokenizer_class):
            if candidate is not None:
                REGISTERED_TOKENIZER_CLASSES[candidate.__name__] = candidate

        if slow_tokenizer_class is not None and fast_tokenizer_class is not None:
            REGISTERED_FAST_ALIASES[slow_tokenizer_class.__name__] = fast_tokenizer_class

        TOKENIZER_MAPPING.register(config_class, tokenizer_class, exist_ok=exist_ok)


__all__ = ["TOKENIZER_MAPPING", "AutoTokenizer"]