repo_description: 🤗 Transformers: the model-definition framework for state-of-the-art machine learning models in text, vision, audio, and multimodal models, for both inference and training. 
repo_topics: audio deep-learning deepseek gemma glm hacktoberfest llm machine-learning model-hub natural-language-processing nlp pretrained-models python pytorch pytorch-transformers qwen speech-recognition transformer vlm
code: # Copyright 2021 The HuggingFace Team. All rights reserved.
  #
  # Licensed under the Apache License, Version 2.0 (the "License");
  # you may not use this file except in compliance with the License.
  # You may obtain a copy of the License at
  #
  #     http://www.apache.org/licenses/LICENSE-2.0
  #
  # Unless required by applicable law or agreed to in writing, software
  # distributed under the License is distributed on an "AS IS" BASIS,
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  # See the License for the specific language governing permissions and
  # limitations under the License.
  
  """
  Simple check list from AllenNLP repo: https://github.com/allenai/allennlp/blob/main/setup.py
  
  To create the package for pypi.
  
  1. Create the release branch named: v<RELEASE>-release, for example v4.19-release. For a patch release checkout the
     current release branch.
  
     If releasing on a special branch, copy the updated README.md on the main branch for the commit you will make
     for the post-release and run `make fix-repo` on the main branch as well.
  
  2. Run `make pre-release` (or `make pre-patch` for a patch release) and commit these changes with the message:
     "Release: <VERSION>" and push.
  
  3. Go back to the main branch and run `make post-release` then `make fix-repo`. Commit these changes with the
     message "v<NEXT_VERSION>.dev.0" and push to main.
  
  # If you were just cutting the branch in preparation for a release, you can stop here for now.
  
  4. Wait for the tests on the release branch to be completed and be green (otherwise revert and fix bugs)
  
  5. On the release branch, add a tag in git to mark the release: "git tag v<VERSION> -m 'Adds tag v<VERSION> for pypi' "
     Push the tag to git: git push --tags origin v<RELEASE>-release
  
  6. Have a core maintainer review and approve the deployment to pypi.
  """
  
  import re
  import shutil
  import sys
  from pathlib import Path
  
  from setuptools import Command, find_packages, setup
  
  
  # Supported Python version range (min, max)
  SUPPORTED_PYTHON_VERSIONS = (10, 14)  # 3.10 to 3.14
  
  PYTHON_MINOR_VERSION = sys.version_info.minor
  
  # Remove stale transformers.egg-info directory to avoid https://github.com/pypa/pip/issues/5466
  stale_egg_info = Path(__file__).parent / "transformers.egg-info"
  if stale_egg_info.exists():
      print(
          f"Warning: {stale_egg_info} exists.\n\n"
          "If you recently updated transformers to 3.0 or later, this is expected,\n"
          "but it may prevent transformers from installing in editable mode.\n\n"
          "This directory is automatically generated by Python's packaging tools.\n"
          "I will remove it now.\n\n"
          "See https://github.com/pypa/pip/issues/5466 for details.\n"
      )
      shutil.rmtree(stale_egg_info)
  
  
  # IMPORTANT:
  # 1. all dependencies should be listed here with their version requirements if any
  # 2. once modified, run: `make fix-repo` to update src/transformers/dependency_versions_table.py
  _deps = [
      "Pillow>=10.0.1,<=15.0",
      "accelerate>=1.1.0",
      "av",
      "beautifulsoup4",
      "blobfile",
      "codecarbon>=2.8.1",
      "datasets>=2.15.0",  # We need either this pin or pyarrow<21.0.0
      "deepspeed>=0.9.3",
      "dill<0.3.5",
      "evaluate>=0.4.6",
      "faiss-cpu",
      "fastapi",
      "filelock",
      "fugashi>=1.0",
      "GitPython<3.1.19",
      "hf-doc-builder",
      "huggingface-hub>=1.5.0,<2.0",
      "ipadic>=1.0.0,<2.0",
      "jinja2>=3.1.0",
      "jmespath>=1.0.1",
      "kenlm",
      "kernels>=0.12.0,<0.13",
      "librosa",
      "mistral-common[image]>=1.10.0",
      "nltk<=3.8.1",
      "num2words",
      "numpy>=1.17",
      "openai>=1.98.0",
      "opencv-python",
      "optimum-benchmark>=0.3.0",
      "optuna",
      "pandas<2.3.0",  # `datasets` requires `pandas` while `pandas==2.3.0` has issues with CircleCI on 2025/06/05
      "packaging>=20.0",
      "parameterized>=0.9",  # older version of parameterized cause pytest collection to fail on .expand
      "peft>=0.18.0",
      "phonemizer",
      "protobuf",
      "psutil",
      "pyyaml>=5.1",
      "pydantic>=2",
      "pytest>=7.2.0,<9.0.0",
      "pytest-asyncio>=1.2.0",
      "pytest-random-order",
      "pytest-rerunfailures<16.0",
      "pytest-timeout",
      "pytest-env",
      "pytest-xdist",
      "pytest-order",
      "python>=3.10.0",
      "regex>=2025.10.22",
      "rhoknp>=1.1.0,<1.3.1",
      "rjieba",
      "rouge-score!=0.0.7,!=0.0.8,!=0.1,!=0.1.1",
      "ruff==0.14.10",
      "transformers-mlinter==0.1.0",
      "ty==0.0.20",
      # `sacrebleu` not used in `transformers`. However, it is needed in several tests, when a test calls
      # `evaluate.load("sacrebleu")`. This metric is used in the examples that we use to test the `Trainer` with, in the
      # `Trainer` tests (see references to `run_translation.py`).
      "sacrebleu>=1.4.12,<2.0.0",
      "sacremoses",
      "safetensors>=0.4.3",
      "sagemaker>=2.31.0",
      "schedulefree>=1.2.6",
      "scikit-learn",
      "scipy",
      "sentencepiece>=0.1.91,!=0.1.92",
      "starlette",
      "sudachipy>=0.6.6",
      "sudachidict_core>=20220729",
      "tensorboard",
      "timeout-decorator",
      "tomli",
      "tiktoken",
      "timm>=1.0.23",
      "tokenizers>=0.22.0,<=0.23.0",
      "torch>=2.4",
      "torchaudio",
      "torchvision",
      "pyctcdecode>=0.4.0",
      "tqdm>=4.27",
      "typer",
      "unidic>=1.0.2",
      "unidic_lite>=1.0.7",
      "urllib3<2.0.0",
      "uvicorn",
      "pytest-rich",
      "libcst",
      "rich",
      "ray[tune]>=2.7.0",
      "opentelemetry-api",
      "opentelemetry-exporter-otlp",
      "opentelemetry-sdk",
  ]
  
  # This is a lookup table with items like: {"tokenizers": "tokenizers==0.9.4", "packaging": "packaging"}, i.e.
  # some of the values are versioned whereas others aren't.
  deps = {b: a for a, b in (re.findall(r"^(([^!=<>~ ]+)(?:[!=<>~ ].*)?$)", x)[0] for x in _deps)}
  
  
  def deps_list(*pkgs):
      return [deps[pkg] for pkg in pkgs]
  
  
  extras = {}
  
  extras["torch"] = deps_list("torch", "accelerate")
  extras["vision"] = deps_list("torchvision", "Pillow")
  extras["audio"] = deps_list("torchaudio", "librosa", "pyctcdecode", "phonemizer")
  if PYTHON_MINOR_VERSION < 13:
      extras["audio"] += deps_list("kenlm")
  extras["video"] = deps_list("av")
  extras["timm"] = deps_list("timm")
  extras["quality"] = deps_list(
      "datasets", "ruff", "GitPython", "urllib3", "libcst", "rich", "ty", "tomli", "transformers-mlinter"
  )
  extras["docs"] = deps_list("hf-doc-builder")
  extras["kernels"] = deps_list("kernels")
  extras["sentencepiece"] = deps_list("sentencepiece", "protobuf")
  extras["tiktoken"] = deps_list("tiktoken", "blobfile")
  extras["mistral-common"] = deps_list("mistral-common[image]")
  extras["chat_template"] = deps_list("jinja2", "jmespath")
  extras["sklearn"] = deps_list("scikit-learn")
  extras["accelerate"] = deps_list("accelerate")
  extras["retrieval"] = deps_list("faiss-cpu", "datasets")
  extras["sagemaker"] = deps_list("sagemaker")
  extras["deepspeed"] = deps_list("deepspeed", "accelerate")
  extras["optuna"] = deps_list("optuna")
  extras["integrations"] = deps_list("kernels", "optuna", "codecarbon")
  if PYTHON_MINOR_VERSION < 14:
      extras["ray"] = deps_list("ray[tune]")
      extras["integrations"] += extras["ray"]
  extras["codecarbon"] = deps_list("codecarbon")
  extras["serving"] = deps_list("openai", "pydantic", "uvicorn", "fastapi", "starlette", "rich") + extras["torch"]
  extras["num2words"] = deps_list("num2words")
  extras["benchmark"] = deps_list("optimum-benchmark")
  extras["ja"] = deps_list("fugashi", "ipadic", "unidic_lite", "unidic", "rhoknp")
  if PYTHON_MINOR_VERSION < 14:
      extras["ja"] += deps_list("sudachipy", "sudachidict_core")
  # OpenTelemetry dependencies for metrics collection in continuous batching
  extras["open-telemetry"] = deps_list("opentelemetry-api", "opentelemetry-exporter-otlp", "opentelemetry-sdk")
  
  extras["testing"] = (
      deps_list(
          "pytest",
          "pytest-asyncio",
          "pytest-random-order",
          "pytest-rich",
          "pytest-xdist",
          "pytest-order",
          "pytest-rerunfailures",
          "pytest-timeout",
          "pytest-env",
          "timeout-decorator",
          "parameterized",
          "psutil",
          "dill",
          "evaluate",
          "rouge-score",
          "nltk",
          "sacremoses",
          "rjieba",
          "beautifulsoup4",
          "tensorboard",
          "sacrebleu",  # needed in trainer tests, see references to `run_translation.py`
          "filelock",  # filesystem locks, e.g., to prevent parallel downloads
      )
      + extras["docs"]
      + extras["quality"]
      + extras["retrieval"]
      + extras["sentencepiece"]
      + extras["serving"]
  )
  extras["testing"] += extras["mistral-common"]
  
  extras["deepspeed-testing"] = extras["deepspeed"] + extras["testing"] + extras["optuna"] + extras["sentencepiece"]
  extras["all"] = (
      extras["torch"]
      + extras["vision"]
      + extras["audio"]
      + extras["video"]
      + extras["kernels"]
      + extras["timm"]
      + extras["sentencepiece"]
      + extras["tiktoken"]
      + extras["chat_template"]
      + extras["num2words"]
  )
  extras["all"] += extras["mistral-common"]
  
  extras["dev"] = extras["all"] + extras["testing"] + extras["ja"] + extras["sklearn"]
  
  # Those define the hard dependencies of `transformers`
  install_requires = [
      deps["huggingface-hub"],
      deps["numpy"],
      deps["packaging"],  # utilities from PyPA to e.g., compare versions
      deps["pyyaml"],  # used for the model cards metadata
      deps["regex"],  # for OpenAI GPT
      deps["tokenizers"],
      deps["typer"],  # CLI utilities. In practice, already a dependency of huggingface_hub but we use it as well
      deps["safetensors"],
      deps["tqdm"],  # progress bars in model download and training scripts
  ]
  
  
  class DepsTableUpdateCommand(Command):
      """
      A custom distutils command that updates the dependency table.
      usage: python setup.py deps_table_update
      """
  
      description = "build runtime dependency table"
      user_options = [
          # format: (long option, short option, description).
          ("dep-table-update", None, "updates src/transformers/dependency_versions_table.py"),
      ]
  
      def initialize_options(self):
          pass
  
      def finalize_options(self):
          pass
  
      def run(self):
          if SUPPORTED_PYTHON_VERSIONS[0] > PYTHON_MINOR_VERSION:
              print(
                  f"Table updated only when running 3.{SUPPORTED_PYTHON_VERSIONS[0]}.x, detected version is {sys.version}."
              )
              return
  
          entries = "\n".join([f'    "{k}": "{v}",' for k, v in deps.items()])
          content = [
              "# THIS FILE HAS BEEN AUTOGENERATED. To update:",
              "# 1. modify the `_deps` dict in setup.py",
              "# 2. run `make fix-repo``",
              "deps = {",
              entries,
              "}",
              "",
          ]
          target = "src/transformers/dependency_versions_table.py"
          with open(target, "w", encoding="utf-8", newline="\n") as f:
              f.write("\n".join(content))
  
  
  if __name__ == "__main__":
      # Generate python_requires from supported version range
      min_version, max_version = SUPPORTED_PYTHON_VERSIONS
      python_requires = f">=3.{min_version}.0"
  
      # Generate Python version classifiers dynamically
      python_classifiers = ["Programming Language :: Python :: 3"]
      for minor in range(min_version, max_version + 1):
          python_classifiers.append(f"Programming Language :: Python :: 3.{minor}")
  
      setup(
          name="transformers",
          version="5.6.0.dev0",  # expected format is one of x.y.z.dev0, or x.y.z.rc1 or x.y.z (no to dashes, yes to dots)
          author="The Hugging Face team (past and future) with the help of all our contributors (https://github.com/huggingface/transformers/graphs/contributors)",
          author_email="[REDACTED_EMAIL]",
          description="Transformers: the model-definition framework for state-of-the-art machine learning models in text, vision, audio, and multimodal models, for both inference and training.",
          long_description=open("README.md", "r", encoding="utf-8").read(),
          long_description_content_type="text/markdown",
          keywords="machine-learning nlp python pytorch transformer llm vlm deep-learning inference training model-hub pretrained-models llama gemma qwen",
          license="Apache 2.0 License",
          url="https://github.com/huggingface/transformers",
          package_dir={"": "src"},
          packages=find_packages("src"),
          include_package_data=True,
          package_data={"": ["**/*.cu", "**/*.cpp", "**/*.cuh", "**/*.h", "**/*.pyx", "py.typed"]},
          zip_safe=False,
          extras_require=extras,
          entry_points={"console_scripts": ["transformers=transformers.cli.transformers:main"]},
          python_requires=python_requires,
          install_requires=list(install_requires),
          classifiers=[
              "Development Status :: 5 - Production/Stable",
              "Intended Audience :: Developers",
              "Intended Audience :: Education",
              "Intended Audience :: Science/Research",
              "Operating System :: OS Independent",
          ]
          + python_classifiers
          + [
              "Topic :: Scientific/Engineering :: Artificial Intelligence",
          ],
          cmdclass={"deps_table_update": DepsTableUpdateCommand},
      )
  