From 4744648a8dd4f59b0c6cf96e3d4ec7561cca00fd Mon Sep 17 00:00:00 2001 From: Sayak Paul Date: Thu, 30 Apr 2026 14:34:50 +0530 Subject: [PATCH] [ci] use tokenizers stable installtion in CI. (#13562) * use tokenizers stable installtion in CI. * up1 * up2 * up3 --- .github/workflows/nightly_tests.yml | 28 ++++++++++++------------ .github/workflows/pr_modular_tests.yml | 4 ++-- .github/workflows/pr_tests.yml | 4 +++- .github/workflows/pr_tests_gpu.yml | 3 +++ .github/workflows/push_tests.yml | 3 +++ .github/workflows/push_tests_mps.yml | 1 + .github/workflows/pypi_publish.yaml | 1 + .github/workflows/release_tests_fast.yml | 7 ++++++ 8 files changed, 34 insertions(+), 17 deletions(-) diff --git a/.github/workflows/nightly_tests.yml b/.github/workflows/nightly_tests.yml index a3f29dbd7e..4bf5f88633 100644 --- a/.github/workflows/nightly_tests.yml +++ b/.github/workflows/nightly_tests.yml @@ -72,9 +72,9 @@ jobs: - name: Install dependencies run: | uv pip install -e ".[quality]" + uv pip uninstall transformers huggingface_hub && uv pip install --prerelease allow -U transformers@git+https://github.com/huggingface/transformers.git + uv pip uninstall tokenizers && uv pip install "tokenizers<=0.23.0" uv pip uninstall accelerate && uv pip install -U accelerate@git+https://github.com/huggingface/accelerate.git - #uv pip uninstall transformers huggingface_hub && uv pip install --prerelease allow -U transformers@git+https://github.com/huggingface/transformers.git - uv pip uninstall transformers huggingface_hub && uv pip install transformers==4.57.1 uv pip install pytest-reportlog - name: Environment run: | @@ -126,10 +126,10 @@ jobs: - name: Install dependencies run: | uv pip install -e ".[quality]" + uv pip uninstall transformers huggingface_hub && uv pip install --prerelease allow -U transformers@git+https://github.com/huggingface/transformers.git + uv pip uninstall tokenizers && uv pip install "tokenizers<=0.23.0" uv pip install peft@git+https://github.com/huggingface/peft.git uv pip uninstall accelerate && uv pip install -U accelerate@git+https://github.com/huggingface/accelerate.git - #uv pip uninstall transformers huggingface_hub && uv pip install --prerelease allow -U transformers@git+https://github.com/huggingface/transformers.git - uv pip uninstall transformers huggingface_hub && uv pip install transformers==4.57.1 uv pip install pytest-reportlog - name: Environment run: python utils/print_env.py @@ -194,8 +194,8 @@ jobs: - name: Install dependencies run: | uv pip install -e ".[quality,training]" - #uv pip uninstall transformers huggingface_hub && uv pip install --prerelease allow -U transformers@git+https://github.com/huggingface/transformers.git - uv pip uninstall transformers huggingface_hub && uv pip install transformers==4.57.1 + uv pip uninstall transformers huggingface_hub && uv pip install --prerelease allow -U transformers@git+https://github.com/huggingface/transformers.git + uv pip uninstall tokenizers && uv pip install "tokenizers<=0.23.0" - name: Environment run: | python utils/print_env.py @@ -236,10 +236,10 @@ jobs: - name: Install dependencies run: | uv pip install -e ".[quality]" + uv pip uninstall transformers huggingface_hub && uv pip install --prerelease allow -U transformers@git+https://github.com/huggingface/transformers.git + uv pip uninstall tokenizers && uv pip install "tokenizers<=0.23.0" uv pip install peft@git+https://github.com/huggingface/peft.git uv pip uninstall accelerate && uv pip install -U accelerate@git+https://github.com/huggingface/accelerate.git - #uv pip uninstall transformers huggingface_hub && uv pip install --prerelease allow -U transformers@git+https://github.com/huggingface/transformers.git - uv pip uninstall transformers huggingface_hub && uv pip install transformers==4.57.1 uv pip install pytest-reportlog - name: Environment run: | @@ -287,10 +287,10 @@ jobs: - name: Install dependencies run: | uv pip install -e ".[quality]" + uv pip uninstall transformers huggingface_hub && uv pip install --prerelease allow -U transformers@git+https://github.com/huggingface/transformers.git + uv pip uninstall tokenizers && uv pip install "tokenizers<=0.23.0" uv pip install peft@git+https://github.com/huggingface/peft.git uv pip uninstall accelerate && uv pip install -U accelerate@git+https://github.com/huggingface/accelerate.git - #uv pip uninstall transformers huggingface_hub && uv pip install --prerelease allow -U transformers@git+https://github.com/huggingface/transformers.git - uv pip uninstall transformers huggingface_hub && uv pip install transformers==4.57.1 - name: Environment run: | @@ -368,8 +368,8 @@ jobs: uv pip install ${{ join(matrix.config.additional_deps, ' ') }} fi uv pip install pytest-reportlog - #uv pip uninstall transformers huggingface_hub && uv pip install --prerelease allow -U transformers@git+https://github.com/huggingface/transformers.git - uv pip uninstall transformers huggingface_hub && uv pip install transformers==4.57.1 + uv pip uninstall transformers huggingface_hub && uv pip install --prerelease allow -U transformers@git+https://github.com/huggingface/transformers.git + uv pip uninstall tokenizers && uv pip install "tokenizers<=0.23.0" - name: Environment run: | python utils/print_env.py @@ -417,8 +417,8 @@ jobs: run: | uv pip install -e ".[quality]" uv pip install -U bitsandbytes optimum_quanto - #uv pip uninstall transformers huggingface_hub && uv pip install --prerelease allow -U transformers@git+https://github.com/huggingface/transformers.git - uv pip uninstall transformers huggingface_hub && uv pip install transformers==4.57.1 + uv pip uninstall transformers huggingface_hub && uv pip install --prerelease allow -U transformers@git+https://github.com/huggingface/transformers.git + uv pip uninstall tokenizers && uv pip install "tokenizers<=0.23.0" uv pip install pytest-reportlog - name: Environment run: | diff --git a/.github/workflows/pr_modular_tests.yml b/.github/workflows/pr_modular_tests.yml index 89b502d364..bbdb9dd327 100644 --- a/.github/workflows/pr_modular_tests.yml +++ b/.github/workflows/pr_modular_tests.yml @@ -118,8 +118,8 @@ jobs: - name: Install dependencies run: | uv pip install -e ".[quality]" - #uv pip uninstall transformers huggingface_hub && uv pip install --prerelease allow -U transformers@git+https://github.com/huggingface/transformers.git - uv pip uninstall transformers huggingface_hub && uv pip install transformers==4.57.1 + uv pip uninstall transformers huggingface_hub && uv pip install --prerelease allow -U transformers@git+https://github.com/huggingface/transformers.git + uv pip uninstall tokenizers && uv pip install "tokenizers<=0.23.0" uv pip uninstall accelerate && uv pip install -U accelerate@git+https://github.com/huggingface/accelerate.git --no-deps - name: Environment diff --git a/.github/workflows/pr_tests.yml b/.github/workflows/pr_tests.yml index 02dee7d541..1cd73566e8 100644 --- a/.github/workflows/pr_tests.yml +++ b/.github/workflows/pr_tests.yml @@ -118,6 +118,7 @@ jobs: run: | uv pip install -e ".[quality]" uv pip uninstall transformers huggingface_hub && uv pip install --prerelease allow -U transformers@git+https://github.com/huggingface/transformers.git + uv pip uninstall tokenizers && uv pip install "tokenizers<=0.23.0" uv pip uninstall accelerate && uv pip install -U accelerate@git+https://github.com/huggingface/accelerate.git --no-deps - name: Environment @@ -247,7 +248,8 @@ jobs: uv pip install -U tokenizers uv pip uninstall accelerate && uv pip install -U accelerate@git+https://github.com/huggingface/accelerate.git --no-deps uv pip uninstall transformers huggingface_hub && uv pip install --prerelease allow -U transformers@git+https://github.com/huggingface/transformers.git - + uv pip uninstall tokenizers && uv pip install "tokenizers<=0.23.0" + - name: Environment run: | python utils/print_env.py diff --git a/.github/workflows/pr_tests_gpu.yml b/.github/workflows/pr_tests_gpu.yml index 9c63ad755f..1791add434 100644 --- a/.github/workflows/pr_tests_gpu.yml +++ b/.github/workflows/pr_tests_gpu.yml @@ -135,6 +135,7 @@ jobs: uv pip install -e ".[quality]" uv pip uninstall accelerate && uv pip install -U accelerate@git+https://github.com/huggingface/accelerate.git uv pip uninstall transformers huggingface_hub && uv pip install --prerelease allow -U transformers@git+https://github.com/huggingface/transformers.git + uv pip uninstall tokenizers && uv pip install "tokenizers<=0.23.0" - name: Environment run: | @@ -205,6 +206,7 @@ jobs: uv pip install peft@git+https://github.com/huggingface/peft.git uv pip uninstall accelerate && uv pip install -U accelerate@git+https://github.com/huggingface/accelerate.git uv pip uninstall transformers huggingface_hub && uv pip install --prerelease allow -U transformers@git+https://github.com/huggingface/transformers.git + uv pip uninstall tokenizers && uv pip install "tokenizers<=0.23.0" - name: Environment run: | @@ -266,6 +268,7 @@ jobs: - name: Install dependencies run: | uv pip uninstall transformers huggingface_hub && uv pip install --prerelease allow -U transformers@git+https://github.com/huggingface/transformers.git + uv pip uninstall tokenizers && uv pip install "tokenizers<=0.23.0" uv pip install -e ".[quality,training]" - name: Environment diff --git a/.github/workflows/push_tests.yml b/.github/workflows/push_tests.yml index 61bb9f0ef6..e8bf71f3a2 100644 --- a/.github/workflows/push_tests.yml +++ b/.github/workflows/push_tests.yml @@ -77,6 +77,7 @@ jobs: uv pip install -e ".[quality]" uv pip uninstall accelerate && uv pip install -U accelerate@git+https://github.com/huggingface/accelerate.git uv pip uninstall transformers huggingface_hub && uv pip install --prerelease allow -U transformers@git+https://github.com/huggingface/transformers.git + uv pip uninstall tokenizers && uv pip install "tokenizers<=0.23.0" - name: Environment run: | python utils/print_env.py @@ -129,6 +130,7 @@ jobs: uv pip install peft@git+https://github.com/huggingface/peft.git uv pip uninstall accelerate && uv pip install -U accelerate@git+https://github.com/huggingface/accelerate.git uv pip uninstall transformers huggingface_hub && uv pip install --prerelease allow -U transformers@git+https://github.com/huggingface/transformers.git + uv pip uninstall tokenizers && uv pip install "tokenizers<=0.23.0" - name: Environment run: | @@ -181,6 +183,7 @@ jobs: run: | uv pip install -e ".[quality,training]" uv pip uninstall transformers huggingface_hub && uv pip install --prerelease allow -U transformers@git+https://github.com/huggingface/transformers.git + uv pip uninstall tokenizers && uv pip install "tokenizers<=0.23.0" - name: Environment run: | python utils/print_env.py diff --git a/.github/workflows/push_tests_mps.yml b/.github/workflows/push_tests_mps.yml index 7f8ce9a4b9..e9f06840d3 100644 --- a/.github/workflows/push_tests_mps.yml +++ b/.github/workflows/push_tests_mps.yml @@ -45,6 +45,7 @@ jobs: ${CONDA_RUN} python -m uv pip install torch torchvision torchaudio ${CONDA_RUN} python -m uv pip install accelerate@git+https://github.com/huggingface/accelerate.git ${CONDA_RUN} python -m uv pip install transformers --upgrade + uv pip uninstall tokenizers && uv pip install "tokenizers<=0.23.0" - name: Environment shell: arch -arch arm64 bash {0} diff --git a/.github/workflows/pypi_publish.yaml b/.github/workflows/pypi_publish.yaml index 6439c5f7f1..77f0c50d1a 100644 --- a/.github/workflows/pypi_publish.yaml +++ b/.github/workflows/pypi_publish.yaml @@ -43,6 +43,7 @@ jobs: - name: Test installing diffusers and importing run: | pip install -U transformers + uv pip uninstall tokenizers && uv pip install "tokenizers<=0.23.0" python utils/print_env.py python -c "from diffusers import __version__; print(__version__)" python -c "from diffusers import DiffusionPipeline; pipe = DiffusionPipeline.from_pretrained('fusing/unet-ldm-dummy-update'); pipe()" diff --git a/.github/workflows/release_tests_fast.yml b/.github/workflows/release_tests_fast.yml index 7d097d1659..77c31b6f8b 100644 --- a/.github/workflows/release_tests_fast.yml +++ b/.github/workflows/release_tests_fast.yml @@ -35,6 +35,7 @@ jobs: run: | uv pip install -e ".[quality]" uv pip uninstall transformers huggingface_hub && uv pip install --prerelease allow -U transformers@git+https://github.com/huggingface/transformers.git + uv pip uninstall tokenizers && uv pip install "tokenizers<=0.23.0" - name: Environment run: | python utils/print_env.py @@ -77,6 +78,7 @@ jobs: uv pip install -e ".[quality]" uv pip uninstall accelerate && uv pip install -U accelerate@git+https://github.com/huggingface/accelerate.git uv pip uninstall transformers huggingface_hub && uv pip install --prerelease allow -U transformers@git+https://github.com/huggingface/transformers.git + uv pip uninstall tokenizers && uv pip install "tokenizers<=0.23.0" - name: Environment run: | python utils/print_env.py @@ -129,6 +131,7 @@ jobs: uv pip install peft@git+https://github.com/huggingface/peft.git uv pip uninstall accelerate && uv pip install -U accelerate@git+https://github.com/huggingface/accelerate.git uv pip uninstall transformers huggingface_hub && uv pip install --prerelease allow -U transformers@git+https://github.com/huggingface/transformers.git + uv pip uninstall tokenizers && uv pip install "tokenizers<=0.23.0" - name: Environment run: | @@ -180,6 +183,7 @@ jobs: uv pip install peft@git+https://github.com/huggingface/peft.git uv pip uninstall accelerate && uv pip install -U accelerate@git+https://github.com/huggingface/accelerate.git uv pip uninstall transformers huggingface_hub && uv pip install --prerelease allow -U transformers@git+https://github.com/huggingface/transformers.git + uv pip uninstall tokenizers && uv pip install "tokenizers<=0.23.0" - name: Environment run: | @@ -238,6 +242,7 @@ jobs: run: | uv pip install -e ".[quality,training]" uv pip uninstall transformers huggingface_hub && uv pip install --prerelease allow -U transformers@git+https://github.com/huggingface/transformers.git + uv pip uninstall tokenizers && uv pip install "tokenizers<=0.23.0" - name: Environment run: | python utils/print_env.py @@ -281,6 +286,7 @@ jobs: run: | uv pip install -e ".[quality,training]" uv pip uninstall transformers huggingface_hub && uv pip install --prerelease allow -U transformers@git+https://github.com/huggingface/transformers.git + uv pip uninstall tokenizers && uv pip install "tokenizers<=0.23.0" - name: Environment run: | python utils/print_env.py @@ -324,6 +330,7 @@ jobs: run: | uv pip install -e ".[quality,training]" uv pip uninstall transformers huggingface_hub && uv pip install --prerelease allow -U transformers@git+https://github.com/huggingface/transformers.git + uv pip uninstall tokenizers && uv pip install "tokenizers<=0.23.0" - name: Environment run: |