Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
80 changes: 51 additions & 29 deletions WORKSPACE
Original file line number Diff line number Diff line change
@@ -1,37 +1,42 @@
workspace(name = "org_tensorflow_text")

load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
load("//third_party/bazel:http.bzl", "custom_http_archive")

http_archive(
custom_http_archive(
name = "icu",
strip_prefix = "icu-release-64-2",
sha256 = "dfc62618aa4bd3ca14a3df548cd65fe393155edd213e49c39f3a30ccd618fc27",
exclude = [
# Build file does not contain all required targets.
"icu4c/source/common/BUILD.bazel",
],
files = {
"BUILD.bazel": "//third_party/icu:BUILD.bzl",
},
sha256 = "e424ba5282d95ad38b52639a08fb82164f0b0cbd7f17b53ae16bf14f8541855f",
strip_prefix = "icu-release-77-1",
urls = [
"https://storage.googleapis.com/mirror.tensorflow.org/github.com/unicode-org/icu/archive/release-64-2.zip",
"https://github.com/unicode-org/icu/archive/release-64-2.zip",
"https://storage.googleapis.com/mirror.tensorflow.org/github.com/unicode-org/icu/archive/release-77-1.zip",
"https://github.com/unicode-org/icu/archive/release-77-1.zip",
],
build_file = "//third_party/icu:BUILD.bzl",
patches = ["//third_party/icu:udata.patch"],
patch_args = ["-p1"],
)

http_archive(
name = "com_google_sentencepiece",
strip_prefix = "sentencepiece-0.1.96",
build_file = "//third_party/sentencepiece:BUILD",
patch_args = ["-p1"],
patches = ["//third_party/sentencepiece:sp.patch"],
sha256 = "8409b0126ebd62b256c685d5757150cf7fcb2b92a2f2b98efb3f38fc36719754",
strip_prefix = "sentencepiece-0.1.96",
urls = [
"https://github.com/google/sentencepiece/archive/refs/tags/v0.1.96.zip"
"https://github.com/google/sentencepiece/archive/refs/tags/v0.1.96.zip",
],
build_file = "//third_party/sentencepiece:BUILD",
patches = ["//third_party/sentencepiece:sp.patch"],
patch_args = ["-p1"],
)

http_archive(
name = "cppitertools",
urls = ["https://github.com/ryanhaining/cppitertools/archive/refs/tags/v2.0.zip"],
sha256 = "e56741b108d6baced98c4ccd83fd0d5a545937f2845978799c28d0312c0dee3d",
strip_prefix = "cppitertools-2.0",
urls = ["https://github.com/ryanhaining/cppitertools/archive/refs/tags/v2.0.zip"],
)

http_archive(
Expand All @@ -56,10 +61,11 @@ http_archive(

http_archive(
name = "org_tensorflow",
strip_prefix = "tensorflow-40998f44c0c500ce0f6e3b1658dfbc54f838a82a",
sha256 = "5a5bc4599964c71277dcac0d687435291e5810d2ac2f6283cc96736febf73aaf",
patch_args = ["-p1"],
patches = ["//third_party/tensorflow:tensorflow.core.BUILD.patch"],
strip_prefix = "tensorflow-72fbba3d20f4616d7312b5e2b7f79daf6e82f2fa",
urls = [
"https://github.com/tensorflow/tensorflow/archive/40998f44c0c500ce0f6e3b1658dfbc54f838a82a.zip"
"https://github.com/tensorflow/tensorflow/archive/72fbba3d20f4616d7312b5e2b7f79daf6e82f2fa.zip",
],
)

Expand All @@ -74,13 +80,13 @@ http_archive(

http_archive(
name = "pybind11",
build_file = "//third_party/pybind11:BUILD.bzl",
sha256 = "efc901aa0aab439a3fea6efeaf930b5a349fb06394bf845c64ce15a9cf8f0240",
strip_prefix = "pybind11-2.13.4",
urls = [
"https://storage.googleapis.com/mirror.tensorflow.org/github.com/pybind/pybind11/archive/v2.13.4.tar.gz",
"https://github.com/pybind/pybind11/archive/v2.13.4.tar.gz",
],
sha256 = "efc901aa0aab439a3fea6efeaf930b5a349fb06394bf845c64ce15a9cf8f0240",
strip_prefix = "pybind11-2.13.4",
build_file = "//third_party/pybind11:BUILD.bzl",
)

http_archive(
Expand All @@ -95,31 +101,37 @@ load("@org_tensorflow//third_party/py:python_init_rules.bzl", "python_init_rules

python_init_rules()

load("//tensorflow_text:tftext.bzl", "py_deps_profile")
load("//third_party/bazel:py_deps_profile.bzl", "py_deps_profile")

py_deps_profile(
name = "release_or_nightly",
requirements_in = "//oss_scripts/pip_package:requirements.in",
pip_repo_name = "pypi",
deps_map = {
"tensorflow": ["tf-nightly", "tf_header_lib", "libtensorflow_framework"],
"tf-keras": ["tf-keras-nightly"]
"tensorflow": [
"tf-nightly",
"tf_headers",
"tf_header_lib",
"libtensorflow_framework",
],
"tf-keras": ["tf-keras-nightly"],
},
pip_repo_name = "pypi",
requirements_in = "//oss_scripts/pip_package:requirements.in",
switch = {
"IS_NIGHTLY": "nightly"
}
"IS_NIGHTLY": "nightly",
},
)

load("@org_tensorflow//third_party/py:python_init_repositories.bzl", "python_init_repositories")

python_init_repositories(
default_python_version = "system",
requirements = {
"3.9": "//oss_scripts/pip_package:requirements_lock_3_9.txt",
"3.10": "//oss_scripts/pip_package:requirements_lock_3_10.txt",
"3.11": "//oss_scripts/pip_package:requirements_lock_3_11.txt",
"3.12": "//oss_scripts/pip_package:requirements_lock_3_12.txt",
"3.13": "//oss_scripts/pip_package:requirements_lock_3_13.txt",
},
default_python_version = "system",
)

load("@org_tensorflow//third_party/py:python_init_toolchains.bzl", "python_init_toolchains")
Expand All @@ -136,18 +148,28 @@ install_deps()

# Initialize TensorFlow dependencies.
load("@org_tensorflow//tensorflow:workspace3.bzl", "tf_workspace3")

tf_workspace3()

load("@org_tensorflow//tensorflow:workspace2.bzl", "tf_workspace2")

tf_workspace2()

load("@org_tensorflow//tensorflow:workspace1.bzl", "tf_workspace1")

tf_workspace1()

load("@org_tensorflow//tensorflow:workspace0.bzl", "tf_workspace0")

tf_workspace0()

# Set up Android.
load("@org_tensorflow//third_party/android:android_configure.bzl", "android_configure")
android_configure(name="local_config_android")

android_configure(name = "local_config_android")

load("@local_config_android//:android.bzl", "android_workspace")

android_workspace()

load(
Expand Down
39 changes: 21 additions & 18 deletions oss_scripts/configure.sh
Original file line number Diff line number Diff line change
Expand Up @@ -32,16 +32,22 @@ function is_macos() {
# Remove .bazelrc if it already exist
[ -e .bazelrc ] && rm .bazelrc

if [[ $(pip show tensorflow) == *tensorflow* ]] ||
[[ $(pip show tensorflow-macos) == *tensorflow-macos* ]] ||
[[ $(pip show tf-nightly) == *tf-nightly* ]]; then
if (which python3) | grep -q "python3"; then
installed_python="python3"
elif (which python) | grep -q "python"; then
installed_python="python"
fi

if [[ $($installed_python -m pip show tensorflow) == *tensorflow* ]] ||
[[ $($installed_python -m pip show tensorflow-macos) == *tensorflow-macos* ]] ||
[[ $($installed_python -m pip show tf-nightly) == *tf-nightly* ]]; then
echo 'Using installed tensorflow.'
else
echo 'Installing tensorflow.'
if [[ "$IS_NIGHTLY" == "nightly" ]]; then
pip install tf-nightly
$installed_python -m pip install --no-deps tf-nightly
else
pip install tensorflow==2.18.0
$installed_python -m pip install --no-deps tensorflow==2.20
fi
fi

Expand All @@ -54,12 +60,7 @@ curl https://raw.githubusercontent.com/tensorflow/tensorflow/master/.bazelrc -o
sed -i -e 's/build --noincompatible_remove_legacy_whole_archive//' .bazelrc

write_to_bazelrc "build:manylinux2014 --config=release_cpu_linux"

if (which python3) | grep -q "python3"; then
installed_python="python3"
elif (which python) | grep -q "python"; then
installed_python="python"
fi
write_to_bazelrc "test --test_env=TF_USE_LEGACY_KERAS=1"

if [ -z "$HERMETIC_PYTHON_VERSION" ]; then
if [ -n "$PY_VERSION" ]; then
Expand All @@ -70,13 +71,15 @@ if [ -z "$HERMETIC_PYTHON_VERSION" ]; then
fi
export HERMETIC_PYTHON_VERSION

echo "TF_VERSION=$TF_VERSION"
REQUIREMENTS_EXTRA_FLAGS="--upgrade"
if [[ "$TF_VERSION" == *"rc"* ]]; then
REQUIREMENTS_EXTRA_FLAGS="$REQUIREMENTS_EXTRA_FLAGS --pre"
fi

bazel run //oss_scripts/pip_package:requirements.update -- $REQUIREMENTS_EXTRA_FLAGS
# TODO - Re-enable requirements update when presubmits changed to python 3.11.
# # Only auto-upgrade requirements for nightly.
# if [[ "$IS_NIGHTLY" == "nightly" ]]; then
# REQUIREMENTS_EXTRA_FLAGS="--upgrade"
# if [[ "$TF_VERSION" == *"rc"* ]]; then
# REQUIREMENTS_EXTRA_FLAGS="$REQUIREMENTS_EXTRA_FLAGS --pre"
# fi
# bazel run //oss_scripts/pip_package:requirements.update -- $REQUIREMENTS_EXTRA_FLAGS
# fi

TF_ABIFLAG=$(bazel run //oss_scripts/pip_package:tensorflow_build_info -- abi)
SHARED_LIBRARY_NAME="libtensorflow_framework.so.2"
Expand Down
7 changes: 5 additions & 2 deletions oss_scripts/pip_package/BUILD
Original file line number Diff line number Diff line change
Expand Up @@ -27,14 +27,17 @@ py_binary(
],
)

sh_binary(
py_binary(
name = "build_pip_package",
srcs = ["build_pip_package.sh"],
srcs = ["build_pip_package.py"],
data = [
"LICENSE",
"MANIFEST.in",
"setup.nightly.py",
"setup.py",
"//tensorflow_text",
],
deps = [
"@pypi_setuptools//:pkg",
],
)
143 changes: 143 additions & 0 deletions oss_scripts/pip_package/build_pip_package.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,143 @@
# coding=utf-8
# Copyright 2025 TF.Text Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Builds the tensorflow_text pip package."""

import os
import pathlib
import platform
import shutil
import subprocess
import sys
import tempfile


def is_windows():
# Matches the bash logic for msys/mingw/cygwin/windows
return (
platform.system()
.lower()
.startswith(("windows", "msys", "mingw", "cygwin"))
)


def is_macos():
return platform.system().lower() == "darwin"


def is_nightly():
return os.environ.get("IS_NIGHTLY") == "nightly"


def get_platform_name():
if not is_macos():
return []

# Check architecture for macOS
arch = platform.machine().lower()
if arch == "arm64":
return ["--plat-name", "macosx_11_0_arm64"]
else:
return ["--plat-name", "macosx-10.9-x86_64"]


def main():
# Handle Bazel workspace directory
workspace_dir = os.environ.get("BUILD_WORKSPACE_DIRECTORY")
if workspace_dir:
os.chdir(workspace_dir)

# Output directory logic
output_dir_arg = (
sys.argv[1] if len(sys.argv) > 1 else "/tmp/tensorflow_text_pkg"
)
output_path = pathlib.Path(output_dir_arg).resolve()
output_path.mkdir(parents=True, exist_ok=True)

print(f"=== Destination directory: {output_path}")
print(f"=== Current directory: {pathlib.Path.cwd()}")

# Verify bazel-bin exists
if not pathlib.Path("bazel-bin/tensorflow_text").exists():
print(
"ERROR: Could not find bazel-bin. Did you run from the root of the"
" build tree?",
file=sys.stderr,
)
sys.exit(1)

# Determine runfiles path
if is_windows():
runfiles_base = pathlib.Path(
"bazel-bin/oss_scripts/pip_package/build_pip_package.exe.runfiles"
)
else:
runfiles_base = pathlib.Path(
"bazel-bin/oss_scripts/pip_package/build_pip_package.runfiles"
)

source_root = runfiles_base / "org_tensorflow_text"

# Create temp directory
with tempfile.TemporaryDirectory() as temp_dir:
temp_path = pathlib.Path(temp_dir)
print(f"=== Using tmpdir {temp_path}")

# Copy source files (Equivalent to cp -LR)
shutil.copytree(
source_root / "tensorflow_text",
temp_path / "tensorflow_text",
dirs_exist_ok=True,
)

# Setup.py selection
setup_file = "setup.nightly.py" if is_nightly() else "setup.py"
shutil.copy2(
source_root / "oss_scripts/pip_package" / setup_file,
temp_path / setup_file,
)

# Manifest and License
shutil.copy2(
source_root / "oss_scripts/pip_package/MANIFEST.in",
temp_path / "MANIFEST.in",
)
shutil.copy2(
source_root / "oss_scripts/pip_package/LICENSE", temp_path / "LICENSE"
)

# Prepare Python execution
python_bin = sys.executable # Uses the currently running python

# Build pip package
build_cmd = [
python_bin,
setup_file,
"bdist_wheel",
"--universal",
] + get_platform_name()

print(f"=== Running: {' '.join(build_cmd)}")
subprocess.run(build_cmd, cwd=temp_path, check=True)

# Copy wheels to output directory
dist_path = temp_path / "dist"
for wheel in dist_path.glob("*.whl"):
shutil.copy2(wheel, output_path)
print(f"=== Built: {output_path / wheel.name}")


if __name__ == "__main__":
main()
Loading
Loading