From bfbe572fde4b0544ce281daf333fef31f8d854ac Mon Sep 17 00:00:00 2001 From: davidhabboosh1 Date: Sun, 10 Aug 2025 11:18:29 -0400 Subject: [PATCH 01/28] add pyproject.toml --- oss_scripts/pip_package/pyproject.toml | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 oss_scripts/pip_package/pyproject.toml diff --git a/oss_scripts/pip_package/pyproject.toml b/oss_scripts/pip_package/pyproject.toml new file mode 100644 index 000000000..eb50f820b --- /dev/null +++ b/oss_scripts/pip_package/pyproject.toml @@ -0,0 +1,3 @@ +[build-system] +requires = ["setuptools>=64", "wheel"] +build-backend = "setuptools.build_meta" \ No newline at end of file From 7f8e864fb2debc25fb71376d7855e755ed9af025 Mon Sep 17 00:00:00 2001 From: davidhabboosh1 Date: Sun, 10 Aug 2025 11:21:34 -0400 Subject: [PATCH 02/28] replace setup.py with pyproject.toml --- oss_scripts/pip_package/pyproject.toml | 46 ++++++++++++++++++- .../{setup.py => setup.py.example} | 0 2 files changed, 44 insertions(+), 2 deletions(-) rename oss_scripts/pip_package/{setup.py => setup.py.example} (100%) diff --git a/oss_scripts/pip_package/pyproject.toml b/oss_scripts/pip_package/pyproject.toml index eb50f820b..e12812d05 100644 --- a/oss_scripts/pip_package/pyproject.toml +++ b/oss_scripts/pip_package/pyproject.toml @@ -1,3 +1,45 @@ [build-system] -requires = ["setuptools>=64", "wheel"] -build-backend = "setuptools.build_meta" \ No newline at end of file +requires = ["setuptools>=65.5.1", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "tensorflow-text" +version = "2.18.0" +description = "TF.Text is a TensorFlow library of text related ops, modules, and subgraphs." +readme = "README.md" # Add your README path here if you want long description included +authors = [ + { name = "Google Inc.", email = "packages@tensorflow.org" } +] +license = { text = "Apache 2.0" } +keywords = ["tensorflow", "text", "machine learning"] +classifiers = [ + "Development Status :: 5 - Production/Stable", + "Intended Audience :: Developers", + "Intended Audience :: Education", + "Intended Audience :: Science/Research", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3 :: Only", + "Topic :: Scientific/Engineering", + "Topic :: Scientific/Engineering :: Artificial Intelligence", + "Topic :: Software Development", + "Topic :: Software Development :: Libraries", + "Topic :: Software Development :: Libraries :: Python Modules", +] + +dependencies = [ + "tensorflow>=2.18.0,<2.19", +] + +[project.optional-dependencies] +tensorflow_cpu = [ + "tensorflow-cpu>=2.18.0,<2.19" +] +tests = [ + "absl-py", + "pytest", + "tensorflow-datasets>=3.2.0", +] \ No newline at end of file diff --git a/oss_scripts/pip_package/setup.py b/oss_scripts/pip_package/setup.py.example similarity index 100% rename from oss_scripts/pip_package/setup.py rename to oss_scripts/pip_package/setup.py.example From ec6db1d5d61fdd1df661f1128458b92c95987e8e Mon Sep 17 00:00:00 2001 From: davidhabboosh1 Date: Sun, 10 Aug 2025 11:39:52 -0400 Subject: [PATCH 03/28] update license line --- oss_scripts/pip_package/pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/oss_scripts/pip_package/pyproject.toml b/oss_scripts/pip_package/pyproject.toml index e12812d05..0a8ffb5fc 100644 --- a/oss_scripts/pip_package/pyproject.toml +++ b/oss_scripts/pip_package/pyproject.toml @@ -10,7 +10,7 @@ readme = "README.md" # Add your README path here if you want long description i authors = [ { name = "Google Inc.", email = "packages@tensorflow.org" } ] -license = { text = "Apache 2.0" } +license = "Apache 2.0" keywords = ["tensorflow", "text", "machine learning"] classifiers = [ "Development Status :: 5 - Production/Stable", From d2db4dc510ccf01a22ebd7dfc540132766f7419c Mon Sep 17 00:00:00 2001 From: davidhabboosh1 Date: Sun, 10 Aug 2025 11:42:07 -0400 Subject: [PATCH 04/28] typo --- oss_scripts/pip_package/pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/oss_scripts/pip_package/pyproject.toml b/oss_scripts/pip_package/pyproject.toml index 0a8ffb5fc..3c6aca8fe 100644 --- a/oss_scripts/pip_package/pyproject.toml +++ b/oss_scripts/pip_package/pyproject.toml @@ -10,7 +10,7 @@ readme = "README.md" # Add your README path here if you want long description i authors = [ { name = "Google Inc.", email = "packages@tensorflow.org" } ] -license = "Apache 2.0" +license = "Apache-2.0" keywords = ["tensorflow", "text", "machine learning"] classifiers = [ "Development Status :: 5 - Production/Stable", From 4db731b482c2193ece174838c4ceadc7dd0b6e07 Mon Sep 17 00:00:00 2001 From: davidhabboosh1 Date: Sun, 10 Aug 2025 11:53:59 -0400 Subject: [PATCH 05/28] remove sh file --- oss_scripts/pip_package/build_pip_package.sh | 109 ------------------- 1 file changed, 109 deletions(-) delete mode 100755 oss_scripts/pip_package/build_pip_package.sh diff --git a/oss_scripts/pip_package/build_pip_package.sh b/oss_scripts/pip_package/build_pip_package.sh deleted file mode 100755 index 8e411ef17..000000000 --- a/oss_scripts/pip_package/build_pip_package.sh +++ /dev/null @@ -1,109 +0,0 @@ -#!/usr/bin/env bash -# Tool to build the TensorFlow Text pip package. -# -# Usage: -# bazel build oss_scripts/pip_package:build_pip_package -# bazel-bin/oss_scripts/build_pip_package -# -# Arguments: -# output_dir: An output directory. Defaults to `/tmp/tensorflow_text_pkg`. - -set -e # fail and exit on any command erroring - -die() { - echo >&2 "$@" - exit 1 -} - -if [ -n "$BUILD_WORKSPACE_DIRECTORY" ]; then - cd "$BUILD_WORKSPACE_DIRECTORY" -fi - -osname="$(uname -s | tr 'A-Z' 'a-z')" -echo $osname - -function is_windows() { - # On windows, the shell script is actually running in msys - [[ "${osname}" =~ msys_nt*|mingw*|cygwin*|uwin* ]] -} - -function is_macos() { - [[ "${osname}" == "darwin" ]] -} - -function is_nightly() { - [[ "$IS_NIGHTLY" == "nightly" ]] -} - -function abspath() { - cd "$(dirname $1)" - echo "$PWD/$(basename $1)" - cd "$OLDPWD" -} - -plat_name="" -if is_macos; then - if [[ x"$(arch)" == x"arm64" ]]; then - plat_name="--plat-name macosx_11_0_arm64" - else - plat_name="--plat-name macosx-10.9-x86_64" - fi -fi - -main() { - local output_dir="$1" - - if [[ -z "${output_dir}" ]]; then - output_dir="/tmp/tensorflow_text_pkg" - fi - mkdir -p ${output_dir} - output_dir=$(abspath "${output_dir}") - echo "=== Destination directory: ${output_dir}" - echo "=== Current directory: ${PWD}" - - if [[ ! -d "bazel-bin/tensorflow_text" ]]; then - die "Could not find bazel-bin. Did you run from the root of the build tree?" - fi - - local temp_dir="$(mktemp -d)" - trap "rm -rf ${temp_dir}" EXIT - echo "=== Using tmpdir ${temp_dir}" - - if is_windows; then - runfiles="bazel-bin/oss_scripts/pip_package/build_pip_package.exe.runfiles" - else - runfiles="bazel-bin/oss_scripts/pip_package/build_pip_package.runfiles" - fi - cp -LR \ - "${runfiles}/org_tensorflow_text/tensorflow_text" \ - "${temp_dir}" - if is_nightly; then - cp "${runfiles}/org_tensorflow_text/oss_scripts/pip_package/setup.nightly.py" \ - "${temp_dir}" - else - cp "${runfiles}/org_tensorflow_text/oss_scripts/pip_package/setup.py" \ - "${temp_dir}" - fi - cp "${runfiles}/org_tensorflow_text/oss_scripts/pip_package/MANIFEST.in" \ - "${temp_dir}" - cp "${runfiles}/org_tensorflow_text/oss_scripts/pip_package/LICENSE" \ - "${temp_dir}" - - pushd "${temp_dir}" > /dev/null - - if (which python3) | grep -q "python3"; then - installed_python="python3" - elif (which python) | grep -q "python"; then - installed_python="python" - fi - - # Build pip package - if is_nightly; then - $installed_python setup.nightly.py bdist_wheel --universal $plat_name - else - $installed_python setup.py bdist_wheel --universal $plat_name - fi - cp dist/*.whl "${output_dir}" -} - -main "$@" From 1c84b574949706974fe8c6ff3050718103e84298 Mon Sep 17 00:00:00 2001 From: davidhabboosh1 Date: Sun, 10 Aug 2025 12:13:46 -0400 Subject: [PATCH 06/28] move BUILD --- .../pip_package/{BUILD => BUILD.example} | 0 .../pip_package/build_pip_package.sh.example | 109 ++++++++++++++++++ 2 files changed, 109 insertions(+) rename oss_scripts/pip_package/{BUILD => BUILD.example} (100%) create mode 100644 oss_scripts/pip_package/build_pip_package.sh.example diff --git a/oss_scripts/pip_package/BUILD b/oss_scripts/pip_package/BUILD.example similarity index 100% rename from oss_scripts/pip_package/BUILD rename to oss_scripts/pip_package/BUILD.example diff --git a/oss_scripts/pip_package/build_pip_package.sh.example b/oss_scripts/pip_package/build_pip_package.sh.example new file mode 100644 index 000000000..8e411ef17 --- /dev/null +++ b/oss_scripts/pip_package/build_pip_package.sh.example @@ -0,0 +1,109 @@ +#!/usr/bin/env bash +# Tool to build the TensorFlow Text pip package. +# +# Usage: +# bazel build oss_scripts/pip_package:build_pip_package +# bazel-bin/oss_scripts/build_pip_package +# +# Arguments: +# output_dir: An output directory. Defaults to `/tmp/tensorflow_text_pkg`. + +set -e # fail and exit on any command erroring + +die() { + echo >&2 "$@" + exit 1 +} + +if [ -n "$BUILD_WORKSPACE_DIRECTORY" ]; then + cd "$BUILD_WORKSPACE_DIRECTORY" +fi + +osname="$(uname -s | tr 'A-Z' 'a-z')" +echo $osname + +function is_windows() { + # On windows, the shell script is actually running in msys + [[ "${osname}" =~ msys_nt*|mingw*|cygwin*|uwin* ]] +} + +function is_macos() { + [[ "${osname}" == "darwin" ]] +} + +function is_nightly() { + [[ "$IS_NIGHTLY" == "nightly" ]] +} + +function abspath() { + cd "$(dirname $1)" + echo "$PWD/$(basename $1)" + cd "$OLDPWD" +} + +plat_name="" +if is_macos; then + if [[ x"$(arch)" == x"arm64" ]]; then + plat_name="--plat-name macosx_11_0_arm64" + else + plat_name="--plat-name macosx-10.9-x86_64" + fi +fi + +main() { + local output_dir="$1" + + if [[ -z "${output_dir}" ]]; then + output_dir="/tmp/tensorflow_text_pkg" + fi + mkdir -p ${output_dir} + output_dir=$(abspath "${output_dir}") + echo "=== Destination directory: ${output_dir}" + echo "=== Current directory: ${PWD}" + + if [[ ! -d "bazel-bin/tensorflow_text" ]]; then + die "Could not find bazel-bin. Did you run from the root of the build tree?" + fi + + local temp_dir="$(mktemp -d)" + trap "rm -rf ${temp_dir}" EXIT + echo "=== Using tmpdir ${temp_dir}" + + if is_windows; then + runfiles="bazel-bin/oss_scripts/pip_package/build_pip_package.exe.runfiles" + else + runfiles="bazel-bin/oss_scripts/pip_package/build_pip_package.runfiles" + fi + cp -LR \ + "${runfiles}/org_tensorflow_text/tensorflow_text" \ + "${temp_dir}" + if is_nightly; then + cp "${runfiles}/org_tensorflow_text/oss_scripts/pip_package/setup.nightly.py" \ + "${temp_dir}" + else + cp "${runfiles}/org_tensorflow_text/oss_scripts/pip_package/setup.py" \ + "${temp_dir}" + fi + cp "${runfiles}/org_tensorflow_text/oss_scripts/pip_package/MANIFEST.in" \ + "${temp_dir}" + cp "${runfiles}/org_tensorflow_text/oss_scripts/pip_package/LICENSE" \ + "${temp_dir}" + + pushd "${temp_dir}" > /dev/null + + if (which python3) | grep -q "python3"; then + installed_python="python3" + elif (which python) | grep -q "python"; then + installed_python="python" + fi + + # Build pip package + if is_nightly; then + $installed_python setup.nightly.py bdist_wheel --universal $plat_name + else + $installed_python setup.py bdist_wheel --universal $plat_name + fi + cp dist/*.whl "${output_dir}" +} + +main "$@" From e676a5a1a9a18c645b2c8c7cce6e831cd2b54adc Mon Sep 17 00:00:00 2001 From: davidhabboosh1 Date: Sun, 10 Aug 2025 12:15:18 -0400 Subject: [PATCH 07/28] put back build_pip_package.sh --- .../{build_pip_package.sh.example => build_pip_package.sh} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename oss_scripts/pip_package/{build_pip_package.sh.example => build_pip_package.sh} (100%) diff --git a/oss_scripts/pip_package/build_pip_package.sh.example b/oss_scripts/pip_package/build_pip_package.sh similarity index 100% rename from oss_scripts/pip_package/build_pip_package.sh.example rename to oss_scripts/pip_package/build_pip_package.sh From 0673d33037acfa6e80fe31c4d7228427ed4976e3 Mon Sep 17 00:00:00 2001 From: davidhabboosh1 Date: Sun, 10 Aug 2025 12:17:54 -0400 Subject: [PATCH 08/28] add back setup.py at root --- oss_scripts/pip_package/setup.py.example => setup.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename oss_scripts/pip_package/setup.py.example => setup.py (100%) diff --git a/oss_scripts/pip_package/setup.py.example b/setup.py similarity index 100% rename from oss_scripts/pip_package/setup.py.example rename to setup.py From 4241e12215c8fe8a8ec4bd153fcc27b7807a4f90 Mon Sep 17 00:00:00 2001 From: davidhabboosh1 Date: Sun, 10 Aug 2025 12:23:03 -0400 Subject: [PATCH 09/28] move build stuff to root --- ...ild_pip_package.sh => build_pip_package.sh | 0 .../pip_package/{BUILD.example => BUILD} | 0 oss_scripts/pip_package/pyproject.toml | 45 ------------------- .../setup.nightly.py => setup.nightly.py | 0 4 files changed, 45 deletions(-) rename oss_scripts/pip_package/build_pip_package.sh => build_pip_package.sh (100%) rename oss_scripts/pip_package/{BUILD.example => BUILD} (100%) delete mode 100644 oss_scripts/pip_package/pyproject.toml rename oss_scripts/pip_package/setup.nightly.py => setup.nightly.py (100%) diff --git a/oss_scripts/pip_package/build_pip_package.sh b/build_pip_package.sh similarity index 100% rename from oss_scripts/pip_package/build_pip_package.sh rename to build_pip_package.sh diff --git a/oss_scripts/pip_package/BUILD.example b/oss_scripts/pip_package/BUILD similarity index 100% rename from oss_scripts/pip_package/BUILD.example rename to oss_scripts/pip_package/BUILD diff --git a/oss_scripts/pip_package/pyproject.toml b/oss_scripts/pip_package/pyproject.toml deleted file mode 100644 index 3c6aca8fe..000000000 --- a/oss_scripts/pip_package/pyproject.toml +++ /dev/null @@ -1,45 +0,0 @@ -[build-system] -requires = ["setuptools>=65.5.1", "wheel"] -build-backend = "setuptools.build_meta" - -[project] -name = "tensorflow-text" -version = "2.18.0" -description = "TF.Text is a TensorFlow library of text related ops, modules, and subgraphs." -readme = "README.md" # Add your README path here if you want long description included -authors = [ - { name = "Google Inc.", email = "packages@tensorflow.org" } -] -license = "Apache-2.0" -keywords = ["tensorflow", "text", "machine learning"] -classifiers = [ - "Development Status :: 5 - Production/Stable", - "Intended Audience :: Developers", - "Intended Audience :: Education", - "Intended Audience :: Science/Research", - "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10", - "Programming Language :: Python :: 3.11", - "Programming Language :: Python :: 3.12", - "Programming Language :: Python :: 3 :: Only", - "Topic :: Scientific/Engineering", - "Topic :: Scientific/Engineering :: Artificial Intelligence", - "Topic :: Software Development", - "Topic :: Software Development :: Libraries", - "Topic :: Software Development :: Libraries :: Python Modules", -] - -dependencies = [ - "tensorflow>=2.18.0,<2.19", -] - -[project.optional-dependencies] -tensorflow_cpu = [ - "tensorflow-cpu>=2.18.0,<2.19" -] -tests = [ - "absl-py", - "pytest", - "tensorflow-datasets>=3.2.0", -] \ No newline at end of file diff --git a/oss_scripts/pip_package/setup.nightly.py b/setup.nightly.py similarity index 100% rename from oss_scripts/pip_package/setup.nightly.py rename to setup.nightly.py From d0894120c1dfda79ed5b5e8110515eb79ed212e4 Mon Sep 17 00:00:00 2001 From: davidhabboosh1 Date: Sun, 10 Aug 2025 12:32:40 -0400 Subject: [PATCH 10/28] put back build pip package --- .../pip_package/build_pip_package.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename build_pip_package.sh => oss_scripts/pip_package/build_pip_package.sh (100%) diff --git a/build_pip_package.sh b/oss_scripts/pip_package/build_pip_package.sh similarity index 100% rename from build_pip_package.sh rename to oss_scripts/pip_package/build_pip_package.sh From 6364e464bf6d7720535da415906fe13c7dfd16c2 Mon Sep 17 00:00:00 2001 From: davidhabboosh1 Date: Sun, 10 Aug 2025 21:45:59 -0400 Subject: [PATCH 11/28] move packages back and rename build --- oss_scripts/pip_package/{BUILD => BUILD.bazel} | 0 setup.nightly.py => oss_scripts/pip_package/setup.nightly.py | 0 setup.py => oss_scripts/pip_package/setup.py | 0 3 files changed, 0 insertions(+), 0 deletions(-) rename oss_scripts/pip_package/{BUILD => BUILD.bazel} (100%) rename setup.nightly.py => oss_scripts/pip_package/setup.nightly.py (100%) rename setup.py => oss_scripts/pip_package/setup.py (100%) diff --git a/oss_scripts/pip_package/BUILD b/oss_scripts/pip_package/BUILD.bazel similarity index 100% rename from oss_scripts/pip_package/BUILD rename to oss_scripts/pip_package/BUILD.bazel diff --git a/setup.nightly.py b/oss_scripts/pip_package/setup.nightly.py similarity index 100% rename from setup.nightly.py rename to oss_scripts/pip_package/setup.nightly.py diff --git a/setup.py b/oss_scripts/pip_package/setup.py similarity index 100% rename from setup.py rename to oss_scripts/pip_package/setup.py From 2e109a591d35b0e64120a1b0e0bab1a981b729c8 Mon Sep 17 00:00:00 2001 From: davidhabboosh1 Date: Sun, 10 Aug 2025 21:50:15 -0400 Subject: [PATCH 12/28] use bzl instead of bazel --- oss_scripts/pip_package/{BUILD.bazel => BUILD.bzl} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename oss_scripts/pip_package/{BUILD.bazel => BUILD.bzl} (100%) diff --git a/oss_scripts/pip_package/BUILD.bazel b/oss_scripts/pip_package/BUILD.bzl similarity index 100% rename from oss_scripts/pip_package/BUILD.bazel rename to oss_scripts/pip_package/BUILD.bzl From 60d4bd8b68f5d430b37212a9c1a788d3580a2e9e Mon Sep 17 00:00:00 2001 From: davidhabboosh1 Date: Sun, 10 Aug 2025 22:55:50 -0400 Subject: [PATCH 13/28] put BUILD back --- oss_scripts/pip_package/{BUILD.bzl => BUILD} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename oss_scripts/pip_package/{BUILD.bzl => BUILD} (100%) diff --git a/oss_scripts/pip_package/BUILD.bzl b/oss_scripts/pip_package/BUILD similarity index 100% rename from oss_scripts/pip_package/BUILD.bzl rename to oss_scripts/pip_package/BUILD From db2c47551d084476759d2a3297c806422055b311 Mon Sep 17 00:00:00 2001 From: davidhabboosh1 Date: Sun, 10 Aug 2025 22:59:19 -0400 Subject: [PATCH 14/28] add custom build folder --- oss_scripts/pip_package/setup.nightly.py | 9 ++++++++- oss_scripts/pip_package/setup.py | 9 ++++++++- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/oss_scripts/pip_package/setup.nightly.py b/oss_scripts/pip_package/setup.nightly.py index 1f4949329..426065a52 100644 --- a/oss_scripts/pip_package/setup.nightly.py +++ b/oss_scripts/pip_package/setup.nightly.py @@ -44,6 +44,7 @@ from setuptools import setup from setuptools.command.install import install from setuptools.dist import Distribution +from setuptools.command.build import build project_name = 'tensorflow-text-nightly' project_version = 'REPLACE_ME' @@ -70,6 +71,12 @@ def finalize_options(self): self.install_lib = os.path.join(self.install_lib, self.extra_dirs) +class BuildToBuildSetuptools(build): + def initialize_options(self): + super().initialize_options() + self.build_base = 'build_setuptools' + + DOCLINES = __doc__.split('\n') setup( @@ -84,7 +91,7 @@ def finalize_options(self): packages=find_packages(), include_package_data=True, zip_safe=False, - cmdclass={'install': InstallPlatlib}, + cmdclass={'install': InstallPlatlib, 'build': BuildToBuildSetuptools}, distclass=BinaryDistribution, install_requires=[ ], diff --git a/oss_scripts/pip_package/setup.py b/oss_scripts/pip_package/setup.py index 4b7422c37..9c607a149 100644 --- a/oss_scripts/pip_package/setup.py +++ b/oss_scripts/pip_package/setup.py @@ -44,6 +44,7 @@ from setuptools import setup from setuptools.command.install import install from setuptools.dist import Distribution +from setuptools.command.build import build project_name = 'tensorflow-text' project_version = '2.18.0' @@ -70,6 +71,12 @@ def finalize_options(self): self.install_lib = os.path.join(self.install_lib, self.extra_dirs) +class BuildToBuildSetuptools(build): + def initialize_options(self): + super().initialize_options() + self.build_base = 'build_setuptools' + + DOCLINES = __doc__.split('\n') setup( @@ -84,7 +91,7 @@ def finalize_options(self): packages=find_packages(), include_package_data=True, zip_safe=False, - cmdclass={'install': InstallPlatlib}, + cmdclass={'install': InstallPlatlib, 'build': BuildToBuildSetuptools}, distclass=BinaryDistribution, install_requires=[ ( From a14fcf91eb9cbb815a60506b5eda2c7c7e36e8cb Mon Sep 17 00:00:00 2001 From: davidhabboosh1 Date: Sun, 10 Aug 2025 23:07:30 -0400 Subject: [PATCH 15/28] made executable again? --- oss_scripts/pip_package/build_pip_package.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 oss_scripts/pip_package/build_pip_package.sh diff --git a/oss_scripts/pip_package/build_pip_package.sh b/oss_scripts/pip_package/build_pip_package.sh old mode 100644 new mode 100755 From 08d45fe83890ac572ac45633faa5a612b7d489e2 Mon Sep 17 00:00:00 2001 From: davidhabboosh1 Date: Tue, 12 Aug 2025 22:16:45 -0400 Subject: [PATCH 16/28] use pip-compile rather than bazel --- oss_scripts/configure.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/oss_scripts/configure.sh b/oss_scripts/configure.sh index a7f4e96a6..9a134691f 100755 --- a/oss_scripts/configure.sh +++ b/oss_scripts/configure.sh @@ -76,7 +76,8 @@ if [[ "$TF_VERSION" == *"rc"* ]]; then REQUIREMENTS_EXTRA_FLAGS="$REQUIREMENTS_EXTRA_FLAGS --pre" fi -bazel run //oss_scripts/pip_package:requirements.update -- $REQUIREMENTS_EXTRA_FLAGS +pip-compile pip_package/requirements.in -- $REQUIREMENTS_EXTRA_FLAGS +# bazel run //oss_scripts/pip_package:requirements.update -- $REQUIREMENTS_EXTRA_FLAGS TF_ABIFLAG=$(bazel run //oss_scripts/pip_package:tensorflow_build_info -- abi) SHARED_LIBRARY_NAME="libtensorflow_framework.so.2" From 6403229c0bd78ef4a022ab9a23822838bd636e91 Mon Sep 17 00:00:00 2001 From: davidhabboosh1 Date: Tue, 12 Aug 2025 22:23:49 -0400 Subject: [PATCH 17/28] fix path --- oss_scripts/configure.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/oss_scripts/configure.sh b/oss_scripts/configure.sh index 9a134691f..662e20359 100755 --- a/oss_scripts/configure.sh +++ b/oss_scripts/configure.sh @@ -76,7 +76,7 @@ if [[ "$TF_VERSION" == *"rc"* ]]; then REQUIREMENTS_EXTRA_FLAGS="$REQUIREMENTS_EXTRA_FLAGS --pre" fi -pip-compile pip_package/requirements.in -- $REQUIREMENTS_EXTRA_FLAGS +pip-compile oss_scripts/pip_package/requirements.in -- $REQUIREMENTS_EXTRA_FLAGS # bazel run //oss_scripts/pip_package:requirements.update -- $REQUIREMENTS_EXTRA_FLAGS TF_ABIFLAG=$(bazel run //oss_scripts/pip_package:tensorflow_build_info -- abi) From 9683a1d8dcc4096e16fd1592f1063e869d92073c Mon Sep 17 00:00:00 2001 From: davidhabboosh1 Date: Tue, 12 Aug 2025 22:24:54 -0400 Subject: [PATCH 18/28] syntax --- oss_scripts/configure.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/oss_scripts/configure.sh b/oss_scripts/configure.sh index 662e20359..c021b1f6b 100755 --- a/oss_scripts/configure.sh +++ b/oss_scripts/configure.sh @@ -76,7 +76,7 @@ if [[ "$TF_VERSION" == *"rc"* ]]; then REQUIREMENTS_EXTRA_FLAGS="$REQUIREMENTS_EXTRA_FLAGS --pre" fi -pip-compile oss_scripts/pip_package/requirements.in -- $REQUIREMENTS_EXTRA_FLAGS +pip-compile -- $REQUIREMENTS_EXTRA_FLAGS oss_scripts/pip_package/requirements.in # bazel run //oss_scripts/pip_package:requirements.update -- $REQUIREMENTS_EXTRA_FLAGS TF_ABIFLAG=$(bazel run //oss_scripts/pip_package:tensorflow_build_info -- abi) From 428ac16caf0f8ac1e5732de82626a550e6225582 Mon Sep 17 00:00:00 2001 From: davidhabboosh1 Date: Tue, 12 Aug 2025 22:27:01 -0400 Subject: [PATCH 19/28] debug --- oss_scripts/configure.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/oss_scripts/configure.sh b/oss_scripts/configure.sh index c021b1f6b..257b8efd9 100755 --- a/oss_scripts/configure.sh +++ b/oss_scripts/configure.sh @@ -76,6 +76,7 @@ if [[ "$TF_VERSION" == *"rc"* ]]; then REQUIREMENTS_EXTRA_FLAGS="$REQUIREMENTS_EXTRA_FLAGS --pre" fi +echo $REQUIREMENTS_EXTRA_FLAGS pip-compile -- $REQUIREMENTS_EXTRA_FLAGS oss_scripts/pip_package/requirements.in # bazel run //oss_scripts/pip_package:requirements.update -- $REQUIREMENTS_EXTRA_FLAGS From 3f9999211ec0a08a61bef77b54318de2d5a70bcf Mon Sep 17 00:00:00 2001 From: davidhabboosh1 Date: Tue, 12 Aug 2025 22:28:03 -0400 Subject: [PATCH 20/28] more debug --- oss_scripts/configure.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/oss_scripts/configure.sh b/oss_scripts/configure.sh index 257b8efd9..ca6f82ccf 100755 --- a/oss_scripts/configure.sh +++ b/oss_scripts/configure.sh @@ -77,6 +77,7 @@ if [[ "$TF_VERSION" == *"rc"* ]]; then fi echo $REQUIREMENTS_EXTRA_FLAGS +echo pip-compile -- $REQUIREMENTS_EXTRA_FLAGS oss_scripts/pip_package/requirements.in pip-compile -- $REQUIREMENTS_EXTRA_FLAGS oss_scripts/pip_package/requirements.in # bazel run //oss_scripts/pip_package:requirements.update -- $REQUIREMENTS_EXTRA_FLAGS From 381963717eca6d96b3b9f1832e73f0ec484aa143 Mon Sep 17 00:00:00 2001 From: davidhabboosh1 Date: Tue, 12 Aug 2025 22:28:43 -0400 Subject: [PATCH 21/28] syntax --- oss_scripts/configure.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/oss_scripts/configure.sh b/oss_scripts/configure.sh index ca6f82ccf..1fb8a5539 100755 --- a/oss_scripts/configure.sh +++ b/oss_scripts/configure.sh @@ -77,8 +77,8 @@ if [[ "$TF_VERSION" == *"rc"* ]]; then fi echo $REQUIREMENTS_EXTRA_FLAGS -echo pip-compile -- $REQUIREMENTS_EXTRA_FLAGS oss_scripts/pip_package/requirements.in -pip-compile -- $REQUIREMENTS_EXTRA_FLAGS oss_scripts/pip_package/requirements.in +echo pip-compile $REQUIREMENTS_EXTRA_FLAGS oss_scripts/pip_package/requirements.in +pip-compile $REQUIREMENTS_EXTRA_FLAGS oss_scripts/pip_package/requirements.in # bazel run //oss_scripts/pip_package:requirements.update -- $REQUIREMENTS_EXTRA_FLAGS TF_ABIFLAG=$(bazel run //oss_scripts/pip_package:tensorflow_build_info -- abi) From 3e10d48ba4999f5c287d7bd8ba7284da7c2c9a1d Mon Sep 17 00:00:00 2001 From: davidhabboosh1 Date: Tue, 12 Aug 2025 22:32:22 -0400 Subject: [PATCH 22/28] revamp TF_ABIFLAG --- oss_scripts/configure.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/oss_scripts/configure.sh b/oss_scripts/configure.sh index 1fb8a5539..feaa41130 100755 --- a/oss_scripts/configure.sh +++ b/oss_scripts/configure.sh @@ -81,7 +81,8 @@ echo pip-compile $REQUIREMENTS_EXTRA_FLAGS oss_scripts/pip_package/requirements. pip-compile $REQUIREMENTS_EXTRA_FLAGS oss_scripts/pip_package/requirements.in # bazel run //oss_scripts/pip_package:requirements.update -- $REQUIREMENTS_EXTRA_FLAGS -TF_ABIFLAG=$(bazel run //oss_scripts/pip_package:tensorflow_build_info -- abi) +TF_ABIFLAG = $(python3 oss_scripts/pip_package/tensorflow_build_info.py --abi) +# TF_ABIFLAG=$(bazel run //oss_scripts/pip_package:tensorflow_build_info -- abi) SHARED_LIBRARY_NAME="libtensorflow_framework.so.2" if is_macos; then SHARED_LIBRARY_NAME="libtensorflow_framework.2.dylib" From c4e5eeaba33beeaf5f422c063528ce3e3cee5c29 Mon Sep 17 00:00:00 2001 From: davidhabboosh1 Date: Tue, 12 Aug 2025 22:33:40 -0400 Subject: [PATCH 23/28] syntax --- oss_scripts/configure.sh | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/oss_scripts/configure.sh b/oss_scripts/configure.sh index feaa41130..9cdf23228 100755 --- a/oss_scripts/configure.sh +++ b/oss_scripts/configure.sh @@ -76,12 +76,10 @@ if [[ "$TF_VERSION" == *"rc"* ]]; then REQUIREMENTS_EXTRA_FLAGS="$REQUIREMENTS_EXTRA_FLAGS --pre" fi -echo $REQUIREMENTS_EXTRA_FLAGS -echo pip-compile $REQUIREMENTS_EXTRA_FLAGS oss_scripts/pip_package/requirements.in pip-compile $REQUIREMENTS_EXTRA_FLAGS oss_scripts/pip_package/requirements.in # bazel run //oss_scripts/pip_package:requirements.update -- $REQUIREMENTS_EXTRA_FLAGS -TF_ABIFLAG = $(python3 oss_scripts/pip_package/tensorflow_build_info.py --abi) +TF_ABIFLAG=$(python3 oss_scripts/pip_package/tensorflow_build_info.py --abi) # TF_ABIFLAG=$(bazel run //oss_scripts/pip_package:tensorflow_build_info -- abi) SHARED_LIBRARY_NAME="libtensorflow_framework.so.2" if is_macos; then From 2683b2efc5bcb2da0ffc1da608e848f730d2c191 Mon Sep 17 00:00:00 2001 From: davidhabboosh1 Date: Tue, 12 Aug 2025 22:36:26 -0400 Subject: [PATCH 24/28] remove bazel references from prepare_tf_dep.sh --- oss_scripts/prepare_tf_dep.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/oss_scripts/prepare_tf_dep.sh b/oss_scripts/prepare_tf_dep.sh index 5210b358d..845c9ca9e 100755 --- a/oss_scripts/prepare_tf_dep.sh +++ b/oss_scripts/prepare_tf_dep.sh @@ -14,7 +14,8 @@ if [[ "${osname}" == "darwin" ]]; then fi # Update setup.nightly.py with current tf version. -tf_version=$(bazel run //oss_scripts/pip_package:tensorflow_build_info -- version) +# tf_version=$(bazel run //oss_scripts/pip_package:tensorflow_build_info -- version) +tf_version=$(python3 oss_scripts/pip_package/tensorflow_build_info.py --version) echo "Updating setup.nightly.py to version $tf_version" sed -i $ext "s/project_version = '.*'/project_version = '${tf_version}'/" oss_scripts/pip_package/setup.nightly.py # Update __version__. @@ -23,7 +24,8 @@ sed -i $ext "s/__version__ = .*\$/__version__ = \"${tf_version}\"/" tensorflow_t # Get git commit sha of installed tensorflow. echo "Querying commit SHA" -short_commit_sha=$(bazel run //oss_scripts/pip_package:tensorflow_build_info -- git_version) +# short_commit_sha=$(bazel run //oss_scripts/pip_package:tensorflow_build_info -- git_version) +short_commit_sha=$(python3 oss_scripts/pip_package/tensorflow_build_info.py --git_version) if [[ "$short_commit_sha" == "unknown" ]]; then # Some nightly builds report "unknown" for tf.__git_version. echo 'TF git version "unknown", assuming nightly.' From 876145a6a5545cf56f5a409d8cc56012eacd6791 Mon Sep 17 00:00:00 2001 From: davidhabboosh1 Date: Tue, 12 Aug 2025 22:44:51 -0400 Subject: [PATCH 25/28] debug --- oss_scripts/run_build.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/oss_scripts/run_build.sh b/oss_scripts/run_build.sh index 029af0b16..b38efae34 100755 --- a/oss_scripts/run_build.sh +++ b/oss_scripts/run_build.sh @@ -19,6 +19,7 @@ if [[ $osname != "darwin" ]] || [[ ! $(sysctl -n machdep.cpu.brand_string) =~ "A fi # Build the pip package. +echo "RUNNING THIS NOW!!!!" bazel run ${BUILD_ARGS[@]} --enable_runfiles //oss_scripts/pip_package:build_pip_package -- "$(realpath .)" if [ -n "${AUDITWHEEL_PLATFORM}" ]; then From 63b45f21c551b22e729690f363d7cc4abffff6ac Mon Sep 17 00:00:00 2001 From: davidhabboosh1 Date: Tue, 12 Aug 2025 22:47:26 -0400 Subject: [PATCH 26/28] debug removing -e --- oss_scripts/prepare_tf_dep.sh | 2 +- oss_scripts/run_build.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/oss_scripts/prepare_tf_dep.sh b/oss_scripts/prepare_tf_dep.sh index 845c9ca9e..92497aea3 100755 --- a/oss_scripts/prepare_tf_dep.sh +++ b/oss_scripts/prepare_tf_dep.sh @@ -1,5 +1,5 @@ #!/bin/bash -set -e # fail and exit on any command erroring +# set -e # fail and exit on any command erroring if (which python3) | grep -q "python3"; then installed_python="python3" diff --git a/oss_scripts/run_build.sh b/oss_scripts/run_build.sh index b38efae34..29289b1e1 100755 --- a/oss_scripts/run_build.sh +++ b/oss_scripts/run_build.sh @@ -1,5 +1,5 @@ #!/bin/bash -set -e # fail and exit on any command erroring +# set -e # fail and exit on any command erroring osname="$(uname -s | tr 'A-Z' 'a-z')" From eb8932b54b0a0212a36c42adc10e197320f7542d Mon Sep 17 00:00:00 2001 From: davidhabboosh1 Date: Tue, 12 Aug 2025 23:13:10 -0400 Subject: [PATCH 27/28] add python version of building --- build_pip_package.py | 73 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 73 insertions(+) create mode 100644 build_pip_package.py diff --git a/build_pip_package.py b/build_pip_package.py new file mode 100644 index 000000000..76085c67b --- /dev/null +++ b/build_pip_package.py @@ -0,0 +1,73 @@ +import os +import shutil +import subprocess +import sys +import tempfile + +def is_nightly(): + return os.environ.get("IS_NIGHTLY") == "nightly" + +def abspath(path): + return os.path.abspath(path) + +def main(output_dir=None): + if output_dir is None: + output_dir = "/tmp/tensorflow_text_pkg" + os.makedirs(output_dir, exist_ok=True) + output_dir = abspath(output_dir) + + print(f"=== Destination directory: {output_dir}") + print(f"=== Current directory: {os.getcwd()}") + + # Source directories/files in your repo (relative to repo root) + src_tensorflow_text = os.path.join("tensorflow_text") + src_setup_py = os.path.join("oss_scripts", "pip_package", "setup.nightly.py" if is_nightly() else "setup.py") + src_manifest = os.path.join("oss_scripts", "pip_package", "MANIFEST.in") + src_license = os.path.join("oss_scripts", "pip_package", "LICENSE") + + # Check if source exists + if not os.path.isdir(src_tensorflow_text): + print(f"Error: directory '{src_tensorflow_text}' not found.", file=sys.stderr) + sys.exit(1) + for f in [src_setup_py, src_manifest, src_license]: + if not os.path.isfile(f): + print(f"Error: file '{f}' not found.", file=sys.stderr) + sys.exit(1) + + temp_dir = tempfile.mkdtemp() + print(f"=== Using tmpdir {temp_dir}") + + # Copy tensorflow_text directory + dest_tf_text = os.path.join(temp_dir, "tensorflow_text") + shutil.copytree(src_tensorflow_text, dest_tf_text) + + # Copy setup.py or setup.nightly.py + shutil.copy(src_setup_py, temp_dir) + setup_script = os.path.basename(src_setup_py) + + # Copy MANIFEST.in and LICENSE + shutil.copy(src_manifest, temp_dir) + shutil.copy(src_license, temp_dir) + + # Run setup.py bdist_wheel --universal + old_cwd = os.getcwd() + os.chdir(temp_dir) + try: + python_cmd = shutil.which("python3") or shutil.which("python") + if python_cmd is None: + print("Python not found in PATH.", file=sys.stderr) + sys.exit(1) + + subprocess.run([python_cmd, setup_script, "bdist_wheel", "--universal"], check=True) + # Copy generated wheel(s) to output directory + dist_dir = os.path.join(temp_dir, "dist") + for filename in os.listdir(dist_dir): + if filename.endswith(".whl"): + shutil.copy(os.path.join(dist_dir, filename), output_dir) + finally: + os.chdir(old_cwd) + shutil.rmtree(temp_dir) + +if __name__ == "__main__": + output_dir_arg = sys.argv[1] if len(sys.argv) > 1 else None + main(output_dir_arg) From 201064587e47c677ac9f0c8fcbf87ad5d28fefa7 Mon Sep 17 00:00:00 2001 From: davidhabboosh1 Date: Tue, 12 Aug 2025 23:30:24 -0400 Subject: [PATCH 28/28] revert sh files to original and create build_pip_package.py --- build_pip_package.py | 8 ++++++-- oss_scripts/configure.sh | 6 ++---- oss_scripts/prepare_tf_dep.sh | 8 +++----- oss_scripts/run_build.sh | 3 +-- 4 files changed, 12 insertions(+), 13 deletions(-) diff --git a/build_pip_package.py b/build_pip_package.py index 76085c67b..7e7bc5c8f 100644 --- a/build_pip_package.py +++ b/build_pip_package.py @@ -53,11 +53,15 @@ def main(output_dir=None): old_cwd = os.getcwd() os.chdir(temp_dir) try: - python_cmd = shutil.which("python3") or shutil.which("python") - if python_cmd is None: + if shutil.which('python3'): + python_cmd = 'python3' + elif shutil.which('python'): + python_cmd = 'python' + else: print("Python not found in PATH.", file=sys.stderr) sys.exit(1) + print(python_cmd, setup_script, "bdist_wheel", "--universal") subprocess.run([python_cmd, setup_script, "bdist_wheel", "--universal"], check=True) # Copy generated wheel(s) to output directory dist_dir = os.path.join(temp_dir, "dist") diff --git a/oss_scripts/configure.sh b/oss_scripts/configure.sh index 9cdf23228..a7f4e96a6 100755 --- a/oss_scripts/configure.sh +++ b/oss_scripts/configure.sh @@ -76,11 +76,9 @@ if [[ "$TF_VERSION" == *"rc"* ]]; then REQUIREMENTS_EXTRA_FLAGS="$REQUIREMENTS_EXTRA_FLAGS --pre" fi -pip-compile $REQUIREMENTS_EXTRA_FLAGS oss_scripts/pip_package/requirements.in -# bazel run //oss_scripts/pip_package:requirements.update -- $REQUIREMENTS_EXTRA_FLAGS +bazel run //oss_scripts/pip_package:requirements.update -- $REQUIREMENTS_EXTRA_FLAGS -TF_ABIFLAG=$(python3 oss_scripts/pip_package/tensorflow_build_info.py --abi) -# TF_ABIFLAG=$(bazel run //oss_scripts/pip_package:tensorflow_build_info -- abi) +TF_ABIFLAG=$(bazel run //oss_scripts/pip_package:tensorflow_build_info -- abi) SHARED_LIBRARY_NAME="libtensorflow_framework.so.2" if is_macos; then SHARED_LIBRARY_NAME="libtensorflow_framework.2.dylib" diff --git a/oss_scripts/prepare_tf_dep.sh b/oss_scripts/prepare_tf_dep.sh index 92497aea3..5210b358d 100755 --- a/oss_scripts/prepare_tf_dep.sh +++ b/oss_scripts/prepare_tf_dep.sh @@ -1,5 +1,5 @@ #!/bin/bash -# set -e # fail and exit on any command erroring +set -e # fail and exit on any command erroring if (which python3) | grep -q "python3"; then installed_python="python3" @@ -14,8 +14,7 @@ if [[ "${osname}" == "darwin" ]]; then fi # Update setup.nightly.py with current tf version. -# tf_version=$(bazel run //oss_scripts/pip_package:tensorflow_build_info -- version) -tf_version=$(python3 oss_scripts/pip_package/tensorflow_build_info.py --version) +tf_version=$(bazel run //oss_scripts/pip_package:tensorflow_build_info -- version) echo "Updating setup.nightly.py to version $tf_version" sed -i $ext "s/project_version = '.*'/project_version = '${tf_version}'/" oss_scripts/pip_package/setup.nightly.py # Update __version__. @@ -24,8 +23,7 @@ sed -i $ext "s/__version__ = .*\$/__version__ = \"${tf_version}\"/" tensorflow_t # Get git commit sha of installed tensorflow. echo "Querying commit SHA" -# short_commit_sha=$(bazel run //oss_scripts/pip_package:tensorflow_build_info -- git_version) -short_commit_sha=$(python3 oss_scripts/pip_package/tensorflow_build_info.py --git_version) +short_commit_sha=$(bazel run //oss_scripts/pip_package:tensorflow_build_info -- git_version) if [[ "$short_commit_sha" == "unknown" ]]; then # Some nightly builds report "unknown" for tf.__git_version. echo 'TF git version "unknown", assuming nightly.' diff --git a/oss_scripts/run_build.sh b/oss_scripts/run_build.sh index 29289b1e1..029af0b16 100755 --- a/oss_scripts/run_build.sh +++ b/oss_scripts/run_build.sh @@ -1,5 +1,5 @@ #!/bin/bash -# set -e # fail and exit on any command erroring +set -e # fail and exit on any command erroring osname="$(uname -s | tr 'A-Z' 'a-z')" @@ -19,7 +19,6 @@ if [[ $osname != "darwin" ]] || [[ ! $(sysctl -n machdep.cpu.brand_string) =~ "A fi # Build the pip package. -echo "RUNNING THIS NOW!!!!" bazel run ${BUILD_ARGS[@]} --enable_runfiles //oss_scripts/pip_package:build_pip_package -- "$(realpath .)" if [ -n "${AUDITWHEEL_PLATFORM}" ]; then