From 5c195e5194b6678f3e245c2f566b1f82c3fa1477 Mon Sep 17 00:00:00 2001 From: sarthfrey Date: Wed, 18 Aug 2021 19:51:38 -0400 Subject: [PATCH 1/2] Unbundle tensorflow from spark-tensorflow-distributor --- spark/spark-tensorflow-distributor/README.md | 2 ++ spark/spark-tensorflow-distributor/requirements.txt | 4 +--- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/spark/spark-tensorflow-distributor/README.md b/spark/spark-tensorflow-distributor/README.md index 3ff88c75..27df3e5f 100644 --- a/spark/spark-tensorflow-distributor/README.md +++ b/spark/spark-tensorflow-distributor/README.md @@ -12,6 +12,8 @@ pip install spark-tensorflow-distributor ``` The installation does not install PySpark because for most users, PySpark is already installed. +In addition, `tensorflow` not installed so that users may choose between regular and CPU-only +installation via `pip install tensorflow` and `pip install tensorflow-cpu`. If you do not have PySpark installed, you can install it directly: ```bash diff --git a/spark/spark-tensorflow-distributor/requirements.txt b/spark/spark-tensorflow-distributor/requirements.txt index d6d877ca..2ab4c98a 100644 --- a/spark/spark-tensorflow-distributor/requirements.txt +++ b/spark/spark-tensorflow-distributor/requirements.txt @@ -6,13 +6,11 @@ ## Tensorflow Requirements ## # Tensorflow is required to run this code but depends on specific configurations. Install from: # https://www.tensorflow.org/get_started/os_setup#overview +# Either `tensorflow` or `tensorflow-cpu` should be installed via pypi depending on if you want GPU support # NOTE: If you want to use the GPU version, you will also need Nvidia's CUDA toolkit and cuDNN: # https://developer.nvidia.com/cuda-downloads # https://developer.nvidia.com/cudnn -# Note that if you want to use the GPU version, you have to `pip uninstall tensorflow` -# and `pip install tensorflow-gpu`, since both cannot coexist. -tensorflow>=2.1.0 # We'll use pytest to run our tests; this isn't really necessary to run the code, but it is to run # the tests. With this here, you can run the tests with `py.test` from the base directory. From 25a99b7d3743eb35cbf1623fe92ceaa315859924 Mon Sep 17 00:00:00 2001 From: sarthfrey Date: Wed, 18 Aug 2021 20:27:32 -0400 Subject: [PATCH 2/2] update version --- spark/spark-tensorflow-distributor/setup.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/spark/spark-tensorflow-distributor/setup.py b/spark/spark-tensorflow-distributor/setup.py index e86cd114..250fd51a 100644 --- a/spark/spark-tensorflow-distributor/setup.py +++ b/spark/spark-tensorflow-distributor/setup.py @@ -6,7 +6,7 @@ setuptools.setup( name="spark_tensorflow_distributor", - version="0.0.3", + version="1.0.0", author="sarthfrey", author_email="sarth.frey@gmail.com", description="This package helps users do distributed training with TensorFlow on their Spark clusters.", @@ -24,8 +24,5 @@ "Topic :: Software Development :: Libraries :: Python Modules", "Topic :: Software Development :: Version Control :: Git", ], - install_requires=[ - "tensorflow>=2.1.0", - ], python_requires='>=3.6', )