From 388be860adb3928f1dbfbd3893788b6647716f72 Mon Sep 17 00:00:00 2001 From: Xiangrui Meng Date: Mon, 22 Jun 2020 11:24:28 -0700 Subject: [PATCH 1/2] prepare 0.1.0 release --- spark/spark-tensorflow-distributor/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spark/spark-tensorflow-distributor/setup.py b/spark/spark-tensorflow-distributor/setup.py index e86cd114..6ad921dc 100644 --- a/spark/spark-tensorflow-distributor/setup.py +++ b/spark/spark-tensorflow-distributor/setup.py @@ -6,7 +6,7 @@ setuptools.setup( name="spark_tensorflow_distributor", - version="0.0.3", + version="0.1.0", author="sarthfrey", author_email="sarth.frey@gmail.com", description="This package helps users do distributed training with TensorFlow on their Spark clusters.", From 9199fe959856f49dfe8a66a6caa15916bf7015e3 Mon Sep 17 00:00:00 2001 From: Xiangrui Meng Date: Mon, 22 Jun 2020 11:26:29 -0700 Subject: [PATCH 2/2] use Spark 3.0 --- spark/spark-tensorflow-distributor/Dockerfile | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/spark/spark-tensorflow-distributor/Dockerfile b/spark/spark-tensorflow-distributor/Dockerfile index 165e8db6..b44c6fa2 100644 --- a/spark/spark-tensorflow-distributor/Dockerfile +++ b/spark/spark-tensorflow-distributor/Dockerfile @@ -15,10 +15,7 @@ RUN apt-get update && \ update-alternatives --install /usr/local/bin/python python /usr/bin/python${PYTHON_INSTALL_VERSION} 1 # Install Spark and update env variables -# Modify this when the next Spark 3 release includes the allGather barrier mode API -RUN pip install \ - https://ml-team-public-read.s3-us-west-2.amazonaws.com/pyspark-3.1.0.dev0-60dd1a690fed62b1d6442cdc8cf3f89ef4304d5a.tar.gz \ - --force-reinstall --upgrade +RUN pip install pyspark==3.0.0 # Set SPARK_HOME so that tests can easily find the scripts in $SPARK_HOME/sbin ENV SPARK_HOME /usr/local/lib/python${PYTHON_INSTALL_VERSION}/dist-packages/pyspark