diff --git a/spark/spark-tensorflow-distributor/Dockerfile b/spark/spark-tensorflow-distributor/Dockerfile index 165e8db6..b44c6fa2 100644 --- a/spark/spark-tensorflow-distributor/Dockerfile +++ b/spark/spark-tensorflow-distributor/Dockerfile @@ -15,10 +15,7 @@ RUN apt-get update && \ update-alternatives --install /usr/local/bin/python python /usr/bin/python${PYTHON_INSTALL_VERSION} 1 # Install Spark and update env variables -# Modify this when the next Spark 3 release includes the allGather barrier mode API -RUN pip install \ - https://ml-team-public-read.s3-us-west-2.amazonaws.com/pyspark-3.1.0.dev0-60dd1a690fed62b1d6442cdc8cf3f89ef4304d5a.tar.gz \ - --force-reinstall --upgrade +RUN pip install pyspark==3.0.0 # Set SPARK_HOME so that tests can easily find the scripts in $SPARK_HOME/sbin ENV SPARK_HOME /usr/local/lib/python${PYTHON_INSTALL_VERSION}/dist-packages/pyspark diff --git a/spark/spark-tensorflow-distributor/setup.py b/spark/spark-tensorflow-distributor/setup.py index e86cd114..6ad921dc 100644 --- a/spark/spark-tensorflow-distributor/setup.py +++ b/spark/spark-tensorflow-distributor/setup.py @@ -6,7 +6,7 @@ setuptools.setup( name="spark_tensorflow_distributor", - version="0.0.3", + version="0.1.0", author="sarthfrey", author_email="sarth.frey@gmail.com", description="This package helps users do distributed training with TensorFlow on their Spark clusters.",