Skip to content
This repository was archived by the owner on Oct 27, 2025. It is now read-only.
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
80 changes: 46 additions & 34 deletions .dots
Original file line number Diff line number Diff line change
Expand Up @@ -10,14 +10,14 @@ function runDots() {
# Run sections based on command line arguments
for ARG in "$@"
do
if [ $ARG == "bootstrap" ] || [ $ARG == "all" ]; then
echo ""
echo "------------------------------"
echo "Syncing the dev-setup repo to your local machine."
echo "------------------------------"
echo ""
cd ~ && curl -#L https://github.com/donnemartin/dev-setup/tarball/master | tar -xzv --strip-components 1 --exclude={README.md,LICENSE}
fi
# if [ $ARG == "bootstrap" ] || [ $ARG == "all" ]; then
# echo ""
# echo "------------------------------"
# echo "Syncing the dev-setup repo to your local machine."
# echo "------------------------------"
# echo ""
# cd ~ && curl -#L https://github.com/donnemartin/dev-setup/tarball/master | tar -xzv --strip-components 1 --exclude={README.md,LICENSE}
# fi
if [ $ARG == "osxprep" ] || [ $ARG == "all" ]; then
# Run the osxprep.sh Script
echo ""
Expand All @@ -40,29 +40,41 @@ function runDots() {
echo ""
./brew.sh
fi
if [ $ARG == "osx" ] || [ $ARG == "all" ]; then
# Run the osx.sh Script
# I strongly suggest you read through the commented osx.sh
# source file and tweak any settings based on your personal
# preferences. The script defaults are intended for you to
# customize. For example, if you are not running an SSD you
# might want to change some of the settings listed in the
# SSD section.
echo ""
echo "------------------------------"
echo "Setting sensible OSX defaults."
echo "------------------------------"
if [ $ARG == "terminal" ] || [ $ARG == "all" ]; then
# Run the brew.sh Script
# For a full listing of installed formulae and apps, refer to
# the commented brew.sh source file directly and tweak it to
# suit your needs.
echo ""
./osx.sh
fi
if [ $ARG == "pydata" ] || [ $ARG == "all" ]; then
# Run the pydata.sh Script
echo "------------------------------"
echo "Setting up Python data development environment."
echo "Installing Terminal updates along with some common formulae and apps."
echo "------------------------------"
echo ""
./pydata.sh
./terminal.sh
fi
# if [ $ARG == "osx" ] || [ $ARG == "all" ]; then
# # Run the osx.sh Script
# # I strongly suggest you read through the commented osx.sh
# # source file and tweak any settings based on your personal
# # preferences. The script defaults are intended for you to
# # customize. For example, if you are not running an SSD you
# # might want to change some of the settings listed in the
# # SSD section.
# echo ""
# echo "------------------------------"
# echo "Setting sensible OSX defaults."
# echo "------------------------------"
# echo ""
# ./osx.sh
# fi
# if [ $ARG == "pydata" ] || [ $ARG == "all" ]; then
# # Run the pydata.sh Script
# echo "------------------------------"
# echo "Setting up Python data development environment."
# echo "------------------------------"
# echo ""
# ./pydata.sh
# fi
if [ $ARG == "aws" ] || [ $ARG == "all" ]; then
# Run the aws.sh Script
echo "------------------------------"
Expand All @@ -87,14 +99,14 @@ function runDots() {
echo ""
./web.sh
fi
if [ $ARG == "android" ] || [ $ARG == "all" ]; then
# Run the android.sh Script
echo "------------------------------"
echo "Setting up Android development environment."
echo "------------------------------"
echo ""
./android.sh
fi
# if [ $ARG == "android" ] || [ $ARG == "all" ]; then
# # Run the android.sh Script
# echo "------------------------------"
# echo "Setting up Android development environment."
# echo "------------------------------"
# echo ""
# ./android.sh
# fi
done

echo "------------------------------"
Expand Down
126 changes: 64 additions & 62 deletions aws.sh
Original file line number Diff line number Diff line change
Expand Up @@ -11,51 +11,53 @@ echo "Setting up AWS."
echo "This script requires pip and virtualenvwrapper to be installed."
echo "See the pydata.sh script."

echo "------------------------------"
echo "Source virtualenvwrapper from ~/.extra"
source ~/.extra
# echo "------------------------------"
# echo "Source virtualenvwrapper from ~/.extra"
# source ~/.extra

###############################################################################
# Python 2 Virtual Enviroment #
###############################################################################

echo "------------------------------"
echo "Updating py2-data virtual environment with AWS modules."
# echo "------------------------------"
# echo "Updating py2-data virtual environment with AWS modules."

# Create a Python2 data environment
# If this environment already exists from running pydata.sh,
# it will not be overwritten
mkvirtualenv py2-data
workon py2-data

pip install boto
pip install awscli
pip install mrjob
pip install s3cmd

EXTRA_PATH=~/.extra
echo $EXTRA_PATH
echo "" >> $EXTRA_PATH
echo "" >> $EXTRA_PATH
echo "# Configure aws cli autocomplete, added by aws.sh" >> $EXTRA_PATH
echo "complete -C '~/.virtualenvs/py2-data/bin/aws_completer' aws" >> $EXTRA_PATH
source $EXTRA_PATH
# mkvirtualenv py2-data
# workon py2-data
curl "https://awscli.amazonaws.com/AWSCLIV2.pkg" -o "/tmp/AWSCLIV2.pkg"
sudo installer -pkg /tmp/AWSCLIV2.pkg -target /

pipenv install boto
# pipenv install awscli
pipenv install s3cmd
# pipenv install mrjob

# EXTRA_PATH=~/.extra
# echo $EXTRA_PATH
# echo "" >> $EXTRA_PATH
# echo "" >> $EXTRA_PATH
# echo "# Configure aws cli autocomplete, added by aws.sh" >> $EXTRA_PATH
# echo "complete -C '~/.virtualenvs/py2-data/bin/aws_completer' aws" >> $EXTRA_PATH
# source $EXTRA_PATH

###############################################################################
# Python 3 Virtual Enviroment #
###############################################################################

echo "------------------------------"
echo "Updating py3-data virtual environment with AWS modules."

# Create a Python3 data environment
# If this environment already exists from running pydata.sh,
# it will not be overwritten
mkvirtualenv --python=/usr/local/bin/python3 py3-data
workon py3-data

pip install boto
pip install awscli
# echo "------------------------------"
# echo "Updating py3-data virtual environment with AWS modules."
#
# # Create a Python3 data environment
# # If this environment already exists from running pydata.sh,
# # it will not be overwritten
# mkvirtualenv --python=/usr/local/bin/python3 py3-data
# workon py3-data
#
# pip install boto
# pip install awscli
#pip install mrjob # Python 2 only
#pip install s3cmd # Python 2 only

Expand All @@ -74,40 +76,40 @@ pip install awscli

# Check for Homebrew,
# Install if we don't have it
if test ! $(which brew); then
echo "Installing homebrew..."
ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)"
fi
# if test ! $(which brew); then
# echo "Installing homebrew..."
# ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)"
# fi

# Make sure we’re using the latest Homebrew.
brew update

brew install apache-spark

###############################################################################
# Install IPython Notebook Spark Integration
###############################################################################

echo "------------------------------"
echo "Installing IPython Notebook Spark integration"

# Add the pyspark IPython profile
cp -r init/profile_pyspark/ ~/.ipython/profile_pyspark

BASH_PROFILE_PATH=~/.bash_profile
echo $BASH_PROFILE_PATH
echo "" >> $BASH_PROFILE_PATH
echo "" >> $BASH_PROFILE_PATH
echo "# IPython Notebook Spark integration, added by aws.sh" >> $BASH_PROFILE_PATH
# Run $ brew info apache-spark to determine the Spark install location
echo "export SPARK_HOME='/usr/local/Cellar/apache-spark/1.4.1'" >> $BASH_PROFILE_PATH
echo "# Appending pyspark-shell is needed for Spark 1.4+" >> $BASH_PROFILE_PATH
echo "export PYSPARK_SUBMIT_ARGS='--master local[2] pyspark-shell'" >> $BASH_PROFILE_PATH
echo "" >> $BASH_PROFILE_PATH
source $BASH_PROFILE_PATH
# brew update
#
# brew install apache-spark
#
# ###############################################################################
# # Install IPython Notebook Spark Integration
# ###############################################################################
#
# echo "------------------------------"
# echo "Installing IPython Notebook Spark integration"
#
# # Add the pyspark IPython profile
# cp -r init/profile_pyspark/ ~/.ipython/profile_pyspark
#
# BASH_PROFILE_PATH=~/.bash_profile
# echo $BASH_PROFILE_PATH
# echo "" >> $BASH_PROFILE_PATH
# echo "" >> $BASH_PROFILE_PATH
# echo "# IPython Notebook Spark integration, added by aws.sh" >> $BASH_PROFILE_PATH
# # Run $ brew info apache-spark to determine the Spark install location
# echo "export SPARK_HOME='/usr/local/Cellar/apache-spark/1.4.1'" >> $BASH_PROFILE_PATH
# echo "# Appending pyspark-shell is needed for Spark 1.4+" >> $BASH_PROFILE_PATH
# echo "export PYSPARK_SUBMIT_ARGS='--master local[2] pyspark-shell'" >> $BASH_PROFILE_PATH
# echo "" >> $BASH_PROFILE_PATH
# source $BASH_PROFILE_PATH

echo "------------------------------"
echo "TODO: Update .aws/ with your AWS credentials and region, or run aws --configure."
echo "TODO: Update .mrjob.conf with your credentials, keypair, keypair location, region, and bucket info."
# echo "TODO: Update .mrjob.conf with your credentials, keypair, keypair location, region, and bucket info."
echo "TODO: Update .s3cfg with your credentials, location, and passphrase or run s3cmd --configure."
echo "Script completed."
echo "Script completed."
Loading