From 7318567944a58293a5272b4d82ebb4a032b8ce8f Mon Sep 17 00:00:00 2001 From: Matt Wormley Date: Tue, 7 Mar 2017 09:33:51 -0800 Subject: [PATCH 1/9] added VAULT_DIRS argument to allow for more than .ssh. Making example docker-vault run command accurate to match ONVAULT parameters @johnleetran --- ONVAULT | 69 +++++++++++++++++++++++++++++++++----------------------- index.js | 42 ++++++++++++++++++++-------------- 2 files changed, 66 insertions(+), 45 deletions(-) diff --git a/ONVAULT b/ONVAULT index 30fd907..3c0fe25 100755 --- a/ONVAULT +++ b/ONVAULT @@ -16,6 +16,11 @@ set -e # allow overriding default VAULT_SSH_KEY at runtime : ${VAULT_SSH_KEY:=id_rsa} +# allow multiple dot directories to be managed +: ${VAULT_DIRS:=ssh} + +: ${VAULT_IMAGE:=dockito/vault} + # parse arguments while [[ "$#" > 1 ]]; do case $1 in --disable-pwd) DISABLE_PASSWORD="$2";; @@ -33,32 +38,29 @@ log () { no_proxy_old="$no_proxy" export no_proxy="$VAULT_HOST" -if ! curl -s "${VAULT_URI}/_ping"; then +if ! curl --connect-timeout 1 -s "${VAULT_URI}/_ping"; then COUNTER=0 - echo 'Waiting 10s for dockito/vault to be ready...' - while ! curl -s "${VAULT_URI}/_ping" && [ $COUNTER != 10 ]; do + echo "Waiting 10s for ${VAULT_IMAGE} to be ready..." + while ! curl --connect-timeout 1 -s "${VAULT_URI}/_ping" && [ $COUNTER != 10 ]; do sleep 1 COUNTER=$[$COUNTER +1] done fi -if curl -s "${VAULT_URI}/_ping"; then - mkdir -p ~/.ssh/ - - # check if is required the ssh backup - ssh_backup_enabled="$(ls -A ~/.ssh)" - - # creating backup of existing ssh directory - if [[ -n "$ssh_backup_enabled" ]]; then - tmp_ssh_vault=~/".vault-backup-ssh-$(date +%s)" - mkdir $tmp_ssh_vault - cp -r ~/.ssh/* $tmp_ssh_vault - fi - +if curl --connect-timeout 1 -s "${VAULT_URI}/_ping"; then log "Downloading private keys..." - curl -s "${VAULT_URI}/ssh.tgz" | tar -C ~/.ssh/ -zxf - - chown -f `whoami` ~/.ssh/* || true - chmod -f 600 ~/.ssh/* || true + tmp_vault=~/".vault-backup-$(date +%s)" + mkdir $tmp_vault + for vault_dir in $VAULT_DIRS; do + if [[ -e ~/.${vault_dir} ]]; then + mv ~/.${vault_dir} ${tmp_vault} + fi + mkdir ~/.${vault_dir} + + curl -s "${VAULT_URI}/${vault_dir}.tgz" | tar -C ~/.${vault_dir}/ -zxf - + chown -f `whoami` ~/.${vault_dir}/* || true + chmod -f 600 ~/.${vault_dir}/* || true + done log "Using ssh key: $VAULT_SSH_KEY" if [[ "$VAULT_SSH_KEY" != "id_rsa" ]]; then @@ -77,15 +79,26 @@ if curl -s "${VAULT_URI}/_ping"; then eval $@ log "Removing private keys..." - rm -rf ~/.ssh/* - - # copying backup to ssh directory - if [[ -n "$ssh_backup_enabled" ]]; then - cp -r $tmp_ssh_vault/* ~/.ssh - rm -rf $tmp_ssh_vault - fi + for vault_dir in $VAULT_DIRS; do + rm -rf ~/.${vault_dir} + if [[ -e ${tmp_vault}/.${vault_dir} ]]; then + mv ${tmp_vault}/.${vault_dir} ~ + fi + done + rmdir $tmp_vault else - log "ERROR: Start the dockito/vault container before using ONVAULT!" - log "ex: docker run -d -p ${VAULT_HOST}:14242:3000 -v ~/.ssh:/vault/.ssh dockito/vault" + log "ERROR: Start the ${VAULT_IMAGE} container before using ONVAULT!" + document_args="" + if [[ "$VAULT_DIRS" != "ssh" ]]; then + document_args="${document_args} -e VAULT_DIRS='${VAULT_DIRS}'" + fi + if [[ "$VAULT_PORT" = "tcp://${VAULT_HOST}:14242" ]]; then + document_args="${document_args} -p ${VAULT_HOST}:14242:3000" + fi + for vault_dir in $VAULT_DIRS; do + document_args="${document_args} -v ~/.${vault_dir}:/vault/.${vault_dir}" + done + + log "ex: docker run -d${document_args} ${VAULT_IMAGE}" exit 1 fi diff --git a/index.js b/index.js index af904c0..f4394fd 100644 --- a/index.js +++ b/index.js @@ -18,26 +18,34 @@ app.get('/_ping', function (req, res) { }); +var keyDirs = (process.env.VAULT_DIRS || 'ssh').split(' '); + /** - Bundle containing all the user's private keys and ssh configuration + Bundle containing all the user's private keys and "keyDir" configuration */ -app.get('/ssh.tgz', function (req, res) { - mkdirp("/vault/.ssh"); - exec('mktemp -q /tmp/ssh.XXXXXX', function (err, stdout) { - var file = stdout.match(/(.+)/)[0]; - - exec('tar -chz -C /vault/.ssh -f '+ file +' .', function (err, stdout, stderr) { - var filename = path.basename(file); - var mimetype = mime.lookup(file); - - res.setHeader('Content-disposition', 'attachment; filename=' + filename); - res.setHeader('Content-type', mimetype); - - var filestream = fs.createReadStream(file); - filestream.pipe(res); - fs.unlink(file) +app.get('/:keyDir\.tgz', function (req, res) { + keyDir = req.params.keyDir; + + if (keyDirs.indexOf(keyDir) == -1) { + res.status(404).send('Not found'); + } else { + mkdirp("/vault/." + keyDir); + exec('mktemp -q /tmp/key_dir.XXXXXX', function (err, stdout) { + var file = stdout.match(/(.+)/)[0]; + + exec('tar -chz -C /vault/.' + keyDir + ' -f '+ file +' .', function (err, stdout, stderr) { + var filename = path.basename(file); + var mimetype = mime.lookup(file); + + res.setHeader('Content-disposition', 'attachment; filename=' + filename); + res.setHeader('Content-type', mimetype); + + var filestream = fs.createReadStream(file); + filestream.pipe(res); + fs.unlink(file) + }); }); - }); + } }); From 6703bd3de0f131d5a92c1163f4d89c76a47d506f Mon Sep 17 00:00:00 2001 From: Matt Wormley Date: Tue, 7 Mar 2017 09:51:35 -0800 Subject: [PATCH 2/9] comment to trigger raw file update in github? --- ONVAULT | 1 + 1 file changed, 1 insertion(+) diff --git a/ONVAULT b/ONVAULT index 3c0fe25..728fd37 100755 --- a/ONVAULT +++ b/ONVAULT @@ -19,6 +19,7 @@ set -e # allow multiple dot directories to be managed : ${VAULT_DIRS:=ssh} +# just for documentation : ${VAULT_IMAGE:=dockito/vault} # parse arguments From 1ee3fcfa8d4d4bf331e1b2dad540b9d11737682b Mon Sep 17 00:00:00 2001 From: Matt Wormley Date: Wed, 22 Mar 2017 09:41:36 -0700 Subject: [PATCH 3/9] only passing through "config" file out of .bundle --- index.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/index.js b/index.js index f4394fd..9c86bd7 100644 --- a/index.js +++ b/index.js @@ -32,8 +32,10 @@ app.get('/:keyDir\.tgz', function (req, res) { mkdirp("/vault/." + keyDir); exec('mktemp -q /tmp/key_dir.XXXXXX', function (err, stdout) { var file = stdout.match(/(.+)/)[0]; + // the only place in .bundle directory for keys is the config file (which may have other bad stuff we'll deal with in ONVAULT) + var filespec = keyDir == 'bundle' ? './config' : '.' - exec('tar -chz -C /vault/.' + keyDir + ' -f '+ file +' .', function (err, stdout, stderr) { + exec('tar -chz -C /vault/.' + keyDir + ' -f '+ file + ' ' + filespec, function (err, stdout, stderr) { var filename = path.basename(file); var mimetype = mime.lookup(file); From a3edcabb41922675974964351ccc70972ce42f09 Mon Sep 17 00:00:00 2001 From: Matt Wormley Date: Wed, 22 Mar 2017 10:17:14 -0700 Subject: [PATCH 4/9] filtering out all bundler config values that are not auth @johnleetran --- ONVAULT | 29 ++++++++++++++++++++++++----- 1 file changed, 24 insertions(+), 5 deletions(-) diff --git a/ONVAULT b/ONVAULT index 728fd37..2acaab4 100755 --- a/ONVAULT +++ b/ONVAULT @@ -22,6 +22,9 @@ set -e # just for documentation : ${VAULT_IMAGE:=dockito/vault} +# allow multiple dot directories to be managed +: ${VAULT_BUNDLE_CREDENTIALS:=} + # parse arguments while [[ "$#" > 1 ]]; do case $1 in --disable-pwd) DISABLE_PASSWORD="$2";; @@ -48,19 +51,33 @@ if ! curl --connect-timeout 1 -s "${VAULT_URI}/_ping"; then done fi + if curl --connect-timeout 1 -s "${VAULT_URI}/_ping"; then log "Downloading private keys..." tmp_vault=~/".vault-backup-$(date +%s)" - mkdir $tmp_vault + mkdir $tmp_vault for vault_dir in $VAULT_DIRS; do if [[ -e ~/.${vault_dir} ]]; then mv ~/.${vault_dir} ${tmp_vault} fi - mkdir ~/.${vault_dir} + mkdir ~/.${vault_dir} curl -s "${VAULT_URI}/${vault_dir}.tgz" | tar -C ~/.${vault_dir}/ -zxf - chown -f `whoami` ~/.${vault_dir}/* || true chmod -f 600 ~/.${vault_dir}/* || true + + if [[ ${vault_dir} == bundle ]]; then + # more special handling of that nasty .bundle/config file. We ONLY want to bring over credentials, but the format + # for credentials is rather open-ended. Example: credentials for google.com would be stored as BUNDLE_GOOGLE__COM: "username:password" + # We solve this by allowing for a VAULT_BUNDLE_CREDENTIALS environment varialbe that is a space-deliminted list + # of gem server hostnames. This list will be transformed into these BUNDLE_HOSTNAME__WITH__UNDERLINES and used as a grep + # filter to filter out ALL OTHER lines in the ~/.bundle config. We do not want actual bundler configuration items + # such as BUNDLE_BUILD... or BUNDLE_PATH... or BUNDLE_WITHOUT to affect our build. + bundle_config_filter=$(tr '[:lower:]' '[:upper:]' <<< $VAULT_BUNDLE_CREDENTIALS | sed 's/\./__/g' | sed 's/ */\\|/g') + grep -- "---\|$bundle_config_filter" ~/.bundle/config > /tmp/bundle_config + mv /tmp/bundle_config ~/.bundle/config + fi + done log "Using ssh key: $VAULT_SSH_KEY" @@ -90,9 +107,9 @@ if curl --connect-timeout 1 -s "${VAULT_URI}/_ping"; then else log "ERROR: Start the ${VAULT_IMAGE} container before using ONVAULT!" document_args="" - if [[ "$VAULT_DIRS" != "ssh" ]]; then - document_args="${document_args} -e VAULT_DIRS='${VAULT_DIRS}'" - fi + if [[ "$VAULT_DIRS" != "ssh" ]]; then + document_args="${document_args} -e VAULT_DIRS='${VAULT_DIRS}'" + fi if [[ "$VAULT_PORT" = "tcp://${VAULT_HOST}:14242" ]]; then document_args="${document_args} -p ${VAULT_HOST}:14242:3000" fi @@ -103,3 +120,5 @@ else log "ex: docker run -d${document_args} ${VAULT_IMAGE}" exit 1 fi + +# vim: set ts=2 sw=2 tw=0 softtabstop=2 et : From bf2c6551273506c53ec2362cdc999a84be8d8a10 Mon Sep 17 00:00:00 2001 From: Matt Wormley Date: Wed, 29 Mar 2017 09:25:37 -0700 Subject: [PATCH 5/9] repassing quoted parameters http://stackoverflow.com/questions/448407/bash-script-to-receive-and-repass-quoted-parameters --- ONVAULT | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ONVAULT b/ONVAULT index 2acaab4..89352ce 100755 --- a/ONVAULT +++ b/ONVAULT @@ -94,7 +94,7 @@ if curl --connect-timeout 1 -s "${VAULT_URI}/_ping"; then export no_proxy="$no_proxy_old" log "Executing command: $@" - eval $@ + eval "$@" log "Removing private keys..." for vault_dir in $VAULT_DIRS; do From 04a84604e22e29939d4363d7c75f6c2ce19157b6 Mon Sep 17 00:00:00 2001 From: Matt Wormley Date: Wed, 29 Mar 2017 10:07:47 -0700 Subject: [PATCH 6/9] turns out easiest and most correct thing was to drop eval --- ONVAULT | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ONVAULT b/ONVAULT index 89352ce..ce1950d 100755 --- a/ONVAULT +++ b/ONVAULT @@ -94,7 +94,7 @@ if curl --connect-timeout 1 -s "${VAULT_URI}/_ping"; then export no_proxy="$no_proxy_old" log "Executing command: $@" - eval "$@" + "$@" log "Removing private keys..." for vault_dir in $VAULT_DIRS; do From e4eaa58a0dab9b601117bfbc9b679ab02a8d4299 Mon Sep 17 00:00:00 2001 From: Matt Wormley Date: Mon, 12 Jun 2017 16:59:47 -0700 Subject: [PATCH 7/9] no more 10 second retry. fail immediately. Use red --- ONVAULT | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/ONVAULT b/ONVAULT index ce1950d..052504a 100755 --- a/ONVAULT +++ b/ONVAULT @@ -38,20 +38,16 @@ log () { echo -e "${GREEN}[Dockito Vault]${NC} $@" } +error () { + RED='\033[1;31m' + NC='\033[0m' # No Color + echo -e "${RED}[Dockito Vault]${NC} $@" +} + # don't go through proxy for accessing vault no_proxy_old="$no_proxy" export no_proxy="$VAULT_HOST" -if ! curl --connect-timeout 1 -s "${VAULT_URI}/_ping"; then - COUNTER=0 - echo "Waiting 10s for ${VAULT_IMAGE} to be ready..." - while ! curl --connect-timeout 1 -s "${VAULT_URI}/_ping" && [ $COUNTER != 10 ]; do - sleep 1 - COUNTER=$[$COUNTER +1] - done -fi - - if curl --connect-timeout 1 -s "${VAULT_URI}/_ping"; then log "Downloading private keys..." tmp_vault=~/".vault-backup-$(date +%s)" @@ -105,7 +101,7 @@ if curl --connect-timeout 1 -s "${VAULT_URI}/_ping"; then done rmdir $tmp_vault else - log "ERROR: Start the ${VAULT_IMAGE} container before using ONVAULT!" + error "ERROR: Start the ${VAULT_IMAGE} container before using ONVAULT using the following command:" document_args="" if [[ "$VAULT_DIRS" != "ssh" ]]; then document_args="${document_args} -e VAULT_DIRS='${VAULT_DIRS}'" @@ -117,7 +113,7 @@ else document_args="${document_args} -v ~/.${vault_dir}:/vault/.${vault_dir}" done - log "ex: docker run -d${document_args} ${VAULT_IMAGE}" + error " docker run -name vault -d${document_args} ${VAULT_IMAGE}" exit 1 fi From 8a2d10601a29689fff26153b81e17ae359b4a86d Mon Sep 17 00:00:00 2001 From: Matt Wormley Date: Tue, 13 Jun 2017 14:31:03 -0700 Subject: [PATCH 8/9] typo --- ONVAULT | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ONVAULT b/ONVAULT index 052504a..f0b357e 100755 --- a/ONVAULT +++ b/ONVAULT @@ -113,7 +113,7 @@ else document_args="${document_args} -v ~/.${vault_dir}:/vault/.${vault_dir}" done - error " docker run -name vault -d${document_args} ${VAULT_IMAGE}" + error " docker run --name vault -d${document_args} ${VAULT_IMAGE}" exit 1 fi From c92479ccfe465b86a8949087a3e5bc485ec9502a Mon Sep 17 00:00:00 2001 From: matt-wormley Date: Tue, 13 Feb 2018 11:47:48 -0800 Subject: [PATCH 9/9] pretty echos go to stderr so we can scan stdout --- ONVAULT | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ONVAULT b/ONVAULT index f0b357e..53dccdf 100755 --- a/ONVAULT +++ b/ONVAULT @@ -35,13 +35,13 @@ done log () { GREEN='\033[1;32m' NC='\033[0m' # No Color - echo -e "${GREEN}[Dockito Vault]${NC} $@" + >&2 echo -e "${GREEN}[Dockito Vault]${NC} $@" } error () { RED='\033[1;31m' NC='\033[0m' # No Color - echo -e "${RED}[Dockito Vault]${NC} $@" + >&2 echo -e "${RED}[Dockito Vault]${NC} $@" } # don't go through proxy for accessing vault