From 0b8916eb578b5dfb8fd5ccddc87e86aa158bc0bd Mon Sep 17 00:00:00 2001 From: Matyas Selmeci Date: Thu, 6 May 2021 20:28:40 -0500 Subject: [PATCH 1/8] Allow having multiple keys for bosco Keys will be looked for in the following order: - `/etc/osg/boscokeys/${ruser}@${rhost}.key` - `/etc/osg/boscokeys/${ruser}.key` - `/etc/osg/bosco.key` --- hosted-ce/30-remote-site-setup.sh | 27 +++++++++++++++++++++------ 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/hosted-ce/30-remote-site-setup.sh b/hosted-ce/30-remote-site-setup.sh index 4c977f5..7953403 100755 --- a/hosted-ce/30-remote-site-setup.sh +++ b/hosted-ce/30-remote-site-setup.sh @@ -11,7 +11,8 @@ fi set -e -BOSCO_KEY=/etc/osg/bosco.key +DEFAULT_BOSCO_KEY=/etc/osg/bosco.key +BOSCOKEYS_DIR=/etc/osg/boscokeys ENDPOINT_CONFIG=/etc/endpoints.ini SKIP_WN_INSTALL=no @@ -32,7 +33,19 @@ function debug_file_contents { function fetch_remote_os_info { ruser=$1 rhost=$2 - ssh -q -i $BOSCO_KEY "$ruser@$rhost" "cat /etc/os-release" + ssh -q -i "$(get_bosco_key "$ruser" "$rhost")" "$ruser@$rhost" "cat /etc/os-release" +} + +function get_bosco_key { + ruser=$1 + rhost=$2 + if [[ -f $BOSCOKEYS_DIR/${ruser}@${rhost}.key ]]; then + echo "$BOSCOKEYS_DIR/${ruser}@${rhost}.key" + elif [[ -f $BOSCOKEYS_DIR/${ruser}.key ]]; then + echo "$BOSCOKEYS_DIR/${ruser}.key" + else + echo "$DEFAULT_BOSCO_KEY" + fi } setup_ssh_config () { @@ -45,7 +58,7 @@ setup_ssh_config () { # copy Bosco key ssh_key=$ssh_dir/bosco_key.rsa - cp $BOSCO_KEY $ssh_key + cp "$(get_bosco_key "$ruser" "$remote_fqdn")" $ssh_key chmod 600 $ssh_key chown "${ruser}": $ssh_key @@ -77,10 +90,11 @@ EOF setup_endpoints_ini () { echo "Setting up endpoint.ini entry for ${ruser}@$remote_fqdn..." remote_os_major_ver=$1 + ssh_key=$(get_bosco_key "$ruser" "$remote_fqdn") # The WN client updater uses "remote_dir" for WN client # configuration and remote copy. We need the absolute path # specifically for fetch-crl - remote_home_dir=$(ssh -q -i $BOSCO_KEY "${ruser}@$remote_fqdn" pwd) + remote_home_dir=$(ssh -q -i $ssh_key "${ruser}@$remote_fqdn" pwd) osg_ver=3.4 if [[ $remote_os_major_ver -gt 6 ]]; then osg_ver=3.5 @@ -91,6 +105,7 @@ local_user = ${ruser} remote_host = $remote_fqdn remote_user = ${ruser} remote_dir = $remote_home_dir/bosco-osg-wn-client +ssh_key = $ssh_key upstream_url = https://repo.opensciencegrid.org/tarball-install/${osg_ver}/osg-wn-client-latest.el${remote_os_major_ver}.x86_64.tar.gz EOF } @@ -111,12 +126,12 @@ REMOTE_HOST_KEY=`ssh-keyscan -p "$remote_port" "$remote_fqdn"` root_ssh_dir=/root/.ssh/ mkdir -p $root_ssh_dir chmod 700 $root_ssh_dir -ln -s $BOSCO_KEY $root_ssh_dir/bosco_key.rsa +ln -s "$(get_bosco_key "root" "$remote_fqdn")" $root_ssh_dir/bosco_key.rsa cat < /etc/ssh/ssh_config Host $remote_fqdn Port $remote_port - IdentityFile ${BOSCO_KEY} + IdentityFile "$(get_bosco_key "root" "$remote_fqdn")" ControlMaster auto ControlPath /tmp/cm-%i-%r@%h:%p ControlPersist 15m From bd87ea96cf4ab2242877758566a5a62f79b883dc Mon Sep 17 00:00:00 2001 From: Matyas Selmeci Date: Fri, 7 May 2021 12:31:51 -0500 Subject: [PATCH 2/8] Don't allow different keys for different hosts Keys will be looked for in the following order: - `/etc/osg/boscokeys/${ruser}.key` - `/etc/osg/bosco.key` --- hosted-ce/30-remote-site-setup.sh | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/hosted-ce/30-remote-site-setup.sh b/hosted-ce/30-remote-site-setup.sh index 7953403..9fb0126 100755 --- a/hosted-ce/30-remote-site-setup.sh +++ b/hosted-ce/30-remote-site-setup.sh @@ -33,15 +33,12 @@ function debug_file_contents { function fetch_remote_os_info { ruser=$1 rhost=$2 - ssh -q -i "$(get_bosco_key "$ruser" "$rhost")" "$ruser@$rhost" "cat /etc/os-release" + ssh -q -i "$(get_bosco_key "$ruser")" "$ruser@$rhost" "cat /etc/os-release" } function get_bosco_key { ruser=$1 - rhost=$2 - if [[ -f $BOSCOKEYS_DIR/${ruser}@${rhost}.key ]]; then - echo "$BOSCOKEYS_DIR/${ruser}@${rhost}.key" - elif [[ -f $BOSCOKEYS_DIR/${ruser}.key ]]; then + if [[ -f $BOSCOKEYS_DIR/${ruser}.key ]]; then echo "$BOSCOKEYS_DIR/${ruser}.key" else echo "$DEFAULT_BOSCO_KEY" @@ -58,7 +55,7 @@ setup_ssh_config () { # copy Bosco key ssh_key=$ssh_dir/bosco_key.rsa - cp "$(get_bosco_key "$ruser" "$remote_fqdn")" $ssh_key + cp "$(get_bosco_key "$ruser")" $ssh_key chmod 600 $ssh_key chown "${ruser}": $ssh_key @@ -90,7 +87,7 @@ EOF setup_endpoints_ini () { echo "Setting up endpoint.ini entry for ${ruser}@$remote_fqdn..." remote_os_major_ver=$1 - ssh_key=$(get_bosco_key "$ruser" "$remote_fqdn") + ssh_key=$(get_bosco_key "$ruser") # The WN client updater uses "remote_dir" for WN client # configuration and remote copy. We need the absolute path # specifically for fetch-crl @@ -126,12 +123,12 @@ REMOTE_HOST_KEY=`ssh-keyscan -p "$remote_port" "$remote_fqdn"` root_ssh_dir=/root/.ssh/ mkdir -p $root_ssh_dir chmod 700 $root_ssh_dir -ln -s "$(get_bosco_key "root" "$remote_fqdn")" $root_ssh_dir/bosco_key.rsa +ln -s "$(get_bosco_key "root")" $root_ssh_dir/bosco_key.rsa cat < /etc/ssh/ssh_config Host $remote_fqdn Port $remote_port - IdentityFile "$(get_bosco_key "root" "$remote_fqdn")" + IdentityFile "$(get_bosco_key "root")" ControlMaster auto ControlPath /tmp/cm-%i-%r@%h:%p ControlPersist 15m From 4d0495ae0b2d0639fab9889f3ba697e8d8150e5b Mon Sep 17 00:00:00 2001 From: Matyas Selmeci Date: Thu, 20 May 2021 20:34:25 -0500 Subject: [PATCH 3/8] Use the key of the first user for some operations instead of making a separate key for root --- hosted-ce/30-remote-site-setup.sh | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/hosted-ce/30-remote-site-setup.sh b/hosted-ce/30-remote-site-setup.sh index 9fb0126..ac83dcc 100755 --- a/hosted-ce/30-remote-site-setup.sh +++ b/hosted-ce/30-remote-site-setup.sh @@ -118,12 +118,20 @@ fi REMOTE_HOST_KEY=`ssh-keyscan -p "$remote_port" "$remote_fqdn"` [[ -n $REMOTE_HOST_KEY ]] || errexit "Failed to determine host key for $remote_fqdn:$remote_port" + +users=$(cat /etc/grid-security/grid-mapfile /etc/grid-security/voms-mapfile | \ + awk '/^"[^"]+" +[a-zA-Z0-9\-\._]+$/ {print $NF}' | \ + sort -u) +[[ -n $users ]] || errexit "Did not find any user mappings in the VOMS or Grid mapfiles" +# Use the first user for things we only need once +firstuser=$(printf "%s\n" $users | head -n1) + # HACK: Symlink the Bosco key to the location expected by # bosco_cluster so it doesn't go and try to generate a new one root_ssh_dir=/root/.ssh/ mkdir -p $root_ssh_dir chmod 700 $root_ssh_dir -ln -s "$(get_bosco_key "root")" $root_ssh_dir/bosco_key.rsa +install -o root -g root -m 0600 "$(get_bosco_key "firstuser")" $root_ssh_dir/bosco_key.rsa cat < /etc/ssh/ssh_config Host $remote_fqdn @@ -145,10 +153,6 @@ if [[ -n $BOSCO_GIT_ENDPOINT && -n $BOSCO_DIRECTORY ]]; then fi unset GIT_SSH_COMMAND -users=$(cat /etc/grid-security/grid-mapfile /etc/grid-security/voms-mapfile | \ - awk '/^"[^"]+" +[a-zA-Z0-9\-\._]+$/ {print $NF}' | \ - sort -u) -[[ -n $users ]] || errexit "Did not find any user mappings in the VOMS or Grid mapfiles" # Allow the condor user to run the WN client updater as the local users CONDOR_SUDO_FILE=/etc/sudoers.d/10-condor-ssh @@ -185,7 +189,7 @@ done ################### # We have to pick a user for SSH, may as well be the first one -remote_os_info=$(fetch_remote_os_info "$(printf "%s\n" $users | head -n1)" "$remote_fqdn") +remote_os_info=$(fetch_remote_os_info "$firstuser" "$remote_fqdn") remote_os_ver=$(echo "$remote_os_info" | awk -F '=' '/^VERSION_ID/ {print $2}' | tr -d '"') # Skip WN client installation for non-RHEL-based remote clusters From fbee4fc3e9690b88b67a844913ac6b985bb6d6ba Mon Sep 17 00:00:00 2001 From: Matyas Selmeci Date: Mon, 24 May 2021 15:43:49 -0500 Subject: [PATCH 4/8] Add TODO --- hosted-ce/30-remote-site-setup.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/hosted-ce/30-remote-site-setup.sh b/hosted-ce/30-remote-site-setup.sh index ac83dcc..b6458c4 100755 --- a/hosted-ce/30-remote-site-setup.sh +++ b/hosted-ce/30-remote-site-setup.sh @@ -119,6 +119,7 @@ REMOTE_HOST_KEY=`ssh-keyscan -p "$remote_port" "$remote_fqdn"` [[ -n $REMOTE_HOST_KEY ]] || errexit "Failed to determine host key for $remote_fqdn:$remote_port" +# TODO: Read from the SciTokens mapfile too? users=$(cat /etc/grid-security/grid-mapfile /etc/grid-security/voms-mapfile | \ awk '/^"[^"]+" +[a-zA-Z0-9\-\._]+$/ {print $NF}' | \ sort -u) From 893109523294390e29b170255479b2ec4f8022d2 Mon Sep 17 00:00:00 2001 From: Matyas Selmeci Date: Thu, 3 Jun 2021 17:45:53 -0500 Subject: [PATCH 5/8] Check for user existence; use get_mapped_users from ce-common-startup --- hosted-ce/30-remote-site-setup.sh | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/hosted-ce/30-remote-site-setup.sh b/hosted-ce/30-remote-site-setup.sh index b6458c4..b164b9c 100755 --- a/hosted-ce/30-remote-site-setup.sh +++ b/hosted-ce/30-remote-site-setup.sh @@ -1,5 +1,7 @@ #!/bin/bash +. /etc/osg/image-config.d/ce-common-startup + set -x # save old -e status @@ -119,25 +121,24 @@ REMOTE_HOST_KEY=`ssh-keyscan -p "$remote_port" "$remote_fqdn"` [[ -n $REMOTE_HOST_KEY ]] || errexit "Failed to determine host key for $remote_fqdn:$remote_port" -# TODO: Read from the SciTokens mapfile too? -users=$(cat /etc/grid-security/grid-mapfile /etc/grid-security/voms-mapfile | \ - awk '/^"[^"]+" +[a-zA-Z0-9\-\._]+$/ {print $NF}' | \ - sort -u) -[[ -n $users ]] || errexit "Did not find any user mappings in the VOMS or Grid mapfiles" +users=$(get_mapped_users) +[[ -n $users ]] || errexit "Did not find any user mappings" # Use the first user for things we only need once -firstuser=$(printf "%s\n" $users | head -n1) +firstuser=$(printf "%s\n" "$users" | head -n1) +id -u "$firstuser" &>/dev/null || errexit "Expected user $firstuser doesn't exist" # HACK: Symlink the Bosco key to the location expected by # bosco_cluster so it doesn't go and try to generate a new one root_ssh_dir=/root/.ssh/ mkdir -p $root_ssh_dir chmod 700 $root_ssh_dir -install -o root -g root -m 0600 "$(get_bosco_key "firstuser")" $root_ssh_dir/bosco_key.rsa +install -o root -g root -m 0600 "$(get_bosco_key "$firstuser")" $root_ssh_dir/bosco_key.rsa cat < /etc/ssh/ssh_config Host $remote_fqdn + User $firstuser Port $remote_port - IdentityFile "$(get_bosco_key "root")" + IdentityFile "$(get_bosco_key "$firstuser")" ControlMaster auto ControlPath /tmp/cm-%i-%r@%h:%p ControlPersist 15m From 810ab39769dc79ffb9e54db0c931119cc28bc792 Mon Sep 17 00:00:00 2001 From: Matyas Selmeci Date: Thu, 3 Jun 2021 17:53:09 -0500 Subject: [PATCH 6/8] Check for existence of $firstuser's key --- hosted-ce/30-remote-site-setup.sh | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/hosted-ce/30-remote-site-setup.sh b/hosted-ce/30-remote-site-setup.sh index b164b9c..82f15f3 100755 --- a/hosted-ce/30-remote-site-setup.sh +++ b/hosted-ce/30-remote-site-setup.sh @@ -123,22 +123,26 @@ REMOTE_HOST_KEY=`ssh-keyscan -p "$remote_port" "$remote_fqdn"` users=$(get_mapped_users) [[ -n $users ]] || errexit "Did not find any user mappings" + # Use the first user for things we only need once firstuser=$(printf "%s\n" "$users" | head -n1) id -u "$firstuser" &>/dev/null || errexit "Expected user $firstuser doesn't exist" +firstuser_key=$(get_bosco_key "$firstuser") +[[ -f $firstuser_key ]] || errexit "Failed to get SSH key for $firstuser" + # HACK: Symlink the Bosco key to the location expected by # bosco_cluster so it doesn't go and try to generate a new one root_ssh_dir=/root/.ssh/ mkdir -p $root_ssh_dir chmod 700 $root_ssh_dir -install -o root -g root -m 0600 "$(get_bosco_key "$firstuser")" $root_ssh_dir/bosco_key.rsa +install -o root -g root -m 0600 "$firstuser_key" $root_ssh_dir/bosco_key.rsa cat < /etc/ssh/ssh_config Host $remote_fqdn User $firstuser Port $remote_port - IdentityFile "$(get_bosco_key "$firstuser")" + IdentityFile $firstuser_key ControlMaster auto ControlPath /tmp/cm-%i-%r@%h:%p ControlPersist 15m From 76c41ecbda8d124f0361d2588de0cfc5bcf5de7c Mon Sep 17 00:00:00 2001 From: Matyas Selmeci Date: Tue, 8 Jun 2021 12:19:23 -0500 Subject: [PATCH 7/8] Don't set default user and key in /etc/ssh/ssh_config; root shouldn't need it and non-root users shouldn't fall back to it --- hosted-ce/30-remote-site-setup.sh | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/hosted-ce/30-remote-site-setup.sh b/hosted-ce/30-remote-site-setup.sh index 82f15f3..7c33b99 100755 --- a/hosted-ce/30-remote-site-setup.sh +++ b/hosted-ce/30-remote-site-setup.sh @@ -131,7 +131,7 @@ id -u "$firstuser" &>/dev/null || errexit "Expected user $firstuser doesn't exis firstuser_key=$(get_bosco_key "$firstuser") [[ -f $firstuser_key ]] || errexit "Failed to get SSH key for $firstuser" -# HACK: Symlink the Bosco key to the location expected by +# HACK: Copy the Bosco key to the location expected by # bosco_cluster so it doesn't go and try to generate a new one root_ssh_dir=/root/.ssh/ mkdir -p $root_ssh_dir @@ -140,9 +140,7 @@ install -o root -g root -m 0600 "$firstuser_key" $root_ssh_dir/bosco_key.rsa cat < /etc/ssh/ssh_config Host $remote_fqdn - User $firstuser Port $remote_port - IdentityFile $firstuser_key ControlMaster auto ControlPath /tmp/cm-%i-%r@%h:%p ControlPersist 15m From 2a6661d3d3e05c95423052b94df55682640afd90 Mon Sep 17 00:00:00 2001 From: Matyas Selmeci Date: Tue, 8 Jun 2021 14:25:54 -0500 Subject: [PATCH 8/8] Run bosco_cluster as the local user instead of root --- hosted-ce/30-remote-site-setup.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hosted-ce/30-remote-site-setup.sh b/hosted-ce/30-remote-site-setup.sh index 7c33b99..1058a93 100755 --- a/hosted-ce/30-remote-site-setup.sh +++ b/hosted-ce/30-remote-site-setup.sh @@ -207,7 +207,7 @@ for ruser in $users; do echo "Installing remote Bosco installation for ${ruser}@$remote_fqdn" [[ $SKIP_WN_INSTALL == 'no' ]] && setup_endpoints_ini "${remote_os_ver%%.*}" # $REMOTE_BATCH needs to be specified in the environment - bosco_cluster "${bosco_cluster_opts[@]}" -a "${ruser}@$remote_fqdn" "$REMOTE_BATCH" + sudo -u $ruser bosco_cluster "${bosco_cluster_opts[@]}" -a "${ruser}@$remote_fqdn" "$REMOTE_BATCH" echo "Installing environment files for $ruser@$remote_fqdn..." # Copy over environment files to allow for dynamic WN variables (SOFTWARE-4117)