From 8e6882ac9c1812f7cd7a1da78ff25bbfa78f9d0d Mon Sep 17 00:00:00 2001 From: Austin Heyne Date: Fri, 18 Nov 2016 10:18:41 -0500 Subject: [PATCH 1/4] Added GeoServer Support * Added a command line parameter that is only read when cloud-local start is envoked. If the parameter -gs is provided and GEOSERVER_HOME is set then the script will start and stop GeoServer with the cloud. Note GeoServer runs in the current thread so ctrl+c is needed to shut it down. * Added documentation of -gs command. Signed-off-by: Austin Heyne --- README.md | 8 ++++++++ bin/cloud-local.sh | 5 +++++ 2 files changed, 13 insertions(+) diff --git a/README.md b/README.md index 377e1ae..edb804b 100644 --- a/README.md +++ b/README.md @@ -92,6 +92,14 @@ Port offseting moves the entire port space by a given numerical amount in order WARNING - you should stop and clean cloud-local before changing any of these parameters since they will modify the config and may prevent cloud-local from cleanly shutting down. Changing port offsets is supported by XML comments in the accumulo and hadoop config files. Removing or changing these comments (CL_port_default) will likely cause failures. +## GoeServer + +If you have the environment variable GEOSERVER_HOME set you can use this parameter to start GeoServer at the same time but running in a child thread. + + bin/cloud-local.sh start -gs + +You can then use ctrl+c to stop GeoServer, when it finishes shutting down, cloud-local will envoke it's own stop command to shutdown the rest of the cloud. + ## Maintenance The `cloud-local.sh` script provides options for maintenance. Best to stop the cloud before performing any of these tasks. Pass in the parameter `clean` to remove software (but not the tar.gz's) and data. The parameter `reconfigure` will first `clean` then `init`. diff --git a/bin/cloud-local.sh b/bin/cloud-local.sh index 592c784..ccdcfb4 100755 --- a/bin/cloud-local.sh +++ b/bin/cloud-local.sh @@ -284,6 +284,7 @@ function clear_data { function show_help { echo "Provide 1 command: (init|start|stop|reconfigure|reyarn|clean|help)" + echo "If the environment variable GEOSERVER_HOME is set then the parameter '-gs' may be used with 'start' to automatically start/stop GeoServer with the cloud." } if [ "$#" -ne 1 ]; then @@ -305,6 +306,10 @@ elif [[ $1 == 'start' ]]; then echo "Starting cloud..." start_cloud echo "Cloud Started" + if [[ $2 == '-gs' && -n "${GEOSERVER_HOME}" ]]; then + trap stop_cloud SIGINT + ($GEOSERVER_HOME/bin/startup.sh) + fi elif [[ $1 == 'stop' ]]; then echo "Stopping Cloud..." stop_cloud From 797d5d60bba0691e75da3c82972b0f0ff829538e Mon Sep 17 00:00:00 2001 From: Austin Heyne Date: Fri, 18 Nov 2016 12:43:13 -0500 Subject: [PATCH 2/4] Moved GeoServer to background * GeoServer will now start in the background and save the PID to data/geoserver/pid/geoserver.pid * GeoServer stdout is redirected to data/geoserver/log/std.out Signed-off-by: Austin Heyne --- bin/cloud-local.sh | 62 +++++++++++++++++++++++++++++++++++----------- bin/config.sh | 4 +++ 2 files changed, 51 insertions(+), 15 deletions(-) diff --git a/bin/cloud-local.sh b/bin/cloud-local.sh index ccdcfb4..430c0f0 100755 --- a/bin/cloud-local.sh +++ b/bin/cloud-local.sh @@ -3,7 +3,7 @@ # thanks accumulo for these resolutions snippets # Start: Resolve Script Directory SOURCE="${BASH_SOURCE[0]}" -while [ -h "${SOURCE}" ]; do # resolve $SOURCE until the file is no longer a symlink +while [[ -h "${SOURCE}" ]]; do # resolve $SOURCE until the file is no longer a symlink bin="$( cd -P "$( dirname "${SOURCE}" )" && pwd )" SOURCE="$(readlink "${SOURCE}")" [[ "${SOURCE}" != /* ]] && SOURCE="${bin}/${SOURCE}" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located @@ -37,7 +37,7 @@ function download_packages() { test -d ${CLOUD_HOME}/pkg || mkdir ${CLOUD_HOME}/pkg local mirror - if [ -z ${pkg_src_mirror+x} ]; then + if [[ -z ${pkg_src_mirror+x} ]]; then local mirror=$(curl 'https://www.apache.org/dyn/closer.cgi' | grep -o '[^<]*' | sed 's/<[^>]*>//g' | head -1) else local mirror=${pkg_src_mirror} @@ -179,15 +179,22 @@ function start_first_time { sleep 5 # starting accumulo - echo "starting accumulo..." + echo "Starting accumulo..." $ACCUMULO_HOME/bin/start-all.sh fi if [[ "$hbase_enable" -eq 1 ]]; then # start hbase - echo "starting hbase..." + echo "Starting hbase..." ${HBASE_HOME}/bin/start-hbase.sh fi + + # init GeoServer Support + mkdir -p "${GEOSERVER_DATA_DIR}" + mkdir "${GEOSERVER_PID_DIR}" + mkdir "${GEOSERVER_LOG_DIR}" + touch "${GEOSERVER_PID_DIR}/geoserver.pid" + touch "${GEOSERVER_LOG_DIR}/std.out" } function start_cloud { @@ -213,16 +220,26 @@ function start_cloud { hdfs dfsadmin -safemode wait if [[ "$acc_enable" -eq 1 ]]; then - # starting accumulo - echo "starting accumulo..." + # Starting accumulo + echo "Starting accumulo..." $ACCUMULO_HOME/bin/start-all.sh fi if [[ "$hbase_enable" -eq 1 ]]; then # start hbase - echo "starting hbase..." + echo "Starting hbase..." ${HBASE_HOME}/bin/start-hbase.sh fi + + if [[ "${geoserver_enabled}" -eq "1" ]]; then + echo "Starting GeoServer..." + (${GEOSERVER_HOME}/bin/startup.sh &> ${GEOSERVER_LOG_DIR}/std.out) & + GEOSERVER_PID=$! + echo "${GEOSERVER_PID}" > ${GEOSERVER_PID_DIR}/geoserver.pid + echo "GeoServer Process Started" + echo "PID: ${GEOSERVER_PID}" + echo "GeoServer Out: ${GEOSERVER_LOG_DIR}/std.out" + fi } function start_yarn { @@ -253,6 +270,17 @@ function stop_cloud { echo "Stopping zookeeper..." $ZOOKEEPER_HOME/bin/zkServer.sh stop + + if [[ "${geoserver_enabled}" -eq "1" ]]; then + echo "Stopping GeoServer..." + GEOSERVER_PID=`cat ${GEOSERVER_PID_DIR}/geoserver.pid` + if [[ -n "${GEOSERVER_PID}" ]]; then + kill -15 ${GEOSERVER_PID} + echo "TERM signal sent to process PID:${GEOSERVER_PID}" + else + echo "No GeoServer PID was saved. This script must be used to start GeoServer in order for this script to be able to stop it." + fi + fi } function stop_yarn { @@ -268,7 +296,7 @@ function clear_sw { rm -rf "${CLOUD_HOME}/kafka_${pkg_kafka_scala_ver}-${pkg_kafka_ver}" rm -rf "${CLOUD_HOME}/spark-${pkg_spark_ver}-bin-without-hadoop" rm -rf "${CLOUD_HOME}/tmp" - if [ -a "${CLOUD_HOME}/zookeeper.out" ]; then rm "${CLOUD_HOME}/zookeeper.out"; fi #hahahaha + if [[ -a "${CLOUD_HOME}/zookeeper.out" ]]; then rm "${CLOUD_HOME}/zookeeper.out"; fi #hahahaha } function clear_data { @@ -279,15 +307,23 @@ function clear_data { rm -rf ${CLOUD_HOME}/data/dfs/name/* rm -rf ${CLOUD_HOME}/data/hadoop/tmp/* rm -rf ${CLOUD_HOME}/data/hadoop/pid/* - if [ -d "${CLOUD_HOME}/data/kafka-logs" ]; then rm -rf ${CLOUD_HOME}/data/kafka-logs; fi # intentionally to clear dot files + if [[ -d "${CLOUD_HOME}/data/kafka-logs" ]]; then rm -rf ${CLOUD_HOME}/data/kafka-logs; fi # intentionally to clear dot files } function show_help { echo "Provide 1 command: (init|start|stop|reconfigure|reyarn|clean|help)" - echo "If the environment variable GEOSERVER_HOME is set then the parameter '-gs' may be used with 'start' to automatically start/stop GeoServer with the cloud." + echo "If the environment variable GEOSERVER_HOME is set then the parameter '-gs' may be used with 'start' to automatically start/stop GeoServer with the cloud." } -if [ "$#" -ne 1 ]; then +if [[ "$2" -eq "-gs" ]]; then + if [[ -n "${GEOSERVER_HOME}" && -e $GEOSERVER_HOME/bin/startup.sh ]]; then + geoserver_enabled=1 + else + echo "The environment variable GEOSERVER_HOME is not set or is not valid." + fi +fi + +if [[ "$#" -ne 1 && "${geoserver_enabled}" -ne "1" ]]; then show_help exit 1 fi @@ -306,10 +342,6 @@ elif [[ $1 == 'start' ]]; then echo "Starting cloud..." start_cloud echo "Cloud Started" - if [[ $2 == '-gs' && -n "${GEOSERVER_HOME}" ]]; then - trap stop_cloud SIGINT - ($GEOSERVER_HOME/bin/startup.sh) - fi elif [[ $1 == 'stop' ]]; then echo "Stopping Cloud..." stop_cloud diff --git a/bin/config.sh b/bin/config.sh index bef3e50..04111fb 100755 --- a/bin/config.sh +++ b/bin/config.sh @@ -66,6 +66,10 @@ function set_env_vars { export YARN_IDENT_STRING="${HADOOP_IDENT_STRING}" export SPARK_HOME="$CLOUD_HOME/spark-${pkg_spark_ver}-bin-without-hadoop" + + export GEOSERVER_DATA_DIR="${CLOUD_HOME}/data/geoserver" + export GEOSERVER_PID_DIR="${GEOSERVER_DATA_DIR}/pid" + export GEOSERVER_LOG_DIR="${GEOSERVER_DATA_DIR}/log" [[ $acc_enable -eq 1 ]] && export ACCUMULO_HOME="$CLOUD_HOME/accumulo-${pkg_accumulo_ver}" [[ $hbase_enable -eq 1 ]] && export HBASE_HOME="${CLOUD_HOME}/hbase-${pkg_hbase_ver}" From e3ba021622a3c53242c161e7efc3c4e522f96564 Mon Sep 17 00:00:00 2001 From: Austin Heyne Date: Mon, 21 Nov 2016 10:39:13 -0500 Subject: [PATCH 3/4] Added clear data confirm and clear geoserver data Signed-off-by: Austin Heyne --- bin/cloud-local.sh | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/bin/cloud-local.sh b/bin/cloud-local.sh index 430c0f0..f956014 100755 --- a/bin/cloud-local.sh +++ b/bin/cloud-local.sh @@ -300,14 +300,19 @@ function clear_sw { } function clear_data { - # TODO prompt - rm -rf ${CLOUD_HOME}/data/yarn/* - rm -rf ${CLOUD_HOME}/data/zookeeper/* - rm -rf ${CLOUD_HOME}/data/dfs/data/* - rm -rf ${CLOUD_HOME}/data/dfs/name/* - rm -rf ${CLOUD_HOME}/data/hadoop/tmp/* - rm -rf ${CLOUD_HOME}/data/hadoop/pid/* - if [[ -d "${CLOUD_HOME}/data/kafka-logs" ]]; then rm -rf ${CLOUD_HOME}/data/kafka-logs; fi # intentionally to clear dot files + read -r -p "Are you sure you want to clear data directories? [y/N] " confirm + confirm=${confirm,,} #lowercasing + if [[ $confirm =~ ^(yes|y) ]]; then + rm -rf ${CLOUD_HOME}/data/yarn/* + rm -rf ${CLOUD_HOME}/data/zookeeper/* + rm -rf ${CLOUD_HOME}/data/dfs/data/* + rm -rf ${CLOUD_HOME}/data/dfs/name/* + rm -rf ${CLOUD_HOME}/data/hadoop/tmp/* + rm -rf ${CLOUD_HOME}/data/hadoop/pid/* + rm -rf ${CLOUD_HOME}/data/geoserver/pid/* + rm -rf ${CLOUD_HOME}/data/geoserver/log/* + if [[ -d "${CLOUD_HOME}/data/kafka-logs" ]]; then rm -rf ${CLOUD_HOME}/data/kafka-logs; fi # intentionally to clear dot files + fi } function show_help { From 189b6e0e9d409fd02da1c846783d38b888681e73 Mon Sep 17 00:00:00 2001 From: Austin Heyne Date: Mon, 21 Nov 2016 11:04:01 -0500 Subject: [PATCH 4/4] Updated GeoServer Options and Docs Signed-off-by: Austin Heyne --- .gitignore | 1 + README.md | 14 +++++++++++--- bin/cloud-local.sh | 40 ++++++++++++++++++++++++++-------------- 3 files changed, 38 insertions(+), 17 deletions(-) diff --git a/.gitignore b/.gitignore index 0657775..9f77eb8 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,7 @@ data/* /zookeeper* /kafka* /spark* +/scala* /.idea /cloud-local.iml *.iml diff --git a/README.md b/README.md index edb804b..0e5b8f0 100644 --- a/README.md +++ b/README.md @@ -95,10 +95,18 @@ WARNING - you should stop and clean cloud-local before changing any of these par ## GoeServer If you have the environment variable GEOSERVER_HOME set you can use this parameter to start GeoServer at the same time but running in a child thread. - - bin/cloud-local.sh start -gs -You can then use ctrl+c to stop GeoServer, when it finishes shutting down, cloud-local will envoke it's own stop command to shutdown the rest of the cloud. + bin/cloud-local.sh start -gs + +Similarly, you can instruct cloud-local to shutdown GeoServer with the cloud using: + + bin/cloud-local.sh stop -gs + +Additionally, if you need to restart GeoServer you may use the command `regeoserver`: + + bin/cloud-local.sh regeoserver + +The GeoServer PID is stored in `$CLOUD_HOME/data/geoserver/pid/geoserver.pid` and GeoServer's stdout is redirected to `$CLOUD_HOME/data/geoserver/log/std.out`. ## Maintenance diff --git a/bin/cloud-local.sh b/bin/cloud-local.sh index f956014..9807003 100755 --- a/bin/cloud-local.sh +++ b/bin/cloud-local.sh @@ -195,6 +195,8 @@ function start_first_time { mkdir "${GEOSERVER_LOG_DIR}" touch "${GEOSERVER_PID_DIR}/geoserver.pid" touch "${GEOSERVER_LOG_DIR}/std.out" + + start_geoserver } function start_cloud { @@ -231,6 +233,15 @@ function start_cloud { ${HBASE_HOME}/bin/start-hbase.sh fi + start_geoserver +} + +function start_yarn { + $HADOOP_HOME/sbin/yarn-daemon.sh --config $HADOOP_CONF_DIR start resourcemanager + $HADOOP_HOME/sbin/yarn-daemon.sh --config $HADOOP_CONF_DIR start nodemanager +} + +function start_geoserver { if [[ "${geoserver_enabled}" -eq "1" ]]; then echo "Starting GeoServer..." (${GEOSERVER_HOME}/bin/startup.sh &> ${GEOSERVER_LOG_DIR}/std.out) & @@ -241,12 +252,6 @@ function start_cloud { echo "GeoServer Out: ${GEOSERVER_LOG_DIR}/std.out" fi } - -function start_yarn { - $HADOOP_HOME/sbin/yarn-daemon.sh --config $HADOOP_CONF_DIR start resourcemanager - $HADOOP_HOME/sbin/yarn-daemon.sh --config $HADOOP_CONF_DIR start nodemanager -} - function stop_cloud { echo "Stopping kafka..." @@ -271,23 +276,27 @@ function stop_cloud { echo "Stopping zookeeper..." $ZOOKEEPER_HOME/bin/zkServer.sh stop + stop_geoserver +} + +function stop_yarn { + $HADOOP_HOME/sbin/yarn-daemon.sh --config $HADOOP_CONF_DIR stop resourcemanager + $HADOOP_HOME/sbin/yarn-daemon.sh --config $HADOOP_CONF_DIR stop nodemanager +} + +function stop_geoserver { if [[ "${geoserver_enabled}" -eq "1" ]]; then echo "Stopping GeoServer..." GEOSERVER_PID=`cat ${GEOSERVER_PID_DIR}/geoserver.pid` if [[ -n "${GEOSERVER_PID}" ]]; then kill -15 ${GEOSERVER_PID} - echo "TERM signal sent to process PID:${GEOSERVER_PID}" - else + echo "TERM signal sent to process PID: ${GEOSERVER_PID}" + else echo "No GeoServer PID was saved. This script must be used to start GeoServer in order for this script to be able to stop it." fi fi } -function stop_yarn { - $HADOOP_HOME/sbin/yarn-daemon.sh --config $HADOOP_CONF_DIR stop resourcemanager - $HADOOP_HOME/sbin/yarn-daemon.sh --config $HADOOP_CONF_DIR stop nodemanager -} - function clear_sw { [[ "$acc_enable" -eq 1 ]] && rm -rf "${CLOUD_HOME}/accumulo-${pkg_accumulo_ver}" [[ "$hbase_enable" -eq 1 ]] && rm -rf "${CLOUD_HOME}/hbase-${pkg_hbase_ver}" @@ -316,7 +325,7 @@ function clear_data { } function show_help { - echo "Provide 1 command: (init|start|stop|reconfigure|reyarn|clean|help)" + echo "Provide 1 command: (init|start|stop|reconfigure|reyarn|regeoserver|clean|help)" echo "If the environment variable GEOSERVER_HOME is set then the parameter '-gs' may be used with 'start' to automatically start/stop GeoServer with the cloud." } @@ -356,6 +365,9 @@ elif [[ $1 == 'reyarn' ]]; then stop_yarn echo "Starting Yarn..." start_yarn +elif [[ $1 == 'regeoserver' ]]; then + stop_geoserver + start_geoserver else show_help fi