diff --git a/README.rst b/README.rst index 6e741b3..55c158c 100644 --- a/README.rst +++ b/README.rst @@ -2,27 +2,16 @@ celery for puppet ================= This installs and configures `Celery`_. -This is a `puppet`_ module for using Python's `pip`_. Puppet has a -built-in pip provider, but it's implementation leaves out a few pieces: - -* No ability to install from requirements file. -* No ability to add extra arguments -* No support for using mirrors or specifying alternate indexes. - -This module fixes this. - - Usage ----- Make sure this module is available by adding this repository's contents in a directory called ``celery`` inside your Puppet's ``moduledir``. -It also requires the `puppet-pip`_ module as well. +It also requires the `puppet-python`_ module as well. Bootstrapping RabbitMQ """""""""""""""""""""" -If you need to bootstrap RabbitMQ (note that this requires that you have -`this version`_ of `puppetlabs-rabbitmq`_ to run on RabbitMQ 2.6):: +If you need to bootstrap RabbitMQ :: class { "celery::rabbitmq": } @@ -57,9 +46,10 @@ Configuration .. _Celery: http://celeryproject.org/ +.. _puppet-python: https://github.com/stankevich/puppet-python .. _distribute: http://packages.python.org/distribute/ .. _pip: http://www.pip-installer.org/ .. _puppet: http://puppetlabs.com/ .. _puppet-pip: https://github.com/armstrong/puppet-pip .. _puppetlabs-rabbitmq: https://github.com/puppetlabs/puppetlabs-rabbitmq/ -.. _this version: https://github.com/puppetlabs/puppetlabs-rabbitmq/pull/8 \ No newline at end of file +.. _this version: https://github.com/puppetlabs/puppetlabs-rabbitmq/pull/8 diff --git a/manifests/init.pp b/manifests/init.pp index 83cbdf6..4d7f86b 100644 --- a/manifests/init.pp +++ b/manifests/init.pp @@ -1,19 +1,18 @@ - - class celery::rabbitmq($user="some_user", $vhost="some_vhost", $password="CHANGEME") { - class { 'rabbitmq::repo::apt': - pin => 900, - before => Class['rabbitmq::server'] - } - - class { 'rabbitmq::server': - delete_guest_user => true, + # rabbitmq module needs this + package {'curl': + ensure => present, + } -> + class { '::rabbitmq': + package_apt_pin => 900, + # use this for the time being remove once working + delete_guest_user => false, } - rabbitmq_user { "$user": + rabbitmq_user { $user: admin => true, password => $password, provider => 'rabbitmqctl', @@ -32,27 +31,26 @@ } } -class celery::server($requirements="/tmp/celery-requirements.txt", +class celery::server($venv="system-wide", + $proroot="", + $venvowner="root", + $requirements="/tmp/celery-requirements.txt", $requirements_template="celery/requirements.txt", $initd_template="celery/init.d.sh", - $config_template="celery/celeryconfig.py", - $defaults_template="celery/defaults.sh", + $defaults_template="celery/celery_defaults.sh.erb", $broker_user="some_user", $broker_vhost="some_vhost", $broker_password="CHANGEME", $broker_host="localhost", - $broker_port="5672") { + $broker_port="5672", + $user="celery", + $usergroup="celery") { file { $requirements: ensure => "present", content => template($requirements_template), } - pip::install {"celery": - requirements => $requirements, - require => [Exec["pip::bootstrapped"], File[$requirements],], - } - file { "/etc/default/celeryd": ensure => "present", content => template($defaults_template), @@ -64,9 +62,10 @@ mode => "0755", } - user { "celery": + user { $user: ensure => "present", - } + groups => $usergroup + } -> file { "/var/celery": ensure => "directory", @@ -74,12 +73,6 @@ require => User["celery"], } - file { "/var/celery/celeryconfig.py": - ensure => "present", - content => template($config_template), - require => File["/var/celery"], - } - file { "/var/log/celery": ensure => "directory", owner => "celery", @@ -90,36 +83,18 @@ owner => "celery", } + python::requirements { $requirements: + virtualenv => $venv, + owner => $venvowner, + group => $venvowner, + } -> service { "celeryd": + hasrestart => true, ensure => "running", - require => [File["/var/celery/celeryconfig.py"], - File["/etc/init.d/celeryd"], - Exec["pip-celery"], + require => [File["/etc/init.d/celeryd"], + File["/etc/default/celeryd"], File["/var/log/celery"], File["/var/run/celery"], Class["rabbitmq::service"], ], } } - -class celery::django($requirements="/tmp/celery-django-requirements.txt", - $requirements_template="celery/django-requirements.txt", - $initd_template="celery/init.d.sh", - $config_template="celery/celeryconfig.py", - $defaults_template="celery/defaults.sh", - $broker_user="some_user", - $broker_vhost="some_vhost", - $broker_password="CHANGEME", - $broker_host="localhost", - $broker_port="5672") { - - file { $requirements: - ensure => "present", - content => template($requirements_template), - } - - pip::install {"celery": - requirements => $requirements, - require => [Exec["pip::bootstrapped"], File[$requirements],], - } - -} \ No newline at end of file diff --git a/templates/celery_defaults.sh.erb b/templates/celery_defaults.sh.erb new file mode 100644 index 0000000..208d064 --- /dev/null +++ b/templates/celery_defaults.sh.erb @@ -0,0 +1,25 @@ +# changed and simplfied for celery version 3.1 and larger + +# Name of nodes to start, here we have a single node +CELERYD_NODES="w1" +# or we could have three nodes: + +# Where to chdir at start. +CELERYD_CHDIR="<%= @proroot %>" + +# Python interpreter from environment. +ENV_PYTHON="<%= @venv %>/bin/python" + +# Use local env copy of celery +CELERY_BIN="<%= @venv %>/bin/celery" + +# Extra arguments to celeryd +CELERYD_OPTS="--concurrency=8" #<% if @django_name != '' %> -A <%= @django_name%><% end %>" + +# %n will be replaced with the nodename. +CELERYD_LOG_FILE="/var/log/celery/%n.log" +CELERYD_PID_FILE="/var/run/celery/%n.pid" + +# Workers should run as an unprivileged user. +CELERYD_USER="celery" +CELERYD_GROUP="celery" diff --git a/templates/celeryconfig.py b/templates/celeryconfig.py index f1e705c..57b9362 100644 --- a/templates/celeryconfig.py +++ b/templates/celeryconfig.py @@ -1,5 +1,5 @@ -BROKER_HOST = "<%= broker_host %>" -BROKER_PORT = <%= broker_port %> -BROKER_USER = "<%= broker_user %>" -BROKER_PASSWORD = "<%= broker_password %>" -BROKER_VHOST = "<%= broker_vhost %>" \ No newline at end of file +BROKER_HOST = "<%= @broker_host %>" +BROKER_PORT = <%= @broker_port %> +BROKER_USER = "<%= @broker_user %>" +BROKER_PASSWORD = "<%= @broker_password %>" +BROKER_VHOST = "<%= @broker_vhost %>" diff --git a/templates/defaults.sh b/templates/defaults.sh index 3cc5642..7391428 100644 --- a/templates/defaults.sh +++ b/templates/defaults.sh @@ -19,4 +19,4 @@ CELERYD_PID_FILE="/var/run/celery/%n.pid" # Workers should run as an unprivileged user. CELERYD_USER="celery" -CELERYD_GROUP="celery" \ No newline at end of file +CELERYD_GROUP="celery" diff --git a/templates/django-requirements.txt b/templates/django-requirements.txt deleted file mode 100644 index 39b97ba..0000000 --- a/templates/django-requirements.txt +++ /dev/null @@ -1,8 +0,0 @@ -amqplib==1.0.2 -anyjson==0.3.1 -celery==2.3.3 -django-celery==2.3.3 -django-picklefield==0.1.9 -kombu==1.4.1 -pyparsing==1.5.6 -python-dateutil==1.5 diff --git a/templates/init.d.sh b/templates/init.d.sh index ad972da..8167fa0 100644 --- a/templates/init.d.sh +++ b/templates/init.d.sh @@ -4,85 +4,9 @@ # ============================================ # # :Usage: /etc/init.d/celeryd {start|stop|force-reload|restart|try-restart|status} -# # :Configuration file: /etc/default/celeryd # -# To configure celeryd you probably need to tell it where to chdir. -# -# EXAMPLE CONFIGURATION -# ===================== -# -# this is an example configuration for a Python project: -# -# /etc/default/celeryd: -# -# # List of nodes to start -# CELERYD_NODES="worker1 worker2 worker3"k -# # ... can also be a number of workers -# CELERYD_NODES=3 -# -# # Where to chdir at start. -# CELERYD_CHDIR="/opt/Myproject/" -# -# # Extra arguments to celeryd -# CELERYD_OPTS="--time-limit=300" -# -# # Name of the celery config module.# -# CELERY_CONFIG_MODULE="celeryconfig" -# -# EXAMPLE DJANGO CONFIGURATION -# ============================ -# -# # Where the Django project is. -# CELERYD_CHDIR="/opt/Project/" -# -# # Name of the projects settings module. -# export DJANGO_SETTINGS_MODULE="settings" -# -# # Path to celeryd -# CELERYD="/opt/Project/manage.py celeryd" -# -# AVAILABLE OPTIONS -# ================= -# -# * CELERYD_NODES -# -# A space separated list of nodes, or a number describing the number of -# nodes, to start -# -# * CELERYD_OPTS -# Additional arguments to celeryd-multi, see `celeryd-multi --help` -# and `celeryd --help` for help. -# -# * CELERYD_CHDIR -# Path to chdir at start. Default is to stay in the current directory. -# -# * CELERYD_PIDFILE -# Full path to the pidfile. Default is /var/run/celeryd.pid. -# -# * CELERYD_LOGFILE -# Full path to the celeryd logfile. Default is /var/log/celeryd.log -# -# * CELERYD_LOG_LEVEL -# Log level to use for celeryd. Default is INFO. -# -# * CELERYD -# Path to the celeryd program. Default is `celeryd`. -# You can point this to an virtualenv, or even use manage.py for django. -# -# * CELERYD_USER -# User to run celeryd as. Default is current user. -# -# * CELERYD_GROUP -# Group to run celeryd as. Default is current user. - -# VARIABLE EXPANSION -# ================== -# -# The following abbreviations will be expanded -# -# * %n -> node name -# * %h -> host name +# See http://docs.celeryproject.org/en/latest/tutorials/daemonizing.html#generic-init-scripts ### BEGIN INIT INFO @@ -93,29 +17,62 @@ # Default-Stop: 0 1 6 # Short-Description: celery task worker daemon ### END INIT INFO +# +# +# To implement separate init scripts, copy this script and give it a different +# name: +# I.e., if my new application, "little-worker" needs an init, I +# should just use: +# +# cp /etc/init.d/celeryd /etc/init.d/little-worker +# +# You can then configure this by manipulating /etc/default/little-worker. +# + -#set -e +# May be a runlevel symlink (e.g. S02celeryd) +if [ -L "$0" ]; then + SCRIPT_FILE=$(readlink "$0") +else + SCRIPT_FILE="$0" +fi +SCRIPT_NAME="$(basename "$SCRIPT_FILE")" -DEFAULT_PID_FILE="/var/run/celeryd@%n.pid" -DEFAULT_LOG_FILE="/var/log/celeryd@%n.log" +DEFAULT_USER="celery" +DEFAULT_PID_FILE="/var/run/celery/%n.pid" +DEFAULT_LOG_FILE="/var/log/celery/%n.log" DEFAULT_LOG_LEVEL="INFO" DEFAULT_NODES="celery" -DEFAULT_CELERYD="-m celery.bin.celeryd_detach" +DEFAULT_CELERYD="-m celery worker --detach" -# /etc/init.d/celeryd: start and stop the celery task worker daemon. - -CELERY_DEFAULTS=${CELERY_DEFAULTS:-"/etc/default/celeryd"} +CELERY_DEFAULTS=${CELERY_DEFAULTS:-"/etc/default/${SCRIPT_NAME}"} test -f "$CELERY_DEFAULTS" && . "$CELERY_DEFAULTS" -if [ -f "/etc/default/celeryd" ]; then - . /etc/default/celeryd + +# Sets --app argument for CELERY_BIN +CELERY_APP_ARG="" +if [ ! -z "$CELERY_APP" ]; then + CELERY_APP_ARG="--app=$CELERY_APP" +fi + +CELERYD_USER=${CELERYD_USER:-$DEFAULT_USER} + +# Set CELERY_CREATE_DIRS to always create log/pid dirs. +CELERY_CREATE_DIRS=${CELERY_CREATE_DIRS:-0} +CELERY_CREATE_RUNDIR=$CELERY_CREATE_DIRS +CELERY_CREATE_LOGDIR=$CELERY_CREATE_DIRS +if [ -z "$CELERYD_PID_FILE" ]; then + CELERYD_PID_FILE="$DEFAULT_PID_FILE" + CELERY_CREATE_RUNDIR=1 +fi +if [ -z "$CELERYD_LOG_FILE" ]; then + CELERYD_LOG_FILE="$DEFAULT_LOG_FILE" + CELERY_CREATE_LOGDIR=1 fi -CELERYD_PID_FILE=${CELERYD_PID_FILE:-${CELERYD_PIDFILE:-$DEFAULT_PID_FILE}} -CELERYD_LOG_FILE=${CELERYD_LOG_FILE:-${CELERYD_LOGFILE:-$DEFAULT_LOG_FILE}} CELERYD_LOG_LEVEL=${CELERYD_LOG_LEVEL:-${CELERYD_LOGLEVEL:-$DEFAULT_LOG_LEVEL}} -CELERYD_MULTI=${CELERYD_MULTI:-"celeryd-multi"} -CELERYD=${CELERYD:-$DEFAULT_CELERYD} +CELERY_BIN=${CELERY_BIN:-"celery"} +CELERYD_MULTI=${CELERYD_MULTI:-"$CELERY_BIN multi"} CELERYD_NODES=${CELERYD_NODES:-$DEFAULT_NODES} export CELERY_LOADER @@ -124,64 +81,194 @@ if [ -n "$2" ]; then CELERYD_OPTS="$CELERYD_OPTS $2" fi -# Extra start-stop-daemon options, like user/group. -if [ -n "$CELERYD_USER" ]; then - DAEMON_OPTS="$DAEMON_OPTS --uid=$CELERYD_USER" -fi -if [ -n "$CELERYD_GROUP" ]; then - DAEMON_OPTS="$DAEMON_OPTS --gid=$CELERYD_GROUP" -fi +CELERYD_LOG_DIR=`dirname $CELERYD_LOG_FILE` +CELERYD_PID_DIR=`dirname $CELERYD_PID_FILE` +# Extra start-stop-daemon options, like user/group. if [ -n "$CELERYD_CHDIR" ]; then - DAEMON_OPTS="$DAEMON_OPTS --workdir=\"$CELERYD_CHDIR\"" + DAEMON_OPTS="$DAEMON_OPTS --workdir=$CELERYD_CHDIR" fi check_dev_null() { if [ ! -c /dev/null ]; then echo "/dev/null is not a character device!" - exit 1 + exit 75 # EX_TEMPFAIL + fi +} + + +maybe_die() { + if [ $? -ne 0 ]; then + echo "Exiting: $* (errno $?)" + exit 77 # EX_NOPERM fi } +create_default_dir() { + if [ ! -d "$1" ]; then + echo "- Creating default directory: '$1'" + mkdir -p "$1" + maybe_die "Couldn't create directory $1" + echo "- Changing permissions of '$1' to 02755" + chmod 02755 "$1" + maybe_die "Couldn't change permissions for $1" + if [ -n "$CELERYD_USER" ]; then + echo "- Changing owner of '$1' to '$CELERYD_USER'" + chown "$CELERYD_USER" "$1" + maybe_die "Couldn't change owner of $1" + fi + if [ -n "$CELERYD_GROUP" ]; then + echo "- Changing group of '$1' to '$CELERYD_GROUP'" + chgrp "$CELERYD_GROUP" "$1" + maybe_die "Couldn't change group of $1" + fi + fi +} + + +check_paths() { + if [ $CELERY_CREATE_LOGDIR -eq 1 ]; then + create_default_dir "$CELERYD_LOG_DIR" + fi + if [ $CELERY_CREATE_RUNDIR -eq 1 ]; then + create_default_dir "$CELERYD_PID_DIR" + fi +} + +create_paths() { + create_default_dir "$CELERYD_LOG_DIR" + create_default_dir "$CELERYD_PID_DIR" +} export PATH="${PATH:+$PATH:}/usr/sbin:/sbin" -stop_workers () { - $CELERYD_MULTI stop $CELERYD_NODES --pidfile="$CELERYD_PID_FILE" +_get_pids() { + found_pids=0 + my_exitcode=0 + + for pid_file in "$CELERYD_PID_DIR"/*.pid; do + local pid=`cat "$pid_file"` + local cleaned_pid=`echo "$pid" | sed -e 's/[^0-9]//g'` + if [ -z "$pid" ] || [ "$cleaned_pid" != "$pid" ]; then + echo "bad pid file ($pid_file)" + one_failed=true + my_exitcode=1 + else + found_pids=1 + echo "$pid" + fi + + if [ $found_pids -eq 0 ]; then + echo "${SCRIPT_NAME}: All nodes down" + exit $my_exitcode + fi + done +} + + +_chuid () { + su "$CELERYD_USER" -c "$CELERYD_MULTI $*" } start_workers () { - $CELERYD_MULTI start $CELERYD_NODES $DAEMON_OPTS \ - --pidfile="$CELERYD_PID_FILE" \ - --logfile="$CELERYD_LOG_FILE" \ - --loglevel="$CELERYD_LOG_LEVEL" \ - --cmd="$CELERYD" \ - $CELERYD_OPTS + if [ ! -z "$CELERYD_ULIMIT" ]; then + ulimit $CELERYD_ULIMIT + fi + _chuid $* start $CELERYD_NODES $DAEMON_OPTS \ + --pidfile="$CELERYD_PID_FILE" \ + --logfile="$CELERYD_LOG_FILE" \ + --loglevel="$CELERYD_LOG_LEVEL" \ + $CELERY_APP_ARG \ + $CELERYD_OPTS +} + + +dryrun () { + (C_FAKEFORK=1 start_workers --verbose) +} + + +stop_workers () { + _chuid stopwait $CELERYD_NODES --pidfile="$CELERYD_PID_FILE" } restart_workers () { - $CELERYD_MULTI restart $CELERYD_NODES $DAEMON_OPTS \ - --pidfile="$CELERYD_PID_FILE" \ - --logfile="$CELERYD_LOG_FILE" \ - --loglevel="$CELERYD_LOG_LEVEL" \ - --cmd="$CELERYD" \ - $CELERYD_OPTS + _chuid restart $CELERYD_NODES $DAEMON_OPTS \ + --pidfile="$CELERYD_PID_FILE" \ + --logfile="$CELERYD_LOG_FILE" \ + --loglevel="$CELERYD_LOG_LEVEL" \ + $CELERY_APP_ARG \ + $CELERYD_OPTS } +kill_workers() { + _chuid kill $CELERYD_NODES --pidfile="$CELERYD_PID_FILE" +} + + +restart_workers_graceful () { + local worker_pids= + worker_pids=`_get_pids` + [ "$one_failed" ] && exit 1 + + for worker_pid in $worker_pids; do + local failed= + kill -HUP $worker_pid 2> /dev/null || failed=true + if [ "$failed" ]; then + echo "${SCRIPT_NAME} worker (pid $worker_pid) could not be restarted" + one_failed=true + else + echo "${SCRIPT_NAME} worker (pid $worker_pid) received SIGHUP" + fi + done + + [ "$one_failed" ] && exit 1 || exit 0 +} + + +check_status () { + my_exitcode=0 + found_pids=0 + + local one_failed= + for pid_file in "$CELERYD_PID_DIR"/*.pid; do + local node=`basename "$pid_file" .pid` + local pid=`cat "$pid_file"` + local cleaned_pid=`echo "$pid" | sed -e 's/[^0-9]//g'` + if [ -z "$pid" ] || [ "$cleaned_pid" != "$pid" ]; then + echo "bad pid file ($pid_file)" + one_failed=true + else + local failed= + kill -0 $pid 2> /dev/null || failed=true + if [ "$failed" ]; then + echo "${SCRIPT_NAME} (node $node) (pid $pid) is stopped, but pid file exists!" + one_failed=true + else + echo "${SCRIPT_NAME} (node $node) (pid $pid) is running..." + fi + fi + done + + [ "$one_failed" ] && exit 1 || exit 0 +} + case "$1" in start) check_dev_null + check_paths start_workers ;; stop) check_dev_null + check_paths stop_workers ;; @@ -190,23 +277,50 @@ case "$1" in ;; status) - celeryctl status + check_status ;; restart) check_dev_null + check_paths restart_workers ;; + graceful) + check_dev_null + restart_workers_graceful + ;; + + kill) + check_dev_null + kill_workers + ;; + + dryrun) + check_dev_null + dryrun + ;; + try-restart) check_dev_null + check_paths restart_workers ;; + create-paths) + check_dev_null + create_paths + ;; + + check-paths) + check_dev_null + check_paths + ;; + *) - echo "Usage: /etc/init.d/celeryd {start|stop|restart|try-restart|kill}" - exit 1 + echo "Usage: /etc/init.d/${SCRIPT_NAME} {start|stop|restart|graceful|kill|dryrun|create-paths}" + exit 64 # EX_USAGE ;; esac -exit 0 \ No newline at end of file +exit 0 diff --git a/templates/requirements.txt b/templates/requirements.txt index 7fc9b00..6367f8a 100644 --- a/templates/requirements.txt +++ b/templates/requirements.txt @@ -1,6 +1 @@ -amqplib==1.0.0 -anyjson==0.3.1 -celery==2.3.3 -kombu==1.3.5 -pyparsing==1.5.6 -python-dateutil==1.5 \ No newline at end of file +celery==3.1.7