From 7803cb7073efa37cb34607a972349dd7367b762f Mon Sep 17 00:00:00 2001 From: Hans Huebner Date: Sat, 5 Apr 2014 10:45:09 +0000 Subject: [PATCH 01/12] allow dynamic changes to named throttle rules --- src/vmod_throttle.c | 513 +++++++++++++++++++++++--------------------- 1 file changed, 267 insertions(+), 246 deletions(-) diff --git a/src/vmod_throttle.c b/src/vmod_throttle.c index b2fb545..1ccc43a 100644 --- a/src/vmod_throttle.c +++ b/src/vmod_throttle.c @@ -1,5 +1,6 @@ #include #include +#include #include "vrt.h" #include "bin/varnishd/cache.h" @@ -31,7 +32,9 @@ struct vmodth_call_win { //Call set, identified by a key struct vmodth_calls { //Def: Key - char* key; + const char* key; + //Def: Unparsed Limit windows + const char* window_limits; //Def: Limit windows struct vmodth_call_win* wins; //Def: Nb of windows @@ -60,10 +63,41 @@ pthread_rwlock_t vmodth_rwlock; #define LOCK_WRITE() assert(pthread_rwlock_wrlock(&vmodth_rwlock) == 0); #define UNLOCK() pthread_rwlock_unlock(&vmodth_rwlock); +// Private: Free one calls structure including all dependents +void +_vmod_free_calls(struct vmodth_calls *calls) { + struct vmodth_call *call = calls->last; + while(call) { + struct vmodth_call *prev = call->prev; + free(call); + call = prev; + } + free((void*) calls->key); + free((void*) calls->window_limits); + free(calls->wins); + free(calls); +} + +// Private: Free all memory +void +_vmod_free_all(void* data) { + //Seek and destroy. Mwahahahahh + struct vmodth_priv *priv = ((struct vmodth_priv*)data); + + for(int i = 0; i < 4096; i++) { + struct vmodth_calls *calls = priv->hashmap[i]; + while(calls) { + struct vmodth_calls *next = calls->next; + _vmod_free_calls(calls); + calls = next; + } + } + free(data); +} // Private: the djb2 string hash algorithm unsigned long -_vmod_hash(unsigned char *str) { +_vmod_hash(const unsigned char *str) { unsigned long hash = 5381; int c; @@ -76,7 +110,7 @@ _vmod_hash(unsigned char *str) { // Private: Parse a window limit list, return the current window limit, put the cursor at // the beginning of the next one. // A result window with a length of 0 means it failed to parse. -struct vmodth_call_win _vmod_parse_win(char** wins) { +struct vmodth_call_win _vmod_parse_win(const char** wins) { struct vmodth_call_win result; int parsed_max = 0; int parsed_length = 0; @@ -150,247 +184,234 @@ struct vmodth_call_win _vmod_parse_win(char** wins) { } // Private: Fetch or create the call set from the given key -struct vmodth_calls* _vmod_get_call_set_from_key(struct vmodth_priv* priv, char* key, int create_if_not_existing, char* window_limits) { - struct vmodth_calls* result = NULL; - int hash_key; - - //Get the hash key from the key - hash_key = _vmod_hash(key) & 0xfff; - - //Search in the hash bucket - struct vmodth_calls* cur = priv->hashmap[hash_key]; - while(cur) { - if(strcmp(cur->key, key) == 0) { - result = cur; - break; - } - cur = cur->next; - } - - //If not found, and we are allowed to create, we have to create this new call set - if(!result && create_if_not_existing) { - //First we do the parsing of the windows. If it fails, we will stop. - struct vmodth_call_win* parsed_wins = NULL; - int parsed_win_count = 0; - struct vmodth_call_win parsed_win; - - //We parse windows until either we reached the end of string, or a parsing failed. - while(*window_limits) { - //Parse a window - parsed_win = _vmod_parse_win(&window_limits); - - if(!parsed_win.length) { - //Parsing failed. Quit. - break; - } - - //Now let's copy the result in the window list - parsed_win_count++; - if(parsed_win_count == 1) { - parsed_wins = malloc(sizeof(struct vmodth_call_win)); - } - else { - parsed_wins = realloc(parsed_wins, sizeof(struct vmodth_call_win) * parsed_win_count); - } - memcpy(parsed_wins + (parsed_win_count - 1), &parsed_win, sizeof(struct vmodth_call_win)); - } - - //If we failed to parse any window, return NULL - if(parsed_win_count == 0) { - return result; - } - - //Create the actual vmodth_calls structure - result = malloc(sizeof(struct vmodth_calls)); - AN(result); - result->key = malloc(strlen(key) + 1); - strncpy(result->key, key, strlen(key) + 1); - result->first = NULL; - result->last = NULL; - result->nb_calls = 0; - result->nb_wins = parsed_win_count; - result->wins = parsed_wins; - - //Place it, in first position - result->next = priv->hashmap[hash_key]; - priv->hashmap[hash_key] = result; - } - - return result; -} - -// Private: Update the window counter (e.g. the last minute calls counter) -void -_vmod_update_window_counter(struct vmodth_call_win* call_win, double now) { - while(call_win->last_call && call_win->last_call->time < now - call_win->length) { - call_win->last_call = call_win->last_call->prev; - call_win->nb_calls--; - } -} - -// Private: Return the amount of time to wait to be allowed in this time window -double -_vmod_get_time_wait_for_window_compliance(struct vmodth_call_win* call_win, double now) { - double result = 0.0; - - if(call_win->nb_calls >= call_win->max_calls) { - result = call_win->length - (now - call_win->last_call->time); - } - - return result; -} - -// Private: Update the window counter (e.g. the last minute calls counter) -void -_vmod_increment_window_counter(struct vmodth_call* new_call, struct vmodth_call_win* call_win) { - call_win->nb_calls++; - if(call_win->last_call == NULL) { - call_win->last_call = new_call; - } -} - -// Private: Remove older entries -void -_vmod_remove_older_entries(struct vmodth_calls* calls, double now) { - struct vmodth_call *prev; - int max_win_max_calls = 0; - int max_win_length = 0; - - //Get the biggest of the max nb calls of the different time windows, and the - //biggest window length - for(int i = 0; i < calls->nb_wins; i++) { - if(calls->wins[i].max_calls > max_win_max_calls) { - max_win_max_calls = calls->wins[i].max_calls; - } - if(calls->wins[i].length > max_win_length) { - max_win_length = calls->wins[i].length; - } - } - - while(calls->last && - (calls->nb_calls > max_win_max_calls || calls->last->time < now - max_win_length)) { - prev = calls->last->prev; - free(calls->last); - calls->last = prev; - if(calls->last) { - calls->last->next = NULL; - } - calls->nb_calls--; - } -} - -// Private: Garbage collector -void -_vmod_garbage_collector(struct vmodth_priv* priv, int hashmap_slot, double now) { - struct vmodth_calls *calls; - struct vmodth_calls *prev_calls; - struct vmodth_calls *tmp; - - prev_calls = NULL; - calls = priv->hashmap[hashmap_slot]; - //Let's look at each vmodth_calls: remove older vmodth_call, and if then empty, - //remove the vmodth_calls itself. - while(calls) { - //First update the windows counters and pointers - for(int i = 0; i < calls->nb_wins; i++) { - _vmod_update_window_counter(&calls->wins[i], now); - } - - //Then update the list of vmodth_call - _vmod_remove_older_entries(calls, now); - - //Finally, if there is no longer any vmodth_call, delete this vmodth_calls, and goto next - if(calls->last == NULL) { - if(prev_calls == NULL) { - priv->hashmap[hashmap_slot] = calls->next; - } - else { - prev_calls->next = calls->next; - } - tmp = calls->next; - free(calls); - calls = tmp; - } - //Else let's just move on to the next one - else { - prev_calls = calls; - calls = calls->next; - } - } -} - -// Private: Free all memory -void -_vmod_free_all(void* data) { - struct vmodth_priv *priv; - struct vmodth_calls *calls; - struct vmodth_call *call; - - //Seek and destroy. Mwahahahahh - priv = ((struct vmodth_priv*)data); - for(int i = 0; i < 4096; i++) { - calls = priv->hashmap[i]; - while(calls) { - //Free call entries - call = calls->last; - while(call) { - calls->last = call->prev; - free(call); - call = calls->last; - } - - //Free wins - free(calls->wins); - - //Free calls itself - priv->hashmap[i] = calls->next; - free(calls); - calls = priv->hashmap[i]; - } - } - free(data); -} - -// Public: Vmod init function, initialize the data structure -int -init_function(struct vmod_priv *pc, const struct VCL_conf *conf) { - struct vmodth_priv *priv; - - //Init the rwlock - pthread_rwlock_init(&vmodth_rwlock, NULL); - - //Initialize the data structure - LOCK_WRITE(); - priv = ((struct vmodth_priv*)pc->priv); - if (!priv) { - pc->priv = malloc(sizeof(struct vmodth_priv)); - AN(pc->priv); - //Is this ever called? - pc->free = _vmod_free_all; - priv = ((struct vmodth_priv*)pc->priv); - memset(priv->hashmap, 0, sizeof(priv->hashmap)); - priv->total_accepted_calls = 0; - } - UNLOCK(); - - return (0); -} - -// Public: is_allowed VCL command -double -vmod_is_allowed(struct sess *sp, struct vmod_priv *pc, const char* key, const char* window_limits) { - struct vmodth_priv *priv; - struct vmodth_calls *calls; - double result = 0; - - //Our persistent data structure - priv = ((struct vmodth_priv*)pc->priv); - AN(priv); - - LOCK_WRITE(); - - //Get the call set for this given key - calls = _vmod_get_call_set_from_key(priv, key, 1, window_limits); +struct vmodth_calls* _vmod_get_call_set_from_key(struct vmodth_priv* priv, const char* key, int create_if_not_existing, const char* window_limits) { + struct vmodth_calls* result = NULL; + int hash_key; + + //Get the hash key from the key + hash_key = _vmod_hash(key) & 0xfff; + + //Search in the hash bucket + struct vmodth_calls* cur = priv->hashmap[hash_key]; + while(cur) { + if(strcmp(cur->key, key) == 0) { + result = cur; + break; + } + cur = cur->next; + } + + if (create_if_not_existing) { + + if(result) { + if(strcmp(result->window_limits, window_limits) != 0) { + //The window_limits are different from what we've had seen + //before, discard this call set and create a new one with the + //newly set spec. + syslog(LOG_DEBUG, "changed window limits for %s from %s to %s", key, result->window_limits, window_limits); + _vmod_free_calls(result); + result = NULL; + } + } else { + syslog(LOG_DEBUG, "new window limits key %s: %s", key, window_limits); + } + + //If not found, and we are allowed to create, we have to create this new call set + if(!result) { + //First we do the parsing of the windows. If it fails, we will stop. + struct vmodth_call_win* parsed_wins = NULL; + int parsed_win_count = 0; + const char *window_limits_p = window_limits; + + //We parse windows until either we reached the end of string, or a parsing failed. + while(*window_limits_p) { + //Parse a window + struct vmodth_call_win parsed_win = _vmod_parse_win(&window_limits_p); + + if(!parsed_win.length) { + //Parsing failed. Quit. + break; + } + + //Now let's copy the result in the window list + parsed_win_count++; + if(parsed_win_count == 1) { + parsed_wins = malloc(sizeof(struct vmodth_call_win)); + } + else { + parsed_wins = realloc(parsed_wins, sizeof(struct vmodth_call_win) * parsed_win_count); + } + memcpy(parsed_wins + (parsed_win_count - 1), &parsed_win, sizeof(struct vmodth_call_win)); + } + + //If we failed to parse any window, return NULL + if(parsed_win_count == 0) { + free(parsed_wins); + return result; + } + + //Create the actual vmodth_calls structure + result = malloc(sizeof(struct vmodth_calls)); + AN(result); + result->key = strdup(key); + result->window_limits = strdup(window_limits); + result->first = NULL; + result->last = NULL; + result->nb_calls = 0; + result->nb_wins = parsed_win_count; + result->wins = parsed_wins; + + //Place it, in first position + result->next = priv->hashmap[hash_key]; + priv->hashmap[hash_key] = result; + } + } + + return result; + } + + // Private: Update the window counter (e.g. the last minute calls counter) + void + _vmod_update_window_counter(struct vmodth_call_win* call_win, double now) { + while(call_win->last_call && call_win->last_call->time < now - call_win->length) { + call_win->last_call = call_win->last_call->prev; + call_win->nb_calls--; + } + } + + // Private: Return the amount of time to wait to be allowed in this time window + double + _vmod_get_time_wait_for_window_compliance(struct vmodth_call_win* call_win, double now) { + double result = 0.0; + + if(call_win->nb_calls >= call_win->max_calls) { + result = call_win->length - (now - call_win->last_call->time); + } + + return result; + } + + // Private: Update the window counter (e.g. the last minute calls counter) + void + _vmod_increment_window_counter(struct vmodth_call* new_call, struct vmodth_call_win* call_win) { + call_win->nb_calls++; + if(call_win->last_call == NULL) { + call_win->last_call = new_call; + } + } + + // Private: Remove older entries + void + _vmod_remove_older_entries(struct vmodth_calls* calls, double now) { + struct vmodth_call *prev; + int max_win_max_calls = 0; + int max_win_length = 0; + + //Get the biggest of the max nb calls of the different time windows, and the + //biggest window length + for(int i = 0; i < calls->nb_wins; i++) { + if(calls->wins[i].max_calls > max_win_max_calls) { + max_win_max_calls = calls->wins[i].max_calls; + } + if(calls->wins[i].length > max_win_length) { + max_win_length = calls->wins[i].length; + } + } + + while(calls->last && + (calls->nb_calls > max_win_max_calls || calls->last->time < now - max_win_length)) { + prev = calls->last->prev; + free(calls->last); + calls->last = prev; + if(calls->last) { + calls->last->next = NULL; + } + calls->nb_calls--; + } + } + + // Private: Garbage collector + void + _vmod_garbage_collector(struct vmodth_priv* priv, int hashmap_slot, double now) { + struct vmodth_calls *calls; + struct vmodth_calls *prev_calls; + struct vmodth_calls *tmp; + + prev_calls = NULL; + calls = priv->hashmap[hashmap_slot]; + //Let's look at each vmodth_calls: remove older vmodth_call, and if then empty, + //remove the vmodth_calls itself. + while(calls) { + //First update the windows counters and pointers + for(int i = 0; i < calls->nb_wins; i++) { + _vmod_update_window_counter(&calls->wins[i], now); + } + + //Then update the list of vmodth_call + _vmod_remove_older_entries(calls, now); + + //Finally, if there is no longer any vmodth_call, delete this vmodth_calls, and goto next + if(calls->last == NULL) { + if(prev_calls == NULL) { + priv->hashmap[hashmap_slot] = calls->next; + } + else { + prev_calls->next = calls->next; + } + tmp = calls->next; + free(calls); + calls = tmp; + } + //Else let's just move on to the next one + else { + prev_calls = calls; + calls = calls->next; + } + } + } + + // Public: Vmod init function, initialize the data structure + int + init_function(struct vmod_priv *pc, const struct VCL_conf *conf) { + struct vmodth_priv *priv; + + openlog("libvmod-throttle", LOG_PID, LOG_LOCAL0); + + //Init the rwlock + pthread_rwlock_init(&vmodth_rwlock, NULL); + + //Initialize the data structure + LOCK_WRITE(); + priv = ((struct vmodth_priv*)pc->priv); + if (!priv) { + pc->priv = malloc(sizeof(struct vmodth_priv)); + AN(pc->priv); + //Is this ever called? + pc->free = _vmod_free_all; + priv = ((struct vmodth_priv*)pc->priv); + memset(priv->hashmap, 0, sizeof(priv->hashmap)); + priv->total_accepted_calls = 0; + } + UNLOCK(); + + return (0); + } + + // Public: is_allowed VCL command + double + vmod_is_allowed(struct sess *sp, struct vmod_priv *pc, const char* key, const char* window_limits) { + struct vmodth_priv *priv; + struct vmodth_calls *calls; + double result = 0; + + //Our persistent data structure + priv = ((struct vmodth_priv*)pc->priv); + AN(priv); + + LOCK_WRITE(); + + //Get the call set for this given key + calls = _vmod_get_call_set_from_key(priv, key, 1, window_limits); //calls can be NULL if the parsing of the windows failed if(calls == NULL) { UNLOCK(); @@ -464,7 +485,7 @@ vmod_is_allowed(struct sess *sp, struct vmod_priv *pc, const char* key, const ch int vmod_remaining_calls(struct sess *sp, struct vmod_priv *pc, const char* key, const char* window_limit) { int result = -1; - char* window_limit_str; + const char* window_limit_str; struct vmodth_priv *priv; struct vmodth_calls *calls; struct vmodth_call_win win; @@ -538,4 +559,4 @@ vmod_memory_usage(struct sess *sp, struct vmod_priv *pc) { UNLOCK(); return result; -} \ No newline at end of file +} From 5ea36d6dc61d32744b1b0a220943afcb7d57f551 Mon Sep 17 00:00:00 2001 From: Hans Huebner Date: Sat, 5 Apr 2014 11:20:44 +0000 Subject: [PATCH 02/12] add logging, don't drop calls entry when changing limit --- src/vmod_throttle.c | 78 ++++++++++++++++++++++++++------------------- 1 file changed, 45 insertions(+), 33 deletions(-) diff --git a/src/vmod_throttle.c b/src/vmod_throttle.c index 1ccc43a..249f9a6 100644 --- a/src/vmod_throttle.c +++ b/src/vmod_throttle.c @@ -186,10 +186,7 @@ struct vmodth_call_win _vmod_parse_win(const char** wins) { // Private: Fetch or create the call set from the given key struct vmodth_calls* _vmod_get_call_set_from_key(struct vmodth_priv* priv, const char* key, int create_if_not_existing, const char* window_limits) { struct vmodth_calls* result = NULL; - int hash_key; - - //Get the hash key from the key - hash_key = _vmod_hash(key) & 0xfff; + int hash_key = hash_key = _vmod_hash(key) & 0xfff; //Search in the hash bucket struct vmodth_calls* cur = priv->hashmap[hash_key]; @@ -202,25 +199,12 @@ struct vmodth_calls* _vmod_get_call_set_from_key(struct vmodth_priv* priv, const } if (create_if_not_existing) { + int changed_window_limits = !result || strcmp(result->window_limits, window_limits); + struct vmodth_call_win* parsed_wins = NULL; + int parsed_win_count = 0; - if(result) { - if(strcmp(result->window_limits, window_limits) != 0) { - //The window_limits are different from what we've had seen - //before, discard this call set and create a new one with the - //newly set spec. - syslog(LOG_DEBUG, "changed window limits for %s from %s to %s", key, result->window_limits, window_limits); - _vmod_free_calls(result); - result = NULL; - } - } else { - syslog(LOG_DEBUG, "new window limits key %s: %s", key, window_limits); - } - - //If not found, and we are allowed to create, we have to create this new call set - if(!result) { - //First we do the parsing of the windows. If it fails, we will stop. - struct vmodth_call_win* parsed_wins = NULL; - int parsed_win_count = 0; + if (changed_window_limits) { + //Parse the window limits string const char *window_limits_p = window_limits; //We parse windows until either we reached the end of string, or a parsing failed. @@ -244,27 +228,46 @@ struct vmodth_calls* _vmod_get_call_set_from_key(struct vmodth_priv* priv, const memcpy(parsed_wins + (parsed_win_count - 1), &parsed_win, sizeof(struct vmodth_call_win)); } - //If we failed to parse any window, return NULL + //If we failed to parse any window, return unmodified entry (may be NULL) if(parsed_win_count == 0) { free(parsed_wins); + if (result) { + syslog(LOG_ERR, "invalid window limits string %s for %s, limits not changed from %s", window_limits, key, result->window_limits); + } else { + syslog(LOG_ERR, "invalid window limits string %s for %s, limits not set", window_limits, key); + } return result; } + } + + if(result && changed_window_limits) { + syslog(LOG_DEBUG, "changed window limits for %s from %s to %s", key, result->window_limits, window_limits); + } else if (!result) { + syslog(LOG_DEBUG, "new window limits key %s: %s", key, window_limits); + } - //Create the actual vmodth_calls structure + if(result) { + free((void*) result->window_limits); + result->nb_wins = 0; + free(result->wins); + result->wins = 0; + } else { result = malloc(sizeof(struct vmodth_calls)); AN(result); result->key = strdup(key); - result->window_limits = strdup(window_limits); + result->first = NULL; result->last = NULL; result->nb_calls = 0; - result->nb_wins = parsed_win_count; - result->wins = parsed_wins; //Place it, in first position result->next = priv->hashmap[hash_key]; priv->hashmap[hash_key] = result; } + + result->window_limits = strdup(window_limits); + result->nb_wins = parsed_win_count; + result->wins = parsed_wins; } return result; @@ -400,12 +403,10 @@ struct vmodth_calls* _vmod_get_call_set_from_key(struct vmodth_priv* priv, const // Public: is_allowed VCL command double vmod_is_allowed(struct sess *sp, struct vmod_priv *pc, const char* key, const char* window_limits) { - struct vmodth_priv *priv; + struct vmodth_priv *priv = ((struct vmodth_priv*)pc->priv); struct vmodth_calls *calls; - double result = 0; + double result = 0; - //Our persistent data structure - priv = ((struct vmodth_priv*)pc->priv); AN(priv); LOCK_WRITE(); @@ -414,6 +415,7 @@ struct vmodth_calls* _vmod_get_call_set_from_key(struct vmodth_priv* priv, const calls = _vmod_get_call_set_from_key(priv, key, 1, window_limits); //calls can be NULL if the parsing of the windows failed if(calls == NULL) { + syslog(LOG_ERR, "could not get create set for key %s", key); UNLOCK(); return -1.0; } @@ -481,7 +483,17 @@ struct vmodth_calls* _vmod_get_call_set_from_key(struct vmodth_priv* priv, const return result; } -// Public: is_allowed VCL command +// Public: set_windows VCL command - This just sets the limit for a key +void +vmod_set_windows(struct sess *sp, struct vmod_priv *pc, const char* key, const char* window_limits) { + struct vmodth_priv *priv = ((struct vmodth_priv*)pc->priv); + + LOCK_WRITE(); + (void) _vmod_get_call_set_from_key(priv, key, 1, window_limits); + UNLOCK(); +} + +// Public: remaining_calls VCL command int vmod_remaining_calls(struct sess *sp, struct vmod_priv *pc, const char* key, const char* window_limit) { int result = -1; @@ -499,7 +511,7 @@ vmod_remaining_calls(struct sess *sp, struct vmod_priv *pc, const char* key, con //Get the call set for this given key calls = _vmod_get_call_set_from_key(priv, key, 0, NULL); if(calls == NULL) { - //calls not found, we return an error (-1) + syslog(LOG_ERR, "could not get call set from key %s", key); UNLOCK(); return result; } From c722317fa678bba7a79e5f8bae0f555addb84b04 Mon Sep 17 00:00:00 2001 From: Hans Huebner Date: Sat, 5 Apr 2014 11:20:55 +0000 Subject: [PATCH 03/12] reindent --- src/vmod_throttle.c | 570 ++++++++++++++++++++++---------------------- 1 file changed, 285 insertions(+), 285 deletions(-) diff --git a/src/vmod_throttle.c b/src/vmod_throttle.c index 249f9a6..15034a3 100644 --- a/src/vmod_throttle.c +++ b/src/vmod_throttle.c @@ -111,308 +111,308 @@ _vmod_hash(const unsigned char *str) { // the beginning of the next one. // A result window with a length of 0 means it failed to parse. struct vmodth_call_win _vmod_parse_win(const char** wins) { - struct vmodth_call_win result; - int parsed_max = 0; - int parsed_length = 0; - char c; - - //Init the result win - result.length = 0; - result.max_calls = 0; + struct vmodth_call_win result; + int parsed_max = 0; + int parsed_length = 0; + char c; + + //Init the result win + result.length = 0; + result.max_calls = 0; + result.nb_calls = 0; + result.last_call = NULL; + + if(c = *(*wins)++) { + //Get rid of any whitespaces + while(c == ' ') { + c = *(*wins)++; + } + + //Expecting a number + while(48 <= c && c <= 57) { + parsed_max = parsed_max*10 + (c - 48); + c = *(*wins)++; + } + if(parsed_max == 0) { + return result; + } + + //Get rid of any whitespaces + while(c == ' ') { + c = *(*wins)++; + } + + //Expecting the string "req/" + if(strncmp(--(*wins), "req/", 4) != 0) { + return result; + } + (*wins) += 4; + c = *(*wins)++; + + //Expecting either a number and a duration qualitifer (s,m,h,d), or a duration qualitifer + while(48 <= c && c <= 57) { + parsed_length = parsed_length*10 + (c - 48); + c = *(*wins)++; + } + if(parsed_length == 0) { + parsed_length = 1; + } + if(c == 'm') { + parsed_length *= 60; + } + else if(c == 'h') { + parsed_length *= 3600; + } + else if(c == 'd') { + parsed_length *= 24*3600; + } + + //Then expecting the next char to be either NULL or the ',' separator + c = *(*wins)++; + if(c != 0 && c != ',') { + return result; + } + + //Now let's create the new win struct + result.length = parsed_length; + result.max_calls = parsed_max; result.nb_calls = 0; result.last_call = NULL; + } - if(c = *(*wins)++) { - //Get rid of any whitespaces - while(c == ' ') { - c = *(*wins)++; - } + return result; +} - //Expecting a number - while(48 <= c && c <= 57) { - parsed_max = parsed_max*10 + (c - 48); - c = *(*wins)++; - } - if(parsed_max == 0) { - return result; - } +// Private: Fetch or create the call set from the given key +struct vmodth_calls* _vmod_get_call_set_from_key(struct vmodth_priv* priv, const char* key, int create_if_not_existing, const char* window_limits) { + struct vmodth_calls* result = NULL; + int hash_key = hash_key = _vmod_hash(key) & 0xfff; + + //Search in the hash bucket + struct vmodth_calls* cur = priv->hashmap[hash_key]; + while(cur) { + if(strcmp(cur->key, key) == 0) { + result = cur; + break; + } + cur = cur->next; + } - //Get rid of any whitespaces - while(c == ' ') { - c = *(*wins)++; + if (create_if_not_existing) { + int changed_window_limits = !result || strcmp(result->window_limits, window_limits); + struct vmodth_call_win* parsed_wins = NULL; + int parsed_win_count = 0; + + if (changed_window_limits) { + //Parse the window limits string + const char *window_limits_p = window_limits; + + //We parse windows until either we reached the end of string, or a parsing failed. + while(*window_limits_p) { + //Parse a window + struct vmodth_call_win parsed_win = _vmod_parse_win(&window_limits_p); + + if(!parsed_win.length) { + //Parsing failed. Quit. + break; + } + + //Now let's copy the result in the window list + parsed_win_count++; + if(parsed_win_count == 1) { + parsed_wins = malloc(sizeof(struct vmodth_call_win)); + } + else { + parsed_wins = realloc(parsed_wins, sizeof(struct vmodth_call_win) * parsed_win_count); + } + memcpy(parsed_wins + (parsed_win_count - 1), &parsed_win, sizeof(struct vmodth_call_win)); } - //Expecting the string "req/" - if(strncmp(--(*wins), "req/", 4) != 0) { + //If we failed to parse any window, return unmodified entry (may be NULL) + if(parsed_win_count == 0) { + free(parsed_wins); + if (result) { + syslog(LOG_ERR, "invalid window limits string %s for %s, limits not changed from %s", window_limits, key, result->window_limits); + } else { + syslog(LOG_ERR, "invalid window limits string %s for %s, limits not set", window_limits, key); + } return result; } - (*wins) += 4; - c = *(*wins)++; + } - //Expecting either a number and a duration qualitifer (s,m,h,d), or a duration qualitifer - while(48 <= c && c <= 57) { - parsed_length = parsed_length*10 + (c - 48); - c = *(*wins)++; - } - if(parsed_length == 0) { - parsed_length = 1; - } - if(c == 'm') { - parsed_length *= 60; - } - else if(c == 'h') { - parsed_length *= 3600; + if(result && changed_window_limits) { + syslog(LOG_DEBUG, "changed window limits for %s from %s to %s", key, result->window_limits, window_limits); + } else if (!result) { + syslog(LOG_DEBUG, "new window limits key %s: %s", key, window_limits); + } + + if(result) { + free((void*) result->window_limits); + result->nb_wins = 0; + free(result->wins); + result->wins = 0; + } else { + result = malloc(sizeof(struct vmodth_calls)); + AN(result); + result->key = strdup(key); + + result->first = NULL; + result->last = NULL; + result->nb_calls = 0; + + //Place it, in first position + result->next = priv->hashmap[hash_key]; + priv->hashmap[hash_key] = result; + } + + result->window_limits = strdup(window_limits); + result->nb_wins = parsed_win_count; + result->wins = parsed_wins; + } + + return result; +} + +// Private: Update the window counter (e.g. the last minute calls counter) +void +_vmod_update_window_counter(struct vmodth_call_win* call_win, double now) { + while(call_win->last_call && call_win->last_call->time < now - call_win->length) { + call_win->last_call = call_win->last_call->prev; + call_win->nb_calls--; + } +} + +// Private: Return the amount of time to wait to be allowed in this time window +double +_vmod_get_time_wait_for_window_compliance(struct vmodth_call_win* call_win, double now) { + double result = 0.0; + + if(call_win->nb_calls >= call_win->max_calls) { + result = call_win->length - (now - call_win->last_call->time); + } + + return result; +} + +// Private: Update the window counter (e.g. the last minute calls counter) +void +_vmod_increment_window_counter(struct vmodth_call* new_call, struct vmodth_call_win* call_win) { + call_win->nb_calls++; + if(call_win->last_call == NULL) { + call_win->last_call = new_call; + } +} + +// Private: Remove older entries +void +_vmod_remove_older_entries(struct vmodth_calls* calls, double now) { + struct vmodth_call *prev; + int max_win_max_calls = 0; + int max_win_length = 0; + + //Get the biggest of the max nb calls of the different time windows, and the + //biggest window length + for(int i = 0; i < calls->nb_wins; i++) { + if(calls->wins[i].max_calls > max_win_max_calls) { + max_win_max_calls = calls->wins[i].max_calls; + } + if(calls->wins[i].length > max_win_length) { + max_win_length = calls->wins[i].length; + } + } + + while(calls->last && + (calls->nb_calls > max_win_max_calls || calls->last->time < now - max_win_length)) { + prev = calls->last->prev; + free(calls->last); + calls->last = prev; + if(calls->last) { + calls->last->next = NULL; + } + calls->nb_calls--; + } +} + +// Private: Garbage collector +void +_vmod_garbage_collector(struct vmodth_priv* priv, int hashmap_slot, double now) { + struct vmodth_calls *calls; + struct vmodth_calls *prev_calls; + struct vmodth_calls *tmp; + + prev_calls = NULL; + calls = priv->hashmap[hashmap_slot]; + //Let's look at each vmodth_calls: remove older vmodth_call, and if then empty, + //remove the vmodth_calls itself. + while(calls) { + //First update the windows counters and pointers + for(int i = 0; i < calls->nb_wins; i++) { + _vmod_update_window_counter(&calls->wins[i], now); + } + + //Then update the list of vmodth_call + _vmod_remove_older_entries(calls, now); + + //Finally, if there is no longer any vmodth_call, delete this vmodth_calls, and goto next + if(calls->last == NULL) { + if(prev_calls == NULL) { + priv->hashmap[hashmap_slot] = calls->next; } - else if(c == 'd') { - parsed_length *= 24*3600; + else { + prev_calls->next = calls->next; } + tmp = calls->next; + free(calls); + calls = tmp; + } + //Else let's just move on to the next one + else { + prev_calls = calls; + calls = calls->next; + } + } +} - //Then expecting the next char to be either NULL or the ',' separator - c = *(*wins)++; - if(c != 0 && c != ',') { - return result; - } +// Public: Vmod init function, initialize the data structure +int +init_function(struct vmod_priv *pc, const struct VCL_conf *conf) { + struct vmodth_priv *priv; - //Now let's create the new win struct - result.length = parsed_length; - result.max_calls = parsed_max; - result.nb_calls = 0; - result.last_call = NULL; - } + openlog("libvmod-throttle", LOG_PID, LOG_LOCAL0); - return result; + //Init the rwlock + pthread_rwlock_init(&vmodth_rwlock, NULL); + + //Initialize the data structure + LOCK_WRITE(); + priv = ((struct vmodth_priv*)pc->priv); + if (!priv) { + pc->priv = malloc(sizeof(struct vmodth_priv)); + AN(pc->priv); + //Is this ever called? + pc->free = _vmod_free_all; + priv = ((struct vmodth_priv*)pc->priv); + memset(priv->hashmap, 0, sizeof(priv->hashmap)); + priv->total_accepted_calls = 0; + } + UNLOCK(); + + return (0); } -// Private: Fetch or create the call set from the given key -struct vmodth_calls* _vmod_get_call_set_from_key(struct vmodth_priv* priv, const char* key, int create_if_not_existing, const char* window_limits) { - struct vmodth_calls* result = NULL; - int hash_key = hash_key = _vmod_hash(key) & 0xfff; - - //Search in the hash bucket - struct vmodth_calls* cur = priv->hashmap[hash_key]; - while(cur) { - if(strcmp(cur->key, key) == 0) { - result = cur; - break; - } - cur = cur->next; - } - - if (create_if_not_existing) { - int changed_window_limits = !result || strcmp(result->window_limits, window_limits); - struct vmodth_call_win* parsed_wins = NULL; - int parsed_win_count = 0; - - if (changed_window_limits) { - //Parse the window limits string - const char *window_limits_p = window_limits; - - //We parse windows until either we reached the end of string, or a parsing failed. - while(*window_limits_p) { - //Parse a window - struct vmodth_call_win parsed_win = _vmod_parse_win(&window_limits_p); - - if(!parsed_win.length) { - //Parsing failed. Quit. - break; - } - - //Now let's copy the result in the window list - parsed_win_count++; - if(parsed_win_count == 1) { - parsed_wins = malloc(sizeof(struct vmodth_call_win)); - } - else { - parsed_wins = realloc(parsed_wins, sizeof(struct vmodth_call_win) * parsed_win_count); - } - memcpy(parsed_wins + (parsed_win_count - 1), &parsed_win, sizeof(struct vmodth_call_win)); - } - - //If we failed to parse any window, return unmodified entry (may be NULL) - if(parsed_win_count == 0) { - free(parsed_wins); - if (result) { - syslog(LOG_ERR, "invalid window limits string %s for %s, limits not changed from %s", window_limits, key, result->window_limits); - } else { - syslog(LOG_ERR, "invalid window limits string %s for %s, limits not set", window_limits, key); - } - return result; - } - } - - if(result && changed_window_limits) { - syslog(LOG_DEBUG, "changed window limits for %s from %s to %s", key, result->window_limits, window_limits); - } else if (!result) { - syslog(LOG_DEBUG, "new window limits key %s: %s", key, window_limits); - } - - if(result) { - free((void*) result->window_limits); - result->nb_wins = 0; - free(result->wins); - result->wins = 0; - } else { - result = malloc(sizeof(struct vmodth_calls)); - AN(result); - result->key = strdup(key); - - result->first = NULL; - result->last = NULL; - result->nb_calls = 0; - - //Place it, in first position - result->next = priv->hashmap[hash_key]; - priv->hashmap[hash_key] = result; - } - - result->window_limits = strdup(window_limits); - result->nb_wins = parsed_win_count; - result->wins = parsed_wins; - } - - return result; - } - - // Private: Update the window counter (e.g. the last minute calls counter) - void - _vmod_update_window_counter(struct vmodth_call_win* call_win, double now) { - while(call_win->last_call && call_win->last_call->time < now - call_win->length) { - call_win->last_call = call_win->last_call->prev; - call_win->nb_calls--; - } - } - - // Private: Return the amount of time to wait to be allowed in this time window - double - _vmod_get_time_wait_for_window_compliance(struct vmodth_call_win* call_win, double now) { - double result = 0.0; - - if(call_win->nb_calls >= call_win->max_calls) { - result = call_win->length - (now - call_win->last_call->time); - } - - return result; - } - - // Private: Update the window counter (e.g. the last minute calls counter) - void - _vmod_increment_window_counter(struct vmodth_call* new_call, struct vmodth_call_win* call_win) { - call_win->nb_calls++; - if(call_win->last_call == NULL) { - call_win->last_call = new_call; - } - } - - // Private: Remove older entries - void - _vmod_remove_older_entries(struct vmodth_calls* calls, double now) { - struct vmodth_call *prev; - int max_win_max_calls = 0; - int max_win_length = 0; - - //Get the biggest of the max nb calls of the different time windows, and the - //biggest window length - for(int i = 0; i < calls->nb_wins; i++) { - if(calls->wins[i].max_calls > max_win_max_calls) { - max_win_max_calls = calls->wins[i].max_calls; - } - if(calls->wins[i].length > max_win_length) { - max_win_length = calls->wins[i].length; - } - } - - while(calls->last && - (calls->nb_calls > max_win_max_calls || calls->last->time < now - max_win_length)) { - prev = calls->last->prev; - free(calls->last); - calls->last = prev; - if(calls->last) { - calls->last->next = NULL; - } - calls->nb_calls--; - } - } - - // Private: Garbage collector - void - _vmod_garbage_collector(struct vmodth_priv* priv, int hashmap_slot, double now) { - struct vmodth_calls *calls; - struct vmodth_calls *prev_calls; - struct vmodth_calls *tmp; - - prev_calls = NULL; - calls = priv->hashmap[hashmap_slot]; - //Let's look at each vmodth_calls: remove older vmodth_call, and if then empty, - //remove the vmodth_calls itself. - while(calls) { - //First update the windows counters and pointers - for(int i = 0; i < calls->nb_wins; i++) { - _vmod_update_window_counter(&calls->wins[i], now); - } - - //Then update the list of vmodth_call - _vmod_remove_older_entries(calls, now); - - //Finally, if there is no longer any vmodth_call, delete this vmodth_calls, and goto next - if(calls->last == NULL) { - if(prev_calls == NULL) { - priv->hashmap[hashmap_slot] = calls->next; - } - else { - prev_calls->next = calls->next; - } - tmp = calls->next; - free(calls); - calls = tmp; - } - //Else let's just move on to the next one - else { - prev_calls = calls; - calls = calls->next; - } - } - } - - // Public: Vmod init function, initialize the data structure - int - init_function(struct vmod_priv *pc, const struct VCL_conf *conf) { - struct vmodth_priv *priv; - - openlog("libvmod-throttle", LOG_PID, LOG_LOCAL0); - - //Init the rwlock - pthread_rwlock_init(&vmodth_rwlock, NULL); - - //Initialize the data structure - LOCK_WRITE(); - priv = ((struct vmodth_priv*)pc->priv); - if (!priv) { - pc->priv = malloc(sizeof(struct vmodth_priv)); - AN(pc->priv); - //Is this ever called? - pc->free = _vmod_free_all; - priv = ((struct vmodth_priv*)pc->priv); - memset(priv->hashmap, 0, sizeof(priv->hashmap)); - priv->total_accepted_calls = 0; - } - UNLOCK(); - - return (0); - } - - // Public: is_allowed VCL command - double - vmod_is_allowed(struct sess *sp, struct vmod_priv *pc, const char* key, const char* window_limits) { - struct vmodth_priv *priv = ((struct vmodth_priv*)pc->priv); - struct vmodth_calls *calls; - double result = 0; - - AN(priv); - - LOCK_WRITE(); - - //Get the call set for this given key - calls = _vmod_get_call_set_from_key(priv, key, 1, window_limits); +// Public: is_allowed VCL command +double +vmod_is_allowed(struct sess *sp, struct vmod_priv *pc, const char* key, const char* window_limits) { + struct vmodth_priv *priv = ((struct vmodth_priv*)pc->priv); + struct vmodth_calls *calls; + double result = 0; + + AN(priv); + + LOCK_WRITE(); + + //Get the call set for this given key + calls = _vmod_get_call_set_from_key(priv, key, 1, window_limits); //calls can be NULL if the parsing of the windows failed if(calls == NULL) { syslog(LOG_ERR, "could not get create set for key %s", key); From 0f64ceda912bd25f6ed8650ba4dc28d990824782 Mon Sep 17 00:00:00 2001 From: Hans Huebner Date: Sat, 5 Apr 2014 11:29:47 +0000 Subject: [PATCH 04/12] add new set_windows api --- src/vmod_throttle.vcc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/vmod_throttle.vcc b/src/vmod_throttle.vcc index cd404ef..e745ec3 100644 --- a/src/vmod_throttle.vcc +++ b/src/vmod_throttle.vcc @@ -1,5 +1,6 @@ Module throttle Init init_function Function DURATION is_allowed(PRIV_VCL, STRING, STRING) +Function VOID set_windows(PRIV_VCL, STRING, STRING) Function INT remaining_calls(PRIV_VCL, STRING, STRING) -Function INT memory_usage(PRIV_VCL) \ No newline at end of file +Function INT memory_usage(PRIV_VCL) From 5275c83f675eebfddc2034ca537ca8556a195676 Mon Sep 17 00:00:00 2001 From: Hans Huebner Date: Sat, 5 Apr 2014 13:25:34 +0000 Subject: [PATCH 05/12] move displaced code to the right position to make things work --- src/vmod_throttle.c | 58 ++++++++++++++++++++++----------------------- 1 file changed, 29 insertions(+), 29 deletions(-) diff --git a/src/vmod_throttle.c b/src/vmod_throttle.c index 15034a3..bf5f66b 100644 --- a/src/vmod_throttle.c +++ b/src/vmod_throttle.c @@ -232,42 +232,42 @@ struct vmodth_calls* _vmod_get_call_set_from_key(struct vmodth_priv* priv, const if(parsed_win_count == 0) { free(parsed_wins); if (result) { - syslog(LOG_ERR, "invalid window limits string %s for %s, limits not changed from %s", window_limits, key, result->window_limits); + syslog(LOG_ERR, "invalid window limits string \"%s\" for \"%s\", limits not changed from \"%s\"", window_limits, key, result->window_limits); } else { - syslog(LOG_ERR, "invalid window limits string %s for %s, limits not set", window_limits, key); + syslog(LOG_ERR, "invalid window limits string \"%s\" for \"%s\", limits not set", window_limits, key); } return result; } - } - if(result && changed_window_limits) { - syslog(LOG_DEBUG, "changed window limits for %s from %s to %s", key, result->window_limits, window_limits); - } else if (!result) { - syslog(LOG_DEBUG, "new window limits key %s: %s", key, window_limits); - } + if(result && changed_window_limits) { + syslog(LOG_INFO, "changed window limits for \"%s\" from \"%s\" to \"%s\"", key, result->window_limits, window_limits); + } else if(!result) { + syslog(LOG_INFO, "new window limits key \"%s\": \"%s\"", key, window_limits); + } - if(result) { - free((void*) result->window_limits); - result->nb_wins = 0; - free(result->wins); - result->wins = 0; - } else { - result = malloc(sizeof(struct vmodth_calls)); - AN(result); - result->key = strdup(key); - - result->first = NULL; - result->last = NULL; - result->nb_calls = 0; - - //Place it, in first position - result->next = priv->hashmap[hash_key]; - priv->hashmap[hash_key] = result; - } + if(result) { + free((void*) result->window_limits); + result->nb_wins = 0; + free(result->wins); + result->wins = 0; + } else { + result = malloc(sizeof(struct vmodth_calls)); + AN(result); + result->key = strdup(key); + + result->first = NULL; + result->last = NULL; + result->nb_calls = 0; + + //Place it, in first position + result->next = priv->hashmap[hash_key]; + priv->hashmap[hash_key] = result; + } - result->window_limits = strdup(window_limits); - result->nb_wins = parsed_win_count; - result->wins = parsed_wins; + result->window_limits = strdup(window_limits); + result->nb_wins = parsed_win_count; + result->wins = parsed_wins; + } } return result; From 50f1bba415f2b21f7ab83dbf41cbbf36282a2ea1 Mon Sep 17 00:00:00 2001 From: Hans Huebner Date: Sat, 5 Apr 2014 14:29:41 +0000 Subject: [PATCH 06/12] indent --- README.rst | 88 +++++++++++++++++++++++++++++++++++++++++------------- 1 file changed, 67 insertions(+), 21 deletions(-) diff --git a/README.rst b/README.rst index dba44ae..d716201 100644 --- a/README.rst +++ b/README.rst @@ -19,14 +19,25 @@ import throttle; DESCRIPTION =========== -This vmod most obvious uses are to handle denial of services by a single user (or bot) punching through the cache, or to set rate limits to API calls you provide. - -It will allow you to set rate limiting, on several different time windows, per path/IP/whatever you want. If a time window limit was reached, this vmod will return you the time to wait before this call will be authorized. -With this return information, you can handle rate limit overflow by either abruptly returning an error, or by actually waiting the necessary time if you deems it reasonable, for a smoother rate limit overflow handling. - -Please note that at the moment, there is no native way (AFAIK) to wait within Varnish (and no, using sleep() in the VCL is a *bad* idea), so you'll have to hack your way to it, e.g. redirect to a separate server that will wait for you and come back later. - -Please read carefully the "Memory management and denial of service" section below to avoid misuses that could lead to denial of services. +This vmod most obvious uses are to handle denial of services by a +single user (or bot) punching through the cache, or to set rate limits +to API calls you provide. + +It will allow you to set rate limiting, on several different time +windows, per path/IP/whatever you want. If a time window limit was +reached, this vmod will return you the time to wait before this call +will be authorized. With this return information, you can handle rate +limit overflow by either abruptly returning an error, or by actually +waiting the necessary time if you deems it reasonable, for a smoother +rate limit overflow handling. + +Please note that at the moment, there is no native way (AFAIK) to wait +within Varnish (and no, using sleep() in the VCL is a *bad* idea), so +you'll have to hack your way to it, e.g. redirect to a separate server +that will wait for you and come back later. + +Please read carefully the "Memory management and denial of service" +section below to avoid misuses that could lead to denial of services. "This is far better than any result I've seen from either HAProxy or Nginx doing layer 7 inspection on such modest [test] hardware." @@ -118,36 +129,71 @@ Example MEMORY MANAGEMENT AND DENIAL OF SERVICE ======================================= -If used incorrectly, this tool could let an attacker force Varnish to consume all available memory and crash. It would be too bad to be DoS'ed by a tool that prevents DoS! -What you need to know is that this vmod will keep in memory the time of the revelant last requests for each key you provide. And this memory is *outside* of the memory you specify to Varnish for caching. (So if you specify 4G of RAM to varnish, this vmod memory will be on top of it.) +If used incorrectly, this tool could let an attacker force Varnish to +consume all available memory and crash. It would be too bad to be +DoS'ed by a tool that prevents DoS! What you need to know is that +this vmod will keep in memory the time of the revelant last requests +for each key you provide. And this memory is *outside* of the memory +you specify to Varnish for caching. (So if you specify 4G of RAM to +varnish, this vmod memory will be on top of it.) -For a given key, the amount of necessary memory is at its maximum fixed to the maximum number of request limit you give to this key, multiplied by 16 bytes. For example:: +For a given key, the amount of necessary memory is at its maximum +fixed to the maximum number of request limit you give to this key, +multiplied by 16 bytes. For example:: if(throttle.is_allowed("pouet", "2req/s, 100req/h, 1000req/d") > 0s) -For the key "pouet", the maximum memory usage will be 1000 (the maximum number between 2, 100, and 1000) multiplied by 16 bytes = 16 kbytes. Now, with a more advanced key:: +For the key "pouet", the maximum memory usage will be 1000 (the +maximum number between 2, 100, and 1000) multiplied by 16 bytes = 16 +kbytes. Now, with a more advanced key:: if(throttle.is_allowed("ip:" + client.ip, "2req/s, 100req/3h, 1000req/d") > 0s) -We now have one key per client IP, which will each consume 16kbytes maximum. That is potentially unlimited. So what you also need to know is that the request times are kept in memory until they get older than the biggest time window: here one day (the biggest between 1s, 3 hours and 1 day). -So if you take an average of 10,000 differents IP per day, that would cost at the maximum (if every IP was making 1000 calls), 10,000 * 16kbytes = 160 mbytes. That begins to be quite a number. So one can reduce this number by keeping request limits lower. For example:: +We now have one key per client IP, which will each consume 16kbytes +maximum. That is potentially unlimited. So what you also need to know +is that the request times are kept in memory until they get older than +the biggest time window: here one day (the biggest between 1s, 3 hours +and 1 day). So if you take an average of 10,000 differents IP per +day, that would cost at the maximum (if every IP was making 1000 +calls), 10,000 * 16kbytes = 160 mbytes. That begins to be quite a +number. So one can reduce this number by keeping request limits +lower. For example:: if(throttle.is_allowed("ip:" + client.ip, "2req/s, 30req/h") > 0s) -This would reduce the maximum memory consumption, with 10,000 differents IP per day, to 10,000 * 30 * 16 = 4.8 mbytes. Much better. But wait! Now that we no longer have the 1 day window, the request times will only be kept for the new largest window, 1 hour. So if we have around 1,000 different IP per hour, that makes a maximum memory consumption of 1,000 * 30 * 16 = 480 kbytes. Muuch better! So we see that the time window sizes and lengths has a big impact on memory consumption. - -With the following example, we are theorically still open to distributed denial of service due to this vmod, but with the required number of necessary clients to consume all memory, it is much more likely that your backend services will fall and crash first. (And remember, we only use at maximum a fixed amount of memory per key, whatever the number of calls for this key). - -When we begin to be vulnerable to denial of service by a single user is when a single user can have an unlimited number of keys:: +This would reduce the maximum memory consumption, with 10,000 +differents IP per day, to 10,000 * 30 * 16 = 4.8 mbytes. Much +better. But wait! Now that we no longer have the 1 day window, the +request times will only be kept for the new largest window, 1 hour. So +if we have around 1,000 different IP per hour, that makes a maximum +memory consumption of 1,000 * 30 * 16 = 480 kbytes. Muuch better! So +we see that the time window sizes and lengths has a big impact on +memory consumption. + +With the following example, we are theorically still open to +distributed denial of service due to this vmod, but with the required +number of necessary clients to consume all memory, it is much more +likely that your backend services will fall and crash first. (And +remember, we only use at maximum a fixed amount of memory per key, +whatever the number of calls for this key). + +When we begin to be vulnerable to denial of service by a single user +is when a single user can have an unlimited number of keys:: if(throttle.is_allowed("ip:" + client.ip + ":path:" + req.url, "2req/s, 30req/h") > 0s) -With this example, you would limit the request rate per IP and per URL. A single user can thus create an unlimited number of keys, and thus consume an unlimited amount of memory, and make a denial of service by crashing varnish. So if you are in a case when you want to have different rate limits per path, it is a good idea to normalize the paths, and have a limited number of them only. For example:: +With this example, you would limit the request rate per IP and per +URL. A single user can thus create an unlimited number of keys, and +thus consume an unlimited amount of memory, and make a denial of +service by crashing varnish. So if you are in a case when you want to +have different rate limits per path, it is a good idea to normalize +the paths, and have a limited number of them only. For example:: if(req.url ~ "^/my_api_path/my_api_name") { if(throttle.is_allowed("ip:" + client.ip + ":api:api_name", "2req/s, 30req/h") > 0s) -Finally, if you want to want to track the memory usage of this throttle vmod , you can use this command:: +Finally, if you want to want to track the memory usage of this +throttle vmod , you can use this command:: if(req.url == "/my_admin_page") { set resp.http.X-throttle-memusage = throttle.memory_usage(); From 643ee419f5b357994027a78b1461ed3d3adbf485 Mon Sep 17 00:00:00 2001 From: Hans Huebner Date: Sat, 5 Apr 2014 14:36:27 +0000 Subject: [PATCH 07/12] update doc to be more realistic about the projected memory usage --- README.rst | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/README.rst b/README.rst index d716201..fcaa880 100644 --- a/README.rst +++ b/README.rst @@ -139,34 +139,35 @@ varnish, this vmod memory will be on top of it.) For a given key, the amount of necessary memory is at its maximum fixed to the maximum number of request limit you give to this key, -multiplied by 16 bytes. For example:: +multiplied by the size of one the internal data structure to keep +track of a key, currently 56 bytes on a 64 bit system. For example:: if(throttle.is_allowed("pouet", "2req/s, 100req/h, 1000req/d") > 0s) For the key "pouet", the maximum memory usage will be 1000 (the -maximum number between 2, 100, and 1000) multiplied by 16 bytes = 16 +maximum number between 2, 100, and 1000) multiplied by 56 bytes ~= 56 kbytes. Now, with a more advanced key:: if(throttle.is_allowed("ip:" + client.ip, "2req/s, 100req/3h, 1000req/d") > 0s) -We now have one key per client IP, which will each consume 16kbytes +We now have one key per client IP, which will each consume ~56kbytes maximum. That is potentially unlimited. So what you also need to know is that the request times are kept in memory until they get older than the biggest time window: here one day (the biggest between 1s, 3 hours and 1 day). So if you take an average of 10,000 differents IP per day, that would cost at the maximum (if every IP was making 1000 -calls), 10,000 * 16kbytes = 160 mbytes. That begins to be quite a +calls), 10,000 * 56kbytes ~= 560 mbytes. That begins to be quite a number. So one can reduce this number by keeping request limits lower. For example:: if(throttle.is_allowed("ip:" + client.ip, "2req/s, 30req/h") > 0s) This would reduce the maximum memory consumption, with 10,000 -differents IP per day, to 10,000 * 30 * 16 = 4.8 mbytes. Much +differents IP per day, to 10,000 * 30 * 56 ~= 16.8 mbytes. Much better. But wait! Now that we no longer have the 1 day window, the request times will only be kept for the new largest window, 1 hour. So if we have around 1,000 different IP per hour, that makes a maximum -memory consumption of 1,000 * 30 * 16 = 480 kbytes. Muuch better! So +memory consumption of 1,000 * 30 * 56 ~= 1,7 mbytes. Muuch better! So we see that the time window sizes and lengths has a big impact on memory consumption. From c2bbbf22cf8ee65f806bae1fa81ea9d0c3988b71 Mon Sep 17 00:00:00 2001 From: Hans Huebner Date: Sat, 5 Apr 2014 14:49:23 +0000 Subject: [PATCH 08/12] more indenting --- README.rst | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index fcaa880..0ef56a7 100644 --- a/README.rst +++ b/README.rst @@ -56,14 +56,23 @@ Prototype Arguments key: A unique key that will identify what you are throttling. Can be used in may ways. See the examples below. - rate_limits: A list of different rate limits you want to put on this call. The syntax is the following: "[nb_of_calls]req/[duration][durations_size], ...". Example: "3req/s, 10req/30s, 30req/5m, 100req/h". Please note that we are using the Varnish duration size identifiers: s, m (and not mn/min), h, d. WARNING: You cannot define different rate_limits for a same key. If you do, only the one given in the first call will be used. See API rate limiting example below. + rate_limits: A list of different rate limits you want to put on + this call. The syntax is the following: + "[nb_of_calls]req/[duration][durations_size], ...". Example: + "3req/s, 10req/30s, 30req/5m, 100req/h". Please note that we are + using the Varnish duration size identifiers: s, m (and not + mn/min), h, d. WARNING: You cannot define different rate_limits + for a same key. If you do, only the one given in the first call + will be used. See API rate limiting example below. Return value DURATION Description Increments the call counters. Returns 0.0 if the call was authorized, or the time to wait if one of the time window limit was reached. Example - Prevent a single user (or crazy googlebot) to make a denial of service by punching through the cache: limit MISS calls on non-static assets by IP, max 3 req/s, 10 req/30s, 30 req/5m:: + Prevent a single user (or crazy googlebot) to make a denial of + service by punching through the cache: limit MISS calls on + non-static assets by IP, max 3 req/s, 10 req/30s, 30 req/5m:: sub vcl_miss { if(req.url !~ "\.(jpg|jpeg|png|gif|ico|swf|css|js|html|htm)$") { From 64c5671ef6c13eac3e2236bf2a57561fa990ba5a Mon Sep 17 00:00:00 2001 From: Hans Huebner Date: Sat, 5 Apr 2014 14:54:15 +0000 Subject: [PATCH 09/12] document set_windows --- README.rst | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/README.rst b/README.rst index 0ef56a7..4b4a820 100644 --- a/README.rst +++ b/README.rst @@ -102,6 +102,27 @@ Example Please note: You cannot set 2 differents set of rate limits for a same key. (If you do, only one will be used, and the other will be ignored). In this example, simply add some extra text to the key to differentiate the authentificated calls from the non-authentificated ones. +set_windows +----------- + +Prototype + :: + + set_windows(STRING key, STRING rate_limits) +Arguments + key: A unique key that will identify what you are throttling. + + rate_limits: A list of different rate limits you want to put on + this call. + + These arguments have the same meaning like in is_allowed. +Return value + VOID +Description + Create a set of named windows under the given key. The rule set + is marked as being an "explicit" rule, which will enable more + logging for it. It is intended to be used from vcl_init or as + part of an internal handler to create global rules. remaining_calls --------------- From e2f8249537e0a0375c6881868048a6a6ed23fbeb Mon Sep 17 00:00:00 2001 From: Hans Huebner Date: Sat, 5 Apr 2014 15:14:16 +0000 Subject: [PATCH 10/12] implement auto/explicit rules --- src/vmod_throttle.c | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/src/vmod_throttle.c b/src/vmod_throttle.c index bf5f66b..0a76ea8 100644 --- a/src/vmod_throttle.c +++ b/src/vmod_throttle.c @@ -7,6 +7,12 @@ #include "vcc_if.h" +enum rule_type { + NO_CREATE = 0, + AUTO_RULE, + EXPLICIT_RULE +}; + //Individual call struct vmodth_call { //Call time @@ -31,6 +37,8 @@ struct vmodth_call_win { //Call set, identified by a key struct vmodth_calls { + //Def: Type of the rule + enum rule_type rule_type; //Def: Key const char* key; //Def: Unparsed Limit windows @@ -38,7 +46,7 @@ struct vmodth_calls { //Def: Limit windows struct vmodth_call_win* wins; //Def: Nb of windows - int nb_wins; + short nb_wins; //Status: Linked list, first call struct vmodth_call* first; //Status: Linked list, last call @@ -184,7 +192,7 @@ struct vmodth_call_win _vmod_parse_win(const char** wins) { } // Private: Fetch or create the call set from the given key -struct vmodth_calls* _vmod_get_call_set_from_key(struct vmodth_priv* priv, const char* key, int create_if_not_existing, const char* window_limits) { +struct vmodth_calls* _vmod_get_call_set_from_key(struct vmodth_priv* priv, const char* key, enum rule_type rule_type, const char* window_limits) { struct vmodth_calls* result = NULL; int hash_key = hash_key = _vmod_hash(key) & 0xfff; @@ -198,7 +206,7 @@ struct vmodth_calls* _vmod_get_call_set_from_key(struct vmodth_priv* priv, const cur = cur->next; } - if (create_if_not_existing) { + if (rule_type != NO_CREATE) { int changed_window_limits = !result || strcmp(result->window_limits, window_limits); struct vmodth_call_win* parsed_wins = NULL; int parsed_win_count = 0; @@ -239,9 +247,9 @@ struct vmodth_calls* _vmod_get_call_set_from_key(struct vmodth_priv* priv, const return result; } - if(result && changed_window_limits) { + if(result && (result->rule_type == EXPLICIT_RULE) && changed_window_limits) { syslog(LOG_INFO, "changed window limits for \"%s\" from \"%s\" to \"%s\"", key, result->window_limits, window_limits); - } else if(!result) { + } else if(!result && (rule_type == EXPLICIT_RULE)) { syslog(LOG_INFO, "new window limits key \"%s\": \"%s\"", key, window_limits); } @@ -254,6 +262,7 @@ struct vmodth_calls* _vmod_get_call_set_from_key(struct vmodth_priv* priv, const result = malloc(sizeof(struct vmodth_calls)); AN(result); result->key = strdup(key); + result->rule_type = rule_type; result->first = NULL; result->last = NULL; @@ -412,7 +421,7 @@ vmod_is_allowed(struct sess *sp, struct vmod_priv *pc, const char* key, const ch LOCK_WRITE(); //Get the call set for this given key - calls = _vmod_get_call_set_from_key(priv, key, 1, window_limits); + calls = _vmod_get_call_set_from_key(priv, key, AUTO_RULE, window_limits); //calls can be NULL if the parsing of the windows failed if(calls == NULL) { syslog(LOG_ERR, "could not get create set for key %s", key); @@ -489,7 +498,7 @@ vmod_set_windows(struct sess *sp, struct vmod_priv *pc, const char* key, const c struct vmodth_priv *priv = ((struct vmodth_priv*)pc->priv); LOCK_WRITE(); - (void) _vmod_get_call_set_from_key(priv, key, 1, window_limits); + (void) _vmod_get_call_set_from_key(priv, key, EXPLICIT_RULE, window_limits); UNLOCK(); } @@ -509,7 +518,7 @@ vmod_remaining_calls(struct sess *sp, struct vmod_priv *pc, const char* key, con AN(priv); //Get the call set for this given key - calls = _vmod_get_call_set_from_key(priv, key, 0, NULL); + calls = _vmod_get_call_set_from_key(priv, key, NO_CREATE, NULL); if(calls == NULL) { syslog(LOG_ERR, "could not get call set from key %s", key); UNLOCK(); From b65d4f56a83e4278789e18bf435b72d3eee7068e Mon Sep 17 00:00:00 2001 From: Hans Huebner Date: Sat, 5 Apr 2014 15:32:21 +0000 Subject: [PATCH 11/12] improv log messages --- src/vmod_throttle.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/vmod_throttle.c b/src/vmod_throttle.c index 0a76ea8..8d03f6a 100644 --- a/src/vmod_throttle.c +++ b/src/vmod_throttle.c @@ -248,9 +248,9 @@ struct vmodth_calls* _vmod_get_call_set_from_key(struct vmodth_priv* priv, const } if(result && (result->rule_type == EXPLICIT_RULE) && changed_window_limits) { - syslog(LOG_INFO, "changed window limits for \"%s\" from \"%s\" to \"%s\"", key, result->window_limits, window_limits); + syslog(LOG_INFO, "changed window limits for key \"%s\" from \"%s\" to \"%s\"", key, result->window_limits, window_limits); } else if(!result && (rule_type == EXPLICIT_RULE)) { - syslog(LOG_INFO, "new window limits key \"%s\": \"%s\"", key, window_limits); + syslog(LOG_INFO, "new window limits for key \"%s\": \"%s\"", key, window_limits); } if(result) { @@ -424,7 +424,7 @@ vmod_is_allowed(struct sess *sp, struct vmod_priv *pc, const char* key, const ch calls = _vmod_get_call_set_from_key(priv, key, AUTO_RULE, window_limits); //calls can be NULL if the parsing of the windows failed if(calls == NULL) { - syslog(LOG_ERR, "could not get create set for key %s", key); + syslog(LOG_ERR, "could not create or re-set key \"%s\"", key); UNLOCK(); return -1.0; } @@ -520,7 +520,7 @@ vmod_remaining_calls(struct sess *sp, struct vmod_priv *pc, const char* key, con //Get the call set for this given key calls = _vmod_get_call_set_from_key(priv, key, NO_CREATE, NULL); if(calls == NULL) { - syslog(LOG_ERR, "could not get call set from key %s", key); + syslog(LOG_ERR, "could not get key \"%s\"", key); UNLOCK(); return result; } From 8bc5e515adefde3d2a39950df6e2056cbbee089d Mon Sep 17 00:00:00 2001 From: Hans Huebner Date: Sat, 5 Apr 2014 17:32:58 +0000 Subject: [PATCH 12/12] update debian package building files --- debian/.gitignore | 4 ++++ debian/rules | 8 ++++---- package.sh | 11 +++++++++++ 3 files changed, 19 insertions(+), 4 deletions(-) create mode 100644 debian/.gitignore create mode 100755 package.sh diff --git a/debian/.gitignore b/debian/.gitignore new file mode 100644 index 0000000..d1c2300 --- /dev/null +++ b/debian/.gitignore @@ -0,0 +1,4 @@ +files +libvmod-throttle +*substvars +*log diff --git a/debian/rules b/debian/rules index 5694512..061b520 100755 --- a/debian/rules +++ b/debian/rules @@ -1,9 +1,9 @@ #!/usr/bin/make -f export DH_VERBOSE=1 -VARNISHSRC = $(DEBIAN_VARNISH_SRC) -VMODDIR = $(shell PKG_CONFIG_PATH="$(VARNISHSRC)" pkg-config --variable=vmoddir varnishapi) -VMOD_ABI = $(shell printf '\#include "vmod_abi.h"\nVMOD_ABI_Version' | cpp - -I$(DEBIAN_VARNISH_SRC)/include | sed '/^\#/D;s/"//g;s/\([A-Z]\)/\L\1/g;s/[^a-z0-9.]/-/g;s/varnish/varnishabi/') +VARNISHSRC ?= $(DEBIAN_VARNISH_SRC) +VMODDIR ?= $(shell PKG_CONFIG_PATH="$(VARNISHSRC)" pkg-config --variable=vmoddir varnishapi) +VMOD_ABI ?= $(shell printf '\#include "vmod_abi.h"\nVMOD_ABI_Version' | cpp - -I$(DEBIAN_VARNISH_SRC)/include | sed '/^\#/D;s/"//g;s/\([A-Z]\)/\L\1/g;s/[^a-z0-9.]/-/g;s/varnish/varnishabi/') override_dh_auto_configure: dh_auto_configure -- VMODDIR="$(VMODDIR)" VARNISHSRC="$(VARNISHSRC)" @@ -14,7 +14,7 @@ override_dh_gencontrol: if [ -n "$$DEBIAN_OVERRIDE_BINARY_VERSION" ]; then \ dh_gencontrol -- -Tdebian/substvars -v$$DEBIAN_OVERRIDE_BINARY_VERSION; \ else \ - dh_gencontrol -Tdebian/substvars; \ + dh_gencontrol -- -Tdebian/substvars; \ fi %: diff --git a/package.sh b/package.sh new file mode 100755 index 0000000..99fa377 --- /dev/null +++ b/package.sh @@ -0,0 +1,11 @@ +#!/bin/sh +set -x + +export VARNISH_VERSION=${VARNISH_VERSION:="3.0.5"} +export VARNISHSRC=../varnish-${VARNISH_VERSION} +export DEBIAN_VARNISH_SRC=$VARNISHSRC +export VMODDIR=/usr/lib/varnish/vmods + +git submodule update --init +./autogen.sh && ./configure && make +dpkg-buildpackage -uc -us -b