From 7bfa84b51beb55117b07eb0e0390223fbbcb0c1d Mon Sep 17 00:00:00 2001 From: michalbiesek Date: Fri, 17 Dec 2021 12:53:05 +0100 Subject: [PATCH 1/2] Parametrize for size of in-memory queue - handle flush bytes limit via environment variable `SCOPE_LOG_BUFFER_THRESHOLD` and config parameter - keep old value as default 32768 closes #189 --- conf/scope.yml | 10 ++++ src/cfg.c | 15 ++++++ src/cfg.h | 2 + src/cfgutils.c | 50 ++++++++++++------- src/ctl.c | 5 +- src/ctl.h | 2 +- src/scope_static.c | 3 ++ src/scopetypes.h | 1 + src/utils.c | 10 ++++ src/utils.h | 1 + test/cfgtest.c | 1 + test/cfgutilstest.c | 1 + test/comtest.c | 6 +-- test/ctltest.c | 9 ++-- .../fluentbit/services/nginx/config/scope.yml | 10 ++++ test/manual/payload/conf_1/scope.yml | 10 ++++ test/manual/payload/conf_2/scope.yml | 10 ++++ test/manual/payload/conf_3/scope.yml | 10 ++++ test/manual/send_event/conf_1/scope.yml | 10 ++++ test/reporttest.c | 2 +- website/src/pages/docs/config-file.md | 10 ++++ 21 files changed, 149 insertions(+), 29 deletions(-) diff --git a/conf/scope.yml b/conf/scope.yml index bfb0bcbf2..479e0d23e 100644 --- a/conf/scope.yml +++ b/conf/scope.yml @@ -611,6 +611,16 @@ libscope: # level: warning + # Set buffer threshold of the in-memory queue + # Type: integer + # Values: (greater than zero) bytes + # Default: 32768 + # Override: $SCOPE_LOG_BUFFER_THRESHOLD + # + # Threshold of raw data after reaching data will be flushed to log + # + buffer_threshold: 32768 + # Backend connection for logs # transport: diff --git a/src/cfg.c b/src/cfg.c index e7f819e09..495fc4c28 100644 --- a/src/cfg.c +++ b/src/cfg.c @@ -57,6 +57,7 @@ struct _config_t struct { cfg_log_level_t level; + unsigned long bufthreshold; } log; struct { @@ -208,6 +209,7 @@ cfgCreateDefault() } c->log.level = DEFAULT_LOG_LEVEL; + c->log.bufthreshold = DEFAULT_LOG_MAX_AGG_BYTES; c->pay.enable = DEFAULT_PAYLOAD_ENABLE; c->pay.dir = (DEFAULT_PAYLOAD_DIR) ? strdup(DEFAULT_PAYLOAD_DIR) : NULL; @@ -561,6 +563,12 @@ cfgLogLevel(config_t* cfg) return (cfg) ? cfg->log.level : DEFAULT_LOG_LEVEL; } +unsigned long +cfgLogBufThreshold(config_t* cfg) +{ + return (cfg) ? cfg->log.bufthreshold : DEFAULT_LOG_MAX_AGG_BYTES; +} + unsigned int cfgPayEnable(config_t *cfg) { @@ -912,6 +920,13 @@ cfgLogLevelSet(config_t* cfg, cfg_log_level_t level) cfg->log.level = level; } +void +cfgLogBufThresholdSet(config_t* cfg, unsigned long threshold) +{ + if (!cfg) return; + cfg->log.bufthreshold = threshold; +} + void cfgPayEnableSet(config_t *cfg, unsigned int val) { diff --git a/src/cfg.h b/src/cfg.h index 4ff350ce2..cffe78414 100644 --- a/src/cfg.h +++ b/src/cfg.h @@ -43,6 +43,7 @@ const char* cfgTransportTlsCACertPath(config_t *, which_transport_t); custom_tag_t** cfgCustomTags(config_t*); const char* cfgCustomTagValue(config_t*, const char*); cfg_log_level_t cfgLogLevel(config_t*); +unsigned long cfgLogBufThreshold(config_t*); unsigned int cfgPayEnable(config_t*); const char * cfgPayDir(config_t*); const char * cfgEvtFormatHeader(config_t *, int); @@ -78,6 +79,7 @@ void cfgTransportTlsValidateServerSet(config_t *, which_transport void cfgTransportTlsCACertPathSet(config_t *, which_transport_t, const char *); void cfgCustomTagAdd(config_t*, const char*, const char*); void cfgLogLevelSet(config_t*, cfg_log_level_t); +void cfgLogBufThresholdSet(config_t*, unsigned long); void cfgPayEnableSet(config_t*, unsigned int); void cfgPayDirSet(config_t*, const char *); void cfgEvtFormatHeaderSet(config_t *, const char *); diff --git a/src/cfgutils.c b/src/cfgutils.c index 2cb186899..a27498633 100644 --- a/src/cfgutils.c +++ b/src/cfgutils.c @@ -44,6 +44,7 @@ #define LIBSCOPE_NODE "libscope" #define LOG_NODE "log" +#define BUF_THRESHOLD_NODE "bufthreshold" #define LEVEL_NODE "level" #define TRANSPORT_NODE "transport" #define SUMMARYPERIOD_NODE "summaryperiod" @@ -165,6 +166,7 @@ void cfgTransportTlsValidateServerSetFromStr(config_t *, which_transport_t, cons void cfgTransportTlsCACertPathSetFromStr(config_t *, which_transport_t, const char *); void cfgCustomTagAddFromStr(config_t*, const char*, const char*); void cfgLogLevelSetFromStr(config_t*, const char*); +void cfgLogBufThresholdSetFromStr(config_t*, const char*); void cfgPayEnableSetFromStr(config_t*, const char*); void cfgPayDirSetFromStr(config_t*, const char*); void cfgAuthTokenSetFromStr(config_t*, const char*); @@ -455,6 +457,8 @@ processEnvStyleInput(config_t *cfg, const char *env_line) cfgMtcVerbositySetFromStr(cfg, value); } else if (!strcmp(env_name, "SCOPE_LOG_LEVEL")) { cfgLogLevelSetFromStr(cfg, value); + } else if (!strcmp(env_name, "SCOPE_LOG_BUFFER_THRESHOLD")) { + cfgLogBufThresholdSetFromStr(cfg, value); } else if (!strcmp(env_name, "SCOPE_METRIC_DEST")) { cfgTransportSetFromStr(cfg, CFG_MTC, value); } else if (!strcmp(env_name, "SCOPE_METRIC_TLS_ENABLE")) { @@ -649,10 +653,8 @@ void cfgMtcStatsDMaxLenSetFromStr(config_t* cfg, const char* value) { if (!cfg || !value) return; - errno = 0; - char* endptr = NULL; - unsigned long x = strtoul(value, &endptr, 10); - if (errno || *endptr) return; + unsigned long x; + if (!strToUnsignedLong(value, &x)) return; cfgMtcStatsDMaxLenSet(cfg, x); } @@ -661,10 +663,8 @@ void cfgMtcPeriodSetFromStr(config_t* cfg, const char* value) { if (!cfg || !value) return; - errno = 0; - char* endptr = NULL; - unsigned long x = strtoul(value, &endptr, 10); - if (errno || *endptr) return; + unsigned long x; + if (!strToUnsignedLong(value, &x)) return; cfgMtcPeriodSet(cfg, x); } @@ -703,10 +703,8 @@ void cfgEvtRateLimitSetFromStr(config_t* cfg, const char* value) { if (!cfg || !value) return; - errno = 0; - char* endptr = NULL; - unsigned long x = strtoul(value, &endptr, 10); - if (errno || *endptr) return; + unsigned long x; + if (!strToUnsignedLong(value, &x)) return; cfgEvtRateLimitSet(cfg, x); } @@ -757,10 +755,8 @@ void cfgMtcVerbositySetFromStr(config_t* cfg, const char* value) { if (!cfg || !value) return; - errno = 0; - char* endptr = NULL; - unsigned long x = strtoul(value, &endptr, 10); - if (errno || *endptr) return; + unsigned long x; + if (!strToUnsignedLong(value, &x)) return; cfgMtcVerbositySet(cfg, x); } @@ -857,6 +853,15 @@ cfgLogLevelSetFromStr(config_t* cfg, const char* value) cfgLogLevelSet(cfg, strToVal(logLevelMap, value)); } +void +cfgLogBufThresholdSetFromStr(config_t* cfg, const char* value) +{ + if (!cfg || !value) return; + unsigned long x; + if (!strToUnsignedLong(value, &x)) return; + cfgLogBufThresholdSet(cfg, x); +} + void cfgPayEnableSetFromStr(config_t* cfg, const char* value) { @@ -947,6 +952,14 @@ processLevel(config_t* config, yaml_document_t* doc, yaml_node_t* node) if (value) free(value); } +static void +processBufThreshold(config_t* config, yaml_document_t* doc, yaml_node_t* node) +{ + char* value = stringVal(node); + cfgLogBufThresholdSetFromStr(config, value); + if (value) free(value); +} + static void processTransportType(config_t* config, yaml_document_t* doc, yaml_node_t* node) { @@ -1086,6 +1099,7 @@ processLogging(config_t* config, yaml_document_t* doc, yaml_node_t* node) parse_table_t t[] = { {YAML_SCALAR_NODE, LEVEL_NODE, processLevel}, + {YAML_SCALAR_NODE, BUF_THRESHOLD_NODE, processBufThreshold}, {YAML_MAPPING_NODE, TRANSPORT_NODE, processTransportLog}, {YAML_NO_NODE, NULL, NULL} }; @@ -2121,6 +2135,8 @@ createLogJson(config_t* cfg) cJSON* transport; if (!(root = cJSON_CreateObject())) goto err; + if (!cJSON_AddNumberToObjLN(root, BUF_THRESHOLD_NODE, + cfgLogBufThreshold(cfg))) goto err; if (!cJSON_AddStringToObjLN(root, LEVEL_NODE, valToStr(logLevelMap, cfgLogLevel(cfg)))) goto err; @@ -2549,7 +2565,7 @@ initEvtFormat(config_t *cfg) ctl_t * initCtl(config_t *cfg) { - ctl_t *ctl = ctlCreate(); + ctl_t *ctl = ctlCreate(cfgLogBufThreshold(cfg)); if (!ctl) return ctl; /* diff --git a/src/ctl.c b/src/ctl.c index 2258d1a0c..18cac7835 100644 --- a/src/ctl.c +++ b/src/ctl.c @@ -14,7 +14,6 @@ #include "state.h" #define FS_ENTRIES 1024 -#define DEFAULT_LOG_MAX_AGG_BYTES 32768 #define DEFAULT_LOG_FLUSH_PERIOD_IN_MS 2000 #define CHANNEL "_channel" @@ -570,7 +569,7 @@ ctlCreateTxMsg(upload_t *upld) } ctl_t * -ctlCreate() +ctlCreate(unsigned long agg_bytes_limit) { ctl_t *ctl = calloc(1, sizeof(ctl_t)); if (!ctl) { @@ -583,7 +582,7 @@ ctlCreate() DBG(NULL); goto err; } - ctl->log.max_agg_bytes = DEFAULT_LOG_MAX_AGG_BYTES; + ctl->log.max_agg_bytes = agg_bytes_limit; ctl->log.flush_period_in_ms = DEFAULT_LOG_FLUSH_PERIOD_IN_MS; ctl->events = cbufInit(DEFAULT_CBUF_SIZE); diff --git a/src/ctl.h b/src/ctl.h index 4f047e788..11eca8ef7 100644 --- a/src/ctl.h +++ b/src/ctl.h @@ -100,7 +100,7 @@ char * ctlCreateTxMsg(upload_t*); typedef struct _ctl_t ctl_t; // Constructors Destructors -ctl_t * ctlCreate(); +ctl_t * ctlCreate(unsigned long); void ctlDestroy(ctl_t **); // Raw Send (without messaging protocol) diff --git a/src/scope_static.c b/src/scope_static.c index 3d8738d6d..c7a00cfde 100644 --- a/src/scope_static.c +++ b/src/scope_static.c @@ -683,6 +683,9 @@ static const char scope_help_configuration[] = " SCOPE_LOG_DEST\n" " same format as SCOPE_METRIC_DEST above.\n" " Default is file:///tmp/scope.log\n" +" SCOPE_LOG_BUFFER_THRESHOLD\n" +" threshold of flushing bytes from memory log queue.\n" +" Default is 32768 bytes.\n" " SCOPE_LOG_TLS_ENABLE\n" " Flag to enable Transport Layer Security (TLS). Only affects\n" " tcp:// destinations. true,false Default is false.\n" diff --git a/src/scopetypes.h b/src/scopetypes.h index 6d947d26d..668709558 100644 --- a/src/scopetypes.h +++ b/src/scopetypes.h @@ -142,6 +142,7 @@ typedef unsigned int bool; #define DEFAULT_LOG_PORT NULL #define DEFAULT_LOG_PATH "/tmp/scope.log" #define DEFAULT_LOG_BUF CFG_BUFFER_LINE +#define DEFAULT_LOG_MAX_AGG_BYTES 32768 #define DEFAULT_TLS_ENABLE FALSE #define DEFAULT_TLS_VALIDATE_SERVER TRUE #define DEFAULT_TLS_CA_CERT NULL diff --git a/src/utils.c b/src/utils.c index 5d1cf296f..c02e70daf 100644 --- a/src/utils.c +++ b/src/utils.c @@ -21,6 +21,16 @@ strToVal(enum_map_t map[], const char *str) return -1; } +int +strToUnsignedLong(const char* str, unsigned long* val) +{ + errno = 0; + char* endptr = NULL; + *val = strtoul(str, &endptr, 10); + if (errno || *endptr) return FALSE; + return TRUE; +} + const char * valToStr(enum_map_t map[], unsigned int val) { diff --git a/src/utils.h b/src/utils.h index cd92556d2..9bbfffff6 100644 --- a/src/utils.h +++ b/src/utils.h @@ -10,6 +10,7 @@ typedef struct { unsigned int strToVal(enum_map_t[], const char*); const char* valToStr(enum_map_t[], unsigned int); +int strToUnsignedLong(const char* str, unsigned long* val); int checkEnv(char *, char *); int fullSetenv(const char *, const char *, int); diff --git a/test/cfgtest.c b/test/cfgtest.c index c1107c241..898ca44ee 100644 --- a/test/cfgtest.c +++ b/test/cfgtest.c @@ -73,6 +73,7 @@ verifyDefaults(config_t* config) assert_null (cfgCustomTags(config)); assert_null (cfgCustomTagValue(config, "tagname")); assert_int_equal (cfgLogLevel(config), DEFAULT_LOG_LEVEL); + assert_int_equal (cfgLogBufThreshold(config), DEFAULT_LOG_MAX_AGG_BYTES); assert_int_equal (cfgPayEnable(config), DEFAULT_PAYLOAD_ENABLE); assert_string_equal (cfgPayDir(config), DEFAULT_PAYLOAD_DIR); } diff --git a/test/cfgutilstest.c b/test/cfgutilstest.c index 9b4d43e47..2df8b3659 100644 --- a/test/cfgutilstest.c +++ b/test/cfgutilstest.c @@ -1125,6 +1125,7 @@ verifyDefaults(config_t* config) assert_null (cfgCustomTags(config)); assert_null (cfgCustomTagValue(config, "tagname")); assert_int_equal (cfgLogLevel(config), DEFAULT_LOG_LEVEL); + assert_int_equal (cfgLogBufThreshold(config), DEFAULT_LOG_MAX_AGG_BYTES); assert_int_equal (cfgPayEnable(config), DEFAULT_PAYLOAD_ENABLE); assert_string_equal (cfgPayDir(config), DEFAULT_PAYLOAD_DIR); diff --git a/test/comtest.c b/test/comtest.c index b4fdab51a..ed1db2599 100644 --- a/test/comtest.c +++ b/test/comtest.c @@ -9,7 +9,7 @@ static void cmdPostInfoMsgDoesNotCrash(void** state) { - ctl_t* ctl = ctlCreate(); + ctl_t* ctl = ctlCreate(DEFAULT_LOG_MAX_AGG_BYTES); assert_non_null(ctl); assert_int_equal(-1, cmdPostInfoMsg(NULL, NULL)); @@ -23,7 +23,7 @@ cmdPostInfoMsgDoesNotCrash(void** state) static void cmdSendInfoStrDoesNotCrash(void** state) { - ctl_t* ctl = ctlCreate(); + ctl_t* ctl = ctlCreate(DEFAULT_LOG_MAX_AGG_BYTES); assert_non_null(ctl); assert_int_equal(-1, cmdSendInfoStr(NULL, NULL)); @@ -37,7 +37,7 @@ cmdSendInfoStrDoesNotCrash(void** state) static void cmdSendResponseDoesNotCrash(void** state) { - ctl_t* ctl = ctlCreate(); + ctl_t* ctl = ctlCreate(DEFAULT_LOG_MAX_AGG_BYTES); assert_non_null(ctl); const char buf[] = "{\"type\": \"req\", \"req\": \"huh?\", \"reqId\": 3.5}"; diff --git a/test/ctltest.c b/test/ctltest.c index 87cd9525a..54ae27c0d 100644 --- a/test/ctltest.c +++ b/test/ctltest.c @@ -259,6 +259,7 @@ ctlParseRxMsgSetCfg(void** state) " \"summaryperiod\": \"13\",\n" " \"log\": {\n" " \"level\": \"debug\",\n" + " \"bufthreshold\": \"32768\",\n" " \"transport\": {\n" " \"type\": \"shm\"\n" " }\n" @@ -641,7 +642,7 @@ ctlSendMsgForNullMtcDoesntCrash(void** state) static void ctlSendMsgForNullMessageDoesntCrash(void** state) { - ctl_t* ctl = ctlCreate(); + ctl_t* ctl = ctlCreate(DEFAULT_LOG_MAX_AGG_BYTES); assert_non_null(ctl); transport_t* t = transportCreateSyslog(); assert_non_null(t); @@ -654,7 +655,7 @@ static void ctlTransportSetAndMtcSend(void** state) { const char* file_path = "/tmp/my.path"; - ctl_t* ctl = ctlCreate(); + ctl_t* ctl = ctlCreate(DEFAULT_LOG_MAX_AGG_BYTES); assert_non_null(ctl); transport_t* t1 = transportCreateUdp("127.0.0.1", "12345"); transport_t* t2 = transportCreateUnix("/var/run/scope.sock"); @@ -736,7 +737,7 @@ ctlSendLogConsoleAsciiData(void **state) .procname = "foo", .cmd = "foo", .id = "foo"}; - ctl_t* ctl = ctlCreate(); + ctl_t* ctl = ctlCreate(DEFAULT_LOG_MAX_AGG_BYTES); assert_non_null(ctl); bool b_res = ctlEvtSourceEnabled(ctl, CFG_SRC_CONSOLE); assert_true(b_res); @@ -769,7 +770,7 @@ ctlSendLogConsoleNoneAsciiData(void **state) .procname = "foo", .cmd = "foo", .id = "foo"}; - ctl_t* ctl = ctlCreate(); + ctl_t* ctl = ctlCreate(DEFAULT_LOG_MAX_AGG_BYTES); assert_non_null(ctl); bool b_res = ctlEvtSourceEnabled(ctl, CFG_SRC_CONSOLE); assert_true(b_res); diff --git a/test/manual/fluentbit/services/nginx/config/scope.yml b/test/manual/fluentbit/services/nginx/config/scope.yml index 567edc690..4aaafa73a 100644 --- a/test/manual/fluentbit/services/nginx/config/scope.yml +++ b/test/manual/fluentbit/services/nginx/config/scope.yml @@ -599,6 +599,16 @@ libscope: # level: warning + # Set buffer threshold of the in-memory queue + # Type: integer + # Values: (greater than zero) bytes + # Default: 32768 + # Override: $SCOPE_LOG_BUFFER_THRESHOLD + # + # Threshold of raw data after reaching data will be flushed to log + # + buffer_threshold: 32768 + # Backend connection for logs # transport: diff --git a/test/manual/payload/conf_1/scope.yml b/test/manual/payload/conf_1/scope.yml index e5e8229aa..ccffbc156 100644 --- a/test/manual/payload/conf_1/scope.yml +++ b/test/manual/payload/conf_1/scope.yml @@ -598,6 +598,16 @@ libscope: # level: warning + # Set buffer threshold of the in-memory queue + # Type: integer + # Values: (greater than zero) bytes + # Default: 32768 + # Override: $SCOPE_LOG_BUFFER_THRESHOLD + # + # Threshold of raw data after reaching data will be flushed to log + # + buffer_threshold: 32768 + # Backend connection for logs # transport: diff --git a/test/manual/payload/conf_2/scope.yml b/test/manual/payload/conf_2/scope.yml index 6c274cd2a..4f0ba6c51 100644 --- a/test/manual/payload/conf_2/scope.yml +++ b/test/manual/payload/conf_2/scope.yml @@ -598,6 +598,16 @@ libscope: # level: warning + # Set buffer threshold of the in-memory queue + # Type: integer + # Values: (greater than zero) bytes + # Default: 32768 + # Override: $SCOPE_LOG_BUFFER_THRESHOLD + # + # Threshold of raw data after reaching data will be flushed to log + # + buffer_threshold: 32768 + # Backend connection for logs # transport: diff --git a/test/manual/payload/conf_3/scope.yml b/test/manual/payload/conf_3/scope.yml index dcfec86a0..3e2294734 100644 --- a/test/manual/payload/conf_3/scope.yml +++ b/test/manual/payload/conf_3/scope.yml @@ -598,6 +598,16 @@ libscope: # level: warning + # Set buffer threshold of the in-memory queue + # Type: integer + # Values: (greater than zero) bytes + # Default: 32768 + # Override: $SCOPE_LOG_BUFFER_THRESHOLD + # + # Threshold of raw data after reaching data will be flushed to log + # + buffer_threshold: 32768 + # Backend connection for logs # transport: diff --git a/test/manual/send_event/conf_1/scope.yml b/test/manual/send_event/conf_1/scope.yml index 72c763a2d..3ae8bd8eb 100644 --- a/test/manual/send_event/conf_1/scope.yml +++ b/test/manual/send_event/conf_1/scope.yml @@ -598,6 +598,16 @@ libscope: # level: warning + # Set buffer threshold of the in-memory queue + # Type: integer + # Values: (greater than zero) bytes + # Default: 32768 + # Override: $SCOPE_LOG_BUFFER_THRESHOLD + # + # Threshold of raw data after reaching data will be flushed to log + # + buffer_threshold: 32768 + # Backend connection for logs # transport: diff --git a/test/reporttest.c b/test/reporttest.c index 8556f71a4..c11a7a2d1 100644 --- a/test/reporttest.c +++ b/test/reporttest.c @@ -150,7 +150,7 @@ countTestSetup(void** state) init_g_proc(); g_log = logCreate(); g_mtc = mtcCreate(); - g_ctl = ctlCreate(); + g_ctl = ctlCreate(DEFAULT_LOG_MAX_AGG_BYTES); initState(); diff --git a/website/src/pages/docs/config-file.md b/website/src/pages/docs/config-file.md index 04ef8ace9..05da32371 100644 --- a/website/src/pages/docs/config-file.md +++ b/website/src/pages/docs/config-file.md @@ -615,6 +615,16 @@ libscope: # level: warning + # Set buffer threshold of the in-memory queue + # Type: integer + # Values: (greater than zero) bytes + # Default: 32768 + # Override: $SCOPE_LOG_BUFFER_THRESHOLD + # + # Threshold of raw data after reaching data will be flushed to log + # + buffer_threshold: 32768 + # Backend connection for logs # transport: From d7062492a4fcf9b2d4f82870163eb0db9ec4e34a Mon Sep 17 00:00:00 2001 From: michalbiesek Date: Mon, 20 Dec 2021 12:39:24 +0100 Subject: [PATCH 2/2] Parametrize flush period of in-memory queue closes #189 --- conf/scope.yml | 10 +++++++ src/cfg.c | 15 +++++++++++ src/cfg.h | 2 ++ src/cfgutils.c | 26 ++++++++++++++++++- src/ctl.c | 7 +++-- src/ctl.h | 2 +- src/scope_static.c | 3 +++ src/scopetypes.h | 1 + test/cfgtest.c | 1 + test/cfgutilstest.c | 1 + test/comtest.c | 12 ++++++--- test/ctltest.c | 17 +++++++++--- .../fluentbit/services/nginx/config/scope.yml | 10 +++++++ test/manual/payload/conf_1/scope.yml | 10 +++++++ test/manual/payload/conf_2/scope.yml | 10 +++++++ test/manual/payload/conf_3/scope.yml | 10 +++++++ test/manual/send_event/conf_1/scope.yml | 10 +++++++ test/reporttest.c | 6 ++++- website/src/pages/docs/config-file.md | 10 +++++++ 19 files changed, 149 insertions(+), 14 deletions(-) diff --git a/conf/scope.yml b/conf/scope.yml index 479e0d23e..3e9c1caa7 100644 --- a/conf/scope.yml +++ b/conf/scope.yml @@ -621,6 +621,16 @@ libscope: # buffer_threshold: 32768 + # Set flush period in-memory queue + # Type: integer + # Values: (greater than zero) miliseconds + # Default: 2000 + # Override: $SCOPE_LOG_FLUSH_PERIOD + # + # Period after data will be flushed to log + # + flush_period: 2000 + # Backend connection for logs # transport: diff --git a/src/cfg.c b/src/cfg.c index 495fc4c28..6117f0a05 100644 --- a/src/cfg.c +++ b/src/cfg.c @@ -58,6 +58,7 @@ struct _config_t struct { cfg_log_level_t level; unsigned long bufthreshold; + unsigned long flushperiod; } log; struct { @@ -210,6 +211,7 @@ cfgCreateDefault() c->log.level = DEFAULT_LOG_LEVEL; c->log.bufthreshold = DEFAULT_LOG_MAX_AGG_BYTES; + c->log.flushperiod = DEFAULT_LOG_FLUSH_PERIOD_IN_MS; c->pay.enable = DEFAULT_PAYLOAD_ENABLE; c->pay.dir = (DEFAULT_PAYLOAD_DIR) ? strdup(DEFAULT_PAYLOAD_DIR) : NULL; @@ -569,6 +571,12 @@ cfgLogBufThreshold(config_t* cfg) return (cfg) ? cfg->log.bufthreshold : DEFAULT_LOG_MAX_AGG_BYTES; } +unsigned long +cfgLogFlushPeriod(config_t* cfg) +{ + return (cfg) ? cfg->log.flushperiod : DEFAULT_LOG_FLUSH_PERIOD_IN_MS; +} + unsigned int cfgPayEnable(config_t *cfg) { @@ -927,6 +935,13 @@ cfgLogBufThresholdSet(config_t* cfg, unsigned long threshold) cfg->log.bufthreshold = threshold; } +void +cfgLogFlushPeriodSet(config_t* cfg, unsigned long period) +{ + if (!cfg) return; + cfg->log.flushperiod = period; +} + void cfgPayEnableSet(config_t *cfg, unsigned int val) { diff --git a/src/cfg.h b/src/cfg.h index cffe78414..c98741a99 100644 --- a/src/cfg.h +++ b/src/cfg.h @@ -44,6 +44,7 @@ custom_tag_t** cfgCustomTags(config_t*); const char* cfgCustomTagValue(config_t*, const char*); cfg_log_level_t cfgLogLevel(config_t*); unsigned long cfgLogBufThreshold(config_t*); +unsigned long cfgLogFlushPeriod(config_t*); unsigned int cfgPayEnable(config_t*); const char * cfgPayDir(config_t*); const char * cfgEvtFormatHeader(config_t *, int); @@ -80,6 +81,7 @@ void cfgTransportTlsCACertPathSet(config_t *, which_transport_t, void cfgCustomTagAdd(config_t*, const char*, const char*); void cfgLogLevelSet(config_t*, cfg_log_level_t); void cfgLogBufThresholdSet(config_t*, unsigned long); +void cfgLogFlushPeriodSet(config_t*, unsigned long); void cfgPayEnableSet(config_t*, unsigned int); void cfgPayDirSet(config_t*, const char *); void cfgEvtFormatHeaderSet(config_t *, const char *); diff --git a/src/cfgutils.c b/src/cfgutils.c index a27498633..42af56c43 100644 --- a/src/cfgutils.c +++ b/src/cfgutils.c @@ -45,6 +45,7 @@ #define LIBSCOPE_NODE "libscope" #define LOG_NODE "log" #define BUF_THRESHOLD_NODE "bufthreshold" +#define FLUSH_PERIOD_NODE "flushperiod" #define LEVEL_NODE "level" #define TRANSPORT_NODE "transport" #define SUMMARYPERIOD_NODE "summaryperiod" @@ -167,6 +168,7 @@ void cfgTransportTlsCACertPathSetFromStr(config_t *, which_transport_t, const ch void cfgCustomTagAddFromStr(config_t*, const char*, const char*); void cfgLogLevelSetFromStr(config_t*, const char*); void cfgLogBufThresholdSetFromStr(config_t*, const char*); +void cfgLogFlushPeriodSetFromStr(config_t*, const char*); void cfgPayEnableSetFromStr(config_t*, const char*); void cfgPayDirSetFromStr(config_t*, const char*); void cfgAuthTokenSetFromStr(config_t*, const char*); @@ -459,6 +461,8 @@ processEnvStyleInput(config_t *cfg, const char *env_line) cfgLogLevelSetFromStr(cfg, value); } else if (!strcmp(env_name, "SCOPE_LOG_BUFFER_THRESHOLD")) { cfgLogBufThresholdSetFromStr(cfg, value); + } else if (!strcmp(env_name, "SCOPE_LOG_FLUSH_PERIOD")) { + cfgLogFlushPeriodSetFromStr(cfg, value); } else if (!strcmp(env_name, "SCOPE_METRIC_DEST")) { cfgTransportSetFromStr(cfg, CFG_MTC, value); } else if (!strcmp(env_name, "SCOPE_METRIC_TLS_ENABLE")) { @@ -862,6 +866,15 @@ cfgLogBufThresholdSetFromStr(config_t* cfg, const char* value) cfgLogBufThresholdSet(cfg, x); } +void +cfgLogFlushPeriodSetFromStr(config_t* cfg, const char* value) +{ + if (!cfg || !value) return; + unsigned long x; + if (!strToUnsignedLong(value, &x)) return; + cfgLogFlushPeriodSet(cfg, x); +} + void cfgPayEnableSetFromStr(config_t* cfg, const char* value) { @@ -960,6 +973,14 @@ processBufThreshold(config_t* config, yaml_document_t* doc, yaml_node_t* node) if (value) free(value); } +static void +processFlushPeriod(config_t* config, yaml_document_t* doc, yaml_node_t* node) +{ + char* value = stringVal(node); + cfgLogFlushPeriodSetFromStr(config, value); + if (value) free(value); +} + static void processTransportType(config_t* config, yaml_document_t* doc, yaml_node_t* node) { @@ -1100,6 +1121,7 @@ processLogging(config_t* config, yaml_document_t* doc, yaml_node_t* node) parse_table_t t[] = { {YAML_SCALAR_NODE, LEVEL_NODE, processLevel}, {YAML_SCALAR_NODE, BUF_THRESHOLD_NODE, processBufThreshold}, + {YAML_SCALAR_NODE, FLUSH_PERIOD_NODE, processFlushPeriod}, {YAML_MAPPING_NODE, TRANSPORT_NODE, processTransportLog}, {YAML_NO_NODE, NULL, NULL} }; @@ -2137,6 +2159,8 @@ createLogJson(config_t* cfg) if (!(root = cJSON_CreateObject())) goto err; if (!cJSON_AddNumberToObjLN(root, BUF_THRESHOLD_NODE, cfgLogBufThreshold(cfg))) goto err; + if (!cJSON_AddNumberToObjLN(root, FLUSH_PERIOD_NODE, + cfgLogFlushPeriod(cfg))) goto err; if (!cJSON_AddStringToObjLN(root, LEVEL_NODE, valToStr(logLevelMap, cfgLogLevel(cfg)))) goto err; @@ -2565,7 +2589,7 @@ initEvtFormat(config_t *cfg) ctl_t * initCtl(config_t *cfg) { - ctl_t *ctl = ctlCreate(cfgLogBufThreshold(cfg)); + ctl_t *ctl = ctlCreate(cfg); if (!ctl) return ctl; /* diff --git a/src/ctl.c b/src/ctl.c index 18cac7835..5a39b2c1d 100644 --- a/src/ctl.c +++ b/src/ctl.c @@ -14,7 +14,6 @@ #include "state.h" #define FS_ENTRIES 1024 -#define DEFAULT_LOG_FLUSH_PERIOD_IN_MS 2000 #define CHANNEL "_channel" #define ID "id" @@ -569,7 +568,7 @@ ctlCreateTxMsg(upload_t *upld) } ctl_t * -ctlCreate(unsigned long agg_bytes_limit) +ctlCreate(config_t *cfg) { ctl_t *ctl = calloc(1, sizeof(ctl_t)); if (!ctl) { @@ -582,8 +581,8 @@ ctlCreate(unsigned long agg_bytes_limit) DBG(NULL); goto err; } - ctl->log.max_agg_bytes = agg_bytes_limit; - ctl->log.flush_period_in_ms = DEFAULT_LOG_FLUSH_PERIOD_IN_MS; + ctl->log.max_agg_bytes = cfgLogBufThreshold(cfg); + ctl->log.flush_period_in_ms = cfgLogFlushPeriod(cfg); ctl->events = cbufInit(DEFAULT_CBUF_SIZE); if (!ctl->events) { diff --git a/src/ctl.h b/src/ctl.h index 11eca8ef7..1c760c205 100644 --- a/src/ctl.h +++ b/src/ctl.h @@ -100,7 +100,7 @@ char * ctlCreateTxMsg(upload_t*); typedef struct _ctl_t ctl_t; // Constructors Destructors -ctl_t * ctlCreate(unsigned long); +ctl_t * ctlCreate(config_t *); void ctlDestroy(ctl_t **); // Raw Send (without messaging protocol) diff --git a/src/scope_static.c b/src/scope_static.c index c7a00cfde..4c152c868 100644 --- a/src/scope_static.c +++ b/src/scope_static.c @@ -686,6 +686,9 @@ static const char scope_help_configuration[] = " SCOPE_LOG_BUFFER_THRESHOLD\n" " threshold of flushing bytes from memory log queue.\n" " Default is 32768 bytes.\n" +" SCOPE_LOG_FLUSH_PERIOD\n" +" period of flushing bytes from memory log queue.\n" +" Default is 2000 ms.\n" " SCOPE_LOG_TLS_ENABLE\n" " Flag to enable Transport Layer Security (TLS). Only affects\n" " tcp:// destinations. true,false Default is false.\n" diff --git a/src/scopetypes.h b/src/scopetypes.h index 668709558..cfb828260 100644 --- a/src/scopetypes.h +++ b/src/scopetypes.h @@ -143,6 +143,7 @@ typedef unsigned int bool; #define DEFAULT_LOG_PATH "/tmp/scope.log" #define DEFAULT_LOG_BUF CFG_BUFFER_LINE #define DEFAULT_LOG_MAX_AGG_BYTES 32768 +#define DEFAULT_LOG_FLUSH_PERIOD_IN_MS 2000 #define DEFAULT_TLS_ENABLE FALSE #define DEFAULT_TLS_VALIDATE_SERVER TRUE #define DEFAULT_TLS_CA_CERT NULL diff --git a/test/cfgtest.c b/test/cfgtest.c index 898ca44ee..a831e0bd2 100644 --- a/test/cfgtest.c +++ b/test/cfgtest.c @@ -74,6 +74,7 @@ verifyDefaults(config_t* config) assert_null (cfgCustomTagValue(config, "tagname")); assert_int_equal (cfgLogLevel(config), DEFAULT_LOG_LEVEL); assert_int_equal (cfgLogBufThreshold(config), DEFAULT_LOG_MAX_AGG_BYTES); + assert_int_equal (cfgLogFlushPeriod(config), DEFAULT_LOG_FLUSH_PERIOD_IN_MS); assert_int_equal (cfgPayEnable(config), DEFAULT_PAYLOAD_ENABLE); assert_string_equal (cfgPayDir(config), DEFAULT_PAYLOAD_DIR); } diff --git a/test/cfgutilstest.c b/test/cfgutilstest.c index 2df8b3659..fd389d593 100644 --- a/test/cfgutilstest.c +++ b/test/cfgutilstest.c @@ -1126,6 +1126,7 @@ verifyDefaults(config_t* config) assert_null (cfgCustomTagValue(config, "tagname")); assert_int_equal (cfgLogLevel(config), DEFAULT_LOG_LEVEL); assert_int_equal (cfgLogBufThreshold(config), DEFAULT_LOG_MAX_AGG_BYTES); + assert_int_equal (cfgLogFlushPeriod(config), DEFAULT_LOG_FLUSH_PERIOD_IN_MS); assert_int_equal (cfgPayEnable(config), DEFAULT_PAYLOAD_ENABLE); assert_string_equal (cfgPayDir(config), DEFAULT_PAYLOAD_DIR); diff --git a/test/comtest.c b/test/comtest.c index ed1db2599..d2ce5774a 100644 --- a/test/comtest.c +++ b/test/comtest.c @@ -9,7 +9,8 @@ static void cmdPostInfoMsgDoesNotCrash(void** state) { - ctl_t* ctl = ctlCreate(DEFAULT_LOG_MAX_AGG_BYTES); + config_t* cfg = cfgCreateDefault(); + ctl_t* ctl = ctlCreate(cfg); assert_non_null(ctl); assert_int_equal(-1, cmdPostInfoMsg(NULL, NULL)); @@ -18,12 +19,14 @@ cmdPostInfoMsgDoesNotCrash(void** state) assert_int_equal(0, cmdPostInfoMsg(ctl, cJSON_CreateString("hey"))); ctlDestroy(&ctl); + cfgDestroy(&cfg); } static void cmdSendInfoStrDoesNotCrash(void** state) { - ctl_t* ctl = ctlCreate(DEFAULT_LOG_MAX_AGG_BYTES); + config_t* cfg = cfgCreateDefault(); + ctl_t* ctl = ctlCreate(cfg); assert_non_null(ctl); assert_int_equal(-1, cmdSendInfoStr(NULL, NULL)); @@ -32,12 +35,14 @@ cmdSendInfoStrDoesNotCrash(void** state) assert_int_equal(0, cmdSendInfoStr(ctl, "hey")); ctlDestroy(&ctl); + cfgDestroy(&cfg); } static void cmdSendResponseDoesNotCrash(void** state) { - ctl_t* ctl = ctlCreate(DEFAULT_LOG_MAX_AGG_BYTES); + config_t* cfg = cfgCreateDefault(); + ctl_t* ctl = ctlCreate(cfg); assert_non_null(ctl); const char buf[] = "{\"type\": \"req\", \"req\": \"huh?\", \"reqId\": 3.5}"; @@ -53,6 +58,7 @@ cmdSendResponseDoesNotCrash(void** state) ctlDestroy(&ctl); destroyReq(&req); + cfgDestroy(&cfg); } static void diff --git a/test/ctltest.c b/test/ctltest.c index 54ae27c0d..a3c76b2ba 100644 --- a/test/ctltest.c +++ b/test/ctltest.c @@ -260,6 +260,7 @@ ctlParseRxMsgSetCfg(void** state) " \"log\": {\n" " \"level\": \"debug\",\n" " \"bufthreshold\": \"32768\",\n" + " \"flushperiod\": \"2000\",\n" " \"transport\": {\n" " \"type\": \"shm\"\n" " }\n" @@ -642,20 +643,23 @@ ctlSendMsgForNullMtcDoesntCrash(void** state) static void ctlSendMsgForNullMessageDoesntCrash(void** state) { - ctl_t* ctl = ctlCreate(DEFAULT_LOG_MAX_AGG_BYTES); + config_t* cfg = cfgCreateDefault(); + ctl_t* ctl = ctlCreate(cfg); assert_non_null(ctl); transport_t* t = transportCreateSyslog(); assert_non_null(t); ctlTransportSet(ctl, t, CFG_CTL); ctlSendMsg(ctl, NULL); ctlDestroy(&ctl); + cfgDestroy(&cfg); } static void ctlTransportSetAndMtcSend(void** state) { const char* file_path = "/tmp/my.path"; - ctl_t* ctl = ctlCreate(DEFAULT_LOG_MAX_AGG_BYTES); + config_t* cfg = cfgCreateDefault(); + ctl_t* ctl = ctlCreate(cfg); assert_non_null(ctl); transport_t* t1 = transportCreateUdp("127.0.0.1", "12345"); transport_t* t2 = transportCreateUnix("/var/run/scope.sock"); @@ -694,6 +698,7 @@ ctlTransportSetAndMtcSend(void** state) fail_msg("Couldn't delete file %s", file_path); ctlDestroy(&ctl); + cfgDestroy(&cfg); } static void @@ -737,7 +742,8 @@ ctlSendLogConsoleAsciiData(void **state) .procname = "foo", .cmd = "foo", .id = "foo"}; - ctl_t* ctl = ctlCreate(DEFAULT_LOG_MAX_AGG_BYTES); + config_t* cfg = cfgCreateDefault(); + ctl_t* ctl = ctlCreate(cfg); assert_non_null(ctl); bool b_res = ctlEvtSourceEnabled(ctl, CFG_SRC_CONSOLE); assert_true(b_res); @@ -748,6 +754,7 @@ ctlSendLogConsoleAsciiData(void **state) const char *val = get_cbuf_data(); assert_string_equal(ascii_text, val); ctlDestroy(&ctl); + cfgDestroy(&cfg); allow_copy_buf_data(FALSE); } @@ -770,7 +777,8 @@ ctlSendLogConsoleNoneAsciiData(void **state) .procname = "foo", .cmd = "foo", .id = "foo"}; - ctl_t* ctl = ctlCreate(DEFAULT_LOG_MAX_AGG_BYTES); + config_t* cfg = cfgCreateDefault(); + ctl_t* ctl = ctlCreate(cfg); assert_non_null(ctl); bool b_res = ctlEvtSourceEnabled(ctl, CFG_SRC_CONSOLE); assert_true(b_res); @@ -782,6 +790,7 @@ ctlSendLogConsoleNoneAsciiData(void **state) assert_string_equal(binary_data_event_msg, val); free(non_basic_ascii_text); ctlDestroy(&ctl); + cfgDestroy(&cfg); allow_copy_buf_data(FALSE); } diff --git a/test/manual/fluentbit/services/nginx/config/scope.yml b/test/manual/fluentbit/services/nginx/config/scope.yml index 4aaafa73a..f78beadb6 100644 --- a/test/manual/fluentbit/services/nginx/config/scope.yml +++ b/test/manual/fluentbit/services/nginx/config/scope.yml @@ -609,6 +609,16 @@ libscope: # buffer_threshold: 32768 + # Set flush period in-memory queue + # Type: integer + # Values: (greater than zero) miliseconds + # Default: 2000 + # Override: $SCOPE_LOG_FLUSH_PERIOD + # + # Period after data will be flushed to log + # + flush_period: 2000 + # Backend connection for logs # transport: diff --git a/test/manual/payload/conf_1/scope.yml b/test/manual/payload/conf_1/scope.yml index ccffbc156..e9bf5abe1 100644 --- a/test/manual/payload/conf_1/scope.yml +++ b/test/manual/payload/conf_1/scope.yml @@ -608,6 +608,16 @@ libscope: # buffer_threshold: 32768 + # Set flush period in-memory queue + # Type: integer + # Values: (greater than zero) miliseconds + # Default: 2000 + # Override: $SCOPE_LOG_FLUSH_PERIOD + # + # Period after data will be flushed to log + # + flush_period: 2000 + # Backend connection for logs # transport: diff --git a/test/manual/payload/conf_2/scope.yml b/test/manual/payload/conf_2/scope.yml index 4f0ba6c51..3edc1976a 100644 --- a/test/manual/payload/conf_2/scope.yml +++ b/test/manual/payload/conf_2/scope.yml @@ -608,6 +608,16 @@ libscope: # buffer_threshold: 32768 + # Set flush period in-memory queue + # Type: integer + # Values: (greater than zero) miliseconds + # Default: 2000 + # Override: $SCOPE_LOG_FLUSH_PERIOD + # + # Period after data will be flushed to log + # + flush_period: 2000 + # Backend connection for logs # transport: diff --git a/test/manual/payload/conf_3/scope.yml b/test/manual/payload/conf_3/scope.yml index 3e2294734..e2bb70497 100644 --- a/test/manual/payload/conf_3/scope.yml +++ b/test/manual/payload/conf_3/scope.yml @@ -608,6 +608,16 @@ libscope: # buffer_threshold: 32768 + # Set flush period in-memory queue + # Type: integer + # Values: (greater than zero) miliseconds + # Default: 2000 + # Override: $SCOPE_LOG_FLUSH_PERIOD + # + # Period after data will be flushed to log + # + flush_period: 2000 + # Backend connection for logs # transport: diff --git a/test/manual/send_event/conf_1/scope.yml b/test/manual/send_event/conf_1/scope.yml index 3ae8bd8eb..434d2bce7 100644 --- a/test/manual/send_event/conf_1/scope.yml +++ b/test/manual/send_event/conf_1/scope.yml @@ -608,6 +608,16 @@ libscope: # buffer_threshold: 32768 + # Set flush period in-memory queue + # Type: integer + # Values: (greater than zero) miliseconds + # Default: 2000 + # Override: $SCOPE_LOG_FLUSH_PERIOD + # + # Period after data will be flushed to log + # + flush_period: 2000 + # Backend connection for logs # transport: diff --git a/test/reporttest.c b/test/reporttest.c index c11a7a2d1..c7c1a59f3 100644 --- a/test/reporttest.c +++ b/test/reporttest.c @@ -19,6 +19,7 @@ event_t evtBuf[BUFSIZE] = {{0}}; int evtBufNext = 0; event_t mtcBuf[BUFSIZE] = {{0}}; int mtcBufNext = 0; +static config_t *test_config; // These signatures satisfy --wrap=cmdSendEvent in the Makefile #ifdef __linux__ @@ -148,9 +149,11 @@ countTestSetup(void** state) // init objects that count has init_g_proc(); + test_config = cfgCreateDefault(); + g_log = logCreate(); g_mtc = mtcCreate(); - g_ctl = ctlCreate(DEFAULT_LOG_MAX_AGG_BYTES); + g_ctl = ctlCreate(test_config); initState(); @@ -169,6 +172,7 @@ countTestTeardown(void** state) logDestroy(&g_log); mtcDestroy(&g_mtc); ctlDestroy(&g_ctl); + cfgDestroy(&test_config); // Call the general groupTeardown() too. return groupTeardown(state); diff --git a/website/src/pages/docs/config-file.md b/website/src/pages/docs/config-file.md index 05da32371..19239592b 100644 --- a/website/src/pages/docs/config-file.md +++ b/website/src/pages/docs/config-file.md @@ -625,6 +625,16 @@ libscope: # buffer_threshold: 32768 + # Set flush period in-memory queue + # Type: integer + # Values: (greater than zero) miliseconds + # Default: 2000 + # Override: $SCOPE_LOG_FLUSH_PERIOD + # + # Period after data will be flushed to log + # + flush_period: 2000 + # Backend connection for logs # transport: