diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..b93d279 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +SOURCES/rsyslog-8.37.0.tar.gz +SOURCES/rsyslog-doc-8.37.0.tar.gz diff --git a/.rsyslog.metadata b/.rsyslog.metadata new file mode 100644 index 0000000..7b6aed8 --- /dev/null +++ b/.rsyslog.metadata @@ -0,0 +1,2 @@ +43076e3010fc3fd5178201a916beb93848b5249c SOURCES/rsyslog-8.37.0.tar.gz +4c75f56e2d55c4c87d07781fb6d9deabf63395fb SOURCES/rsyslog-doc-8.37.0.tar.gz diff --git a/SOURCES/rsyslog-8.32.0-service.patch b/SOURCES/rsyslog-8.32.0-service.patch new file mode 100644 index 0000000..cb59f3f --- /dev/null +++ b/SOURCES/rsyslog-8.32.0-service.patch @@ -0,0 +1,21 @@ +diff -up ./rsyslog.service.in.service ./rsyslog.service.in +--- ./rsyslog.service.in.service 2018-03-01 13:58:11.480598935 +0100 ++++ ./rsyslog.service.in 2018-03-01 13:58:25.433518607 +0100 +@@ -1,12 +1,16 @@ + [Unit] + Description=System Logging Service + Requires=syslog.socket ++Wants=network.target network-online.target ++After=network.target network-online.target + Documentation=man:rsyslogd(8) + Documentation=http://www.rsyslog.com/doc/ + + [Service] + Type=notify +-ExecStart=@sbindir@/rsyslogd -n -iNONE ++EnvironmentFile=-/etc/sysconfig/rsyslog ++ExecStart=@sbindir@/rsyslogd -n $SYSLOGD_OPTIONS ++UMask=0066 + StandardOutput=null + Restart=on-failure + diff --git a/SOURCES/rsyslog-8.37.0-rhbz1614179-imfile-symlink-support.patch b/SOURCES/rsyslog-8.37.0-rhbz1614179-imfile-symlink-support.patch new file mode 100644 index 0000000..125846d --- /dev/null +++ b/SOURCES/rsyslog-8.37.0-rhbz1614179-imfile-symlink-support.patch @@ -0,0 +1,416 @@ +From 3822da837e4d531e8a9cd78ae76359a410f8d98d Mon Sep 17 00:00:00 2001 +From: Jiri Vymazal +Date: Thu, 31 May 2018 16:30:08 +0200 +Subject: [PATCH] Symlink support for imfile + +this introduces symlink detection and following as well +as monitoring changes on them. Also added test for the new +functionality and ensuring the original symlink behavior +stays as well. +--- + plugins/imfile/imfile.c | 182 +++++++++++++++++++++++++++---------- + 1 file changed, 133 insertions(+), 49 deletions(-) + +diff --git a/plugins/imfile/imfile.c b/plugins/imfile/imfile.c +index 3c9308bfe..4ca23d2ca 100644 +--- a/plugins/imfile/imfile.c ++++ b/plugins/imfile/imfile.c +@@ -152,6 +152,7 @@ struct act_obj_s { + fs_edge_t *edge; /* edge which this object belongs to */ + char *name; /* full path name of active object */ + char *basename; /* only basename */ //TODO: remove when refactoring rename support ++ char *source_name; /* if this object is target of a symlink, source_name is its name (else NULL) */ + //char *statefile; /* base name of state file (for move operations) */ + int wd; + #if defined(OS_SOLARIS) && defined (HAVE_PORT_SOURCE_FILE) +@@ -167,6 +168,7 @@ struct act_obj_s { + int nRecords; /**< How many records did we process before persisting the stream? */ + ratelimit_t *ratelimiter; + multi_submit_t multiSub; ++ int is_symlink; + }; + struct fs_edge_s { + fs_node_t *parent; +@@ -181,7 +182,8 @@ struct act_obj_s { + instanceConf_t **instarr; + }; + struct fs_node_s { +- fs_edge_t *edges; ++ fs_edge_t *edges; /* NULL in leaf nodes */ ++ fs_node_t *root; /* node one level up (NULL for file system root) */ + }; + + +@@ -189,7 +191,7 @@ static rsRetVal persistStrmState(act_obj_t *); + static rsRetVal resetConfigVariables(uchar __attribute__((unused)) *pp, void __attribute__((unused)) *pVal); + static rsRetVal ATTR_NONNULL(1) pollFile(act_obj_t *act); + static int ATTR_NONNULL() getBasename(uchar *const __restrict__ basen, uchar *const __restrict__ path); +-static void ATTR_NONNULL() act_obj_unlink(act_obj_t *const act); ++static void ATTR_NONNULL() act_obj_unlink(act_obj_t *act); + static uchar * ATTR_NONNULL(1, 2) getStateFileName(const act_obj_t *, uchar *, const size_t); + static int ATTR_NONNULL() getFullStateFileName(const uchar *const, uchar *const pszout, const size_t ilenout); + +@@ -483,14 +485,17 @@ in_setupWatch(act_obj_t *const act, const int is_file) + goto done; + + wd = inotify_add_watch(ino_fd, act->name, +- (is_file) ? IN_MODIFY : IN_CREATE|IN_DELETE|IN_MOVED_FROM|IN_MOVED_TO); ++ (is_file) ? IN_MODIFY|IN_DONT_FOLLOW : IN_CREATE|IN_DELETE|IN_MOVED_FROM|IN_MOVED_TO); + if(wd < 0) { +- LogError(errno, RS_RET_IO_ERROR, "imfile: cannot watch object '%s'", +- act->name); ++ if (errno == EACCES) { /* There is high probability of selinux denial on top-level paths */ ++ DBGPRINTF("imfile: permission denied when adding watch for '%s'\n", act->name); ++ } else { ++ LogError(errno, RS_RET_IO_ERROR, "imfile: cannot watch object '%s'", act->name); ++ } + goto done; + } + wdmapAdd(wd, act); +- DBGPRINTF("in_setupDirWatch: watch %d added for dir %s(%p)\n", wd, act->name, act); ++ DBGPRINTF("in_setupWatch: watch %d added for %s(object %p)\n", wd, act->name, act); + done: return wd; + } + +@@ -605,7 +610,7 @@ done: return; + static void ATTR_NONNULL() + fen_setupWatch(act_obj_t *const act __attribute__((unused))) + { +- DBGPRINTF("fen_setupWatch: DUMMY CALLED - not on Solaris?"); ++ DBGPRINTF("fen_setupWatch: DUMMY CALLED - not on Solaris?\n"); + } + #endif /* FEN */ + +@@ -633,38 +638,48 @@ fs_node_print(const fs_node_t *const node, const int level) + } + } + +- + /* add a new file system object if it not yet exists, ignore call + * if it already does. + */ +-static rsRetVal ATTR_NONNULL() ++static rsRetVal ATTR_NONNULL(1,2) + act_obj_add(fs_edge_t *const edge, const char *const name, const int is_file, +- const ino_t ino) ++ const ino_t ino, const int is_symlink, const char *const source) + { + act_obj_t *act; + char basename[MAXFNAME]; + DEFiRet; + +- DBGPRINTF("act_obj_add: edge %p, name '%s'\n", edge, name); ++ DBGPRINTF("act_obj_add: edge %p, name '%s' (source '%s')\n", edge, name, source? source : "---"); + for(act = edge->active ; act != NULL ; act = act->next) { + if(!strcmp(act->name, name)) { +- DBGPRINTF("active object '%s' already exists in '%s' - no need to add\n", +- name, edge->path); +- FINALIZE; ++ if (!source || !act->source_name || !strcmp(act->source_name, source)) { ++ DBGPRINTF("active object '%s' already exists in '%s' - no need to add\n", ++ name, edge->path); ++ FINALIZE; ++ } + } + } + DBGPRINTF("add new active object '%s' in '%s'\n", name, edge->path); + CHKmalloc(act = calloc(sizeof(act_obj_t), 1)); + CHKmalloc(act->name = strdup(name)); +- getBasename((uchar*)basename, (uchar*)name); +- CHKmalloc(act->basename = strdup(basename)); ++ if (-1 == getBasename((uchar*)basename, (uchar*)name)) { ++ CHKmalloc(act->basename = strdup(name)); /* assume basename is same as name */ ++ } else { ++ CHKmalloc(act->basename = strdup(basename)); ++ } + act->edge = edge; + act->ino = ino; ++ act->is_symlink = is_symlink; ++ if (source) { /* we are target of symlink */ ++ CHKmalloc(act->source_name = strdup(source)); ++ } else { ++ act->source_name = NULL; ++ } + #ifdef HAVE_INOTIFY_INIT + act->wd = in_setupWatch(act, is_file); + #endif + fen_setupWatch(act); +- if(is_file) { ++ if(is_file && !is_symlink) { + const instanceConf_t *const inst = edge->instarr[0];// TODO: same file, multiple instances? + CHKiRet(ratelimitNew(&act->ratelimiter, "imfile", name)); + CHKmalloc(act->multiSub.ppMsgs = MALLOC(inst->nMultiSub * sizeof(smsg_t *))); +@@ -702,27 +717,24 @@ detect_updates(fs_edge_t *const edge) + { + act_obj_t *act; + struct stat fileInfo; ++ int restart = 0; + +- for(act = edge->active ; act != NULL ; ) { ++ for(act = edge->active ; act != NULL ; act = act->next) { + DBGPRINTF("detect_updates checking active obj '%s'\n", act->name); +- const int r = stat(act->name, &fileInfo); ++ const int r = lstat(act->name, &fileInfo); + if(r == -1) { /* object gone away? */ + DBGPRINTF("object gone away, unlinking: '%s'\n", act->name); +- act_obj_t *toDel = act; +- act = act->next; +- DBGPRINTF("new next act %p\n", act); +- act_obj_unlink(toDel); +- continue; ++ act_obj_unlink(act); ++ restart = 1; ++ break; + } + // TODO: add inode check for change notification! + +- /* Note: active nodes may get deleted, so we need to do the +- * pointer advancement at the end of the for loop! +- */ +- act = act->next; + } + +- ++ if (restart) { ++ detect_updates(edge); ++ } + } + + +@@ -746,14 +758,52 @@ poll_active_files(fs_edge_t *const edge) + } + } + ++static rsRetVal ATTR_NONNULL() ++process_symlink(fs_edge_t *const chld, const char *symlink) ++{ ++ DEFiRet; ++ char *target = NULL; ++ CHKmalloc(target = realpath(symlink, target)); ++ struct stat fileInfo; ++ if(lstat(target, &fileInfo) != 0) { ++ LogError(errno, RS_RET_ERR, "imfile: process_symlink: cannot stat file '%s' - ignored", target); ++ FINALIZE; ++ } ++ const int is_file = (S_ISREG(fileInfo.st_mode)); ++ DBGPRINTF("process_symlink: found '%s', File: %d (config file: %d), symlink: %d\n", ++ target, is_file, chld->is_file, 0); ++ if (act_obj_add(chld, target, is_file, fileInfo.st_ino, 0, symlink) == RS_RET_OK) { ++ /* need to watch parent target as well for proper rotation support */ ++ uint idx = ustrlen(chld->active->name) - ustrlen(chld->active->basename); ++ if (idx) { /* basename is different from name */ ++ char parent[MAXFNAME]; ++ idx--; /* move past trailing slash */ ++ memcpy(parent, chld->active->name, idx); ++ parent[idx] = '\0'; ++ if(lstat(parent, &fileInfo) != 0) { ++ LogError(errno, RS_RET_ERR, ++ "imfile: process_symlink: cannot stat directory '%s' - ignored", parent); ++ FINALIZE; ++ } ++ if (chld->parent->root->edges) { ++ DBGPRINTF("process_symlink: adding parent '%s' of target '%s'\n", parent, target); ++ act_obj_add(chld->parent->root->edges, parent, 0, fileInfo.st_ino, 0, NULL); ++ } ++ } ++ } ++ ++finalize_it: ++ free(target); ++ RETiRet; ++} + +-static void ATTR_NONNULL() poll_tree(fs_edge_t *const chld); + static void ATTR_NONNULL() + poll_tree(fs_edge_t *const chld) + { + struct stat fileInfo; + glob_t files; + int need_globfree = 0; ++ int issymlink; + DBGPRINTF("poll_tree: chld %p, name '%s', path: %s\n", chld, chld->name, chld->path); + detect_updates(chld); + const int ret = glob((char*)chld->path, runModConf->sortFiles|GLOB_BRACE, NULL, &files); +@@ -766,18 +803,27 @@ poll_tree(fs_edge_t *const chld) + goto done; + } + char *const file = files.gl_pathv[i]; +- if(stat(file, &fileInfo) != 0) { ++ if(lstat(file, &fileInfo) != 0) { + LogError(errno, RS_RET_ERR, + "imfile: poll_tree cannot stat file '%s' - ignored", file); + continue; + } + +- const int is_file = S_ISREG(fileInfo.st_mode); +- DBGPRINTF("poll_tree: found '%s', File: %d (config file: %d)\n", +- file, is_file, chld->is_file); ++ if (S_ISLNK(fileInfo.st_mode)) { ++ rsRetVal slink_ret = process_symlink(chld, file); ++ if (slink_ret != RS_RET_OK) { ++ continue; ++ } ++ issymlink = 1; ++ } else { ++ issymlink = 0; ++ } ++ const int is_file = (S_ISREG(fileInfo.st_mode) || issymlink); ++ DBGPRINTF("poll_tree: found '%s', File: %d (config file: %d), symlink: %d\n", ++ file, is_file, chld->is_file, issymlink); + if(!is_file && S_ISREG(fileInfo.st_mode)) { + LogMsg(0, RS_RET_ERR, LOG_WARNING, +- "imfile: '%s' is neither a regular file nor a " ++ "imfile: '%s' is neither a regular file, symlink, nor a " + "directory - ignored", file); + continue; + } +@@ -788,7 +834,7 @@ poll_tree(fs_edge_t *const chld) + (chld->is_file) ? "FILE" : "DIRECTORY"); + continue; + } +- act_obj_add(chld, file, is_file, fileInfo.st_ino); ++ act_obj_add(chld, file, is_file, fileInfo.st_ino, issymlink, NULL); + } + } + +@@ -829,8 +875,20 @@ act_obj_destroy(act_obj_t *const act, const int is_deleted) + if(act == NULL) + return; + +- DBGPRINTF("act_obj_destroy: act %p '%s', wd %d, pStrm %p, is_deleted %d, in_move %d\n", +- act, act->name, act->wd, act->pStrm, is_deleted, act->in_move); ++ DBGPRINTF("act_obj_destroy: act %p '%s' (source '%s'), wd %d, pStrm %p, is_deleted %d, in_move %d\n", ++ act, act->name, act->source_name? act->source_name : "---", act->wd, act->pStrm, is_deleted, ++ act->in_move); ++ if(act->is_symlink && is_deleted) { ++ act_obj_t *target_act; ++ for(target_act = act->edge->active ; target_act != NULL ; target_act = target_act->next) { ++ if(target_act->source_name && !strcmp(target_act->source_name, act->name)) { ++ DBGPRINTF("act_obj_destroy: unlinking slink target %s of %s " ++ "symlink\n", target_act->name, act->name); ++ act_obj_unlink(target_act); ++ break; ++ } ++ } ++ } + if(act->ratelimiter != NULL) { + ratelimitDestruct(act->ratelimiter); + } +@@ -862,6 +920,7 @@ act_obj_destroy(act_obj_t *const act, const int is_deleted) + } + #endif + free(act->basename); ++ free(act->source_name); + //free(act->statefile); + free(act->multiSub.ppMsgs); + #if defined(OS_SOLARIS) && defined (HAVE_PORT_SOURCE_FILE) +@@ -909,7 +968,7 @@ chk_active(const act_obj_t *act, const act_obj_t *const deleted) + * destruct it. + */ + static void //ATTR_NONNULL() +-act_obj_unlink(act_obj_t *const act) ++act_obj_unlink(act_obj_t *act) + { + DBGPRINTF("act_obj_unlink %p: %s\n", act, act->name); + if(act->prev == NULL) { +@@ -921,6 +980,7 @@ act_obj_unlink(act_obj_t *const act) + act->next->prev = act->prev; + } + act_obj_destroy(act, 1); ++ act = NULL; + //dbgprintf("printout of fs tree post unlink\n"); + //fs_node_print(runModConf->conf_tree, 0); + //dbg_wdmapPrint("wdmap after"); +@@ -1025,6 +1038,7 @@ fs_node_walk(fs_node_t *const node, + */ + static rsRetVal + fs_node_add(fs_node_t *const node, ++ fs_node_t *const source, + const uchar *const toFind, + const size_t pathIdx, + instanceConf_t *const inst) +@@ -1053,6 +1067,7 @@ fs_node_add(fs_node_t *const node, + memcpy(name, toFind+pathIdx, len); + name[len] = '\0'; + DBGPRINTF("fs_node_add: name '%s'\n", name); ++ node->root = source; + + fs_edge_t *chld; + for(chld = node->edges ; chld != NULL ; chld = chld->next) { +@@ -1064,7 +1079,7 @@ fs_node_add(fs_node_t *const node, + chld->instarr[chld->ninst-1] = inst; + /* recurse */ + if(!isFile) { +- CHKiRet(fs_node_add(chld->node, toFind, nextPathIdx, inst)); ++ CHKiRet(fs_node_add(chld->node, node, toFind, nextPathIdx, inst)); + } + FINALIZE; + } +@@ -1086,7 +1101,7 @@ fs_node_add(fs_node_t *const node, + DBGPRINTF("fs_node_add(%p, '%s') returns %p\n", node, toFind, newchld->node); + + if(!isFile) { +- CHKiRet(fs_node_add(newchld->node, toFind, nextPathIdx, inst)); ++ CHKiRet(fs_node_add(newchld->node, node, toFind, nextPathIdx, inst)); + } + + /* link to list */ +@@ -1162,7 +1222,11 @@ enqLine(act_obj_t *const act, + msgSetPRI(pMsg, inst->iFacility | inst->iSeverity); + MsgSetRuleset(pMsg, inst->pBindRuleset); + if(inst->addMetadata) { +- metadata_values[0] = (const uchar*)act->name; ++ if (act->source_name) { ++ metadata_values[0] = (const uchar*)act->source_name; ++ } else { ++ metadata_values[0] = (const uchar*)act->name; ++ } + snprintf((char *)file_offset, MAX_OFFSET_REPRESENTATION_NUM_BYTES+1, "%lld", strtOffs); + metadata_values[1] = file_offset; + msgAddMultiMetadata(pMsg, metadata_names, metadata_values, 2); +@@ -1389,13 +1453,16 @@ pollFile(act_obj_t *const act) + { + cstr_t *pCStr = NULL; + DEFiRet; ++ if (act->is_symlink) { ++ FINALIZE; /* no reason to poll symlink file */ ++ } + /* Note: we must do pthread_cleanup_push() immediately, because the POSIX macros + * otherwise do not work if I include the _cleanup_pop() inside an if... -- rgerhards, 2008-08-14 + */ + pthread_cleanup_push(pollFileCancelCleanup, &pCStr); + iRet = pollFileReal(act, &pCStr); + pthread_cleanup_pop(0); +- RETiRet; ++finalize_it: RETiRet; + } + + +@@ -1931,7 +1946,7 @@ CODESTARTactivateCnf + "be processed. Reason", inst->pszFileName); + } + } +- fs_node_add(runModConf->conf_tree, inst->pszFileName, 0, inst); ++ fs_node_add(runModConf->conf_tree, NULL, inst->pszFileName, 0, inst); + } + + if(Debug) { +@@ -2031,6 +2113,9 @@ flag_in_move(fs_edge_t *const edge, const char *name_moved) + DBGPRINTF("name check fails, '%s' != '%s'\n", act->basename, name_moved); + } + } ++ if (!act && edge->next) { ++ flag_in_move(edge->next, name_moved); ++ } + } + + static void ATTR_NONNULL(1) +@@ -2057,7 +2145,7 @@ in_processEvent(struct inotify_event *ev) + } + if(ev->mask & (IN_MOVED_FROM | IN_MOVED_TO)) { + fs_node_walk(etry->act->edge->node, poll_tree); +- } else if(etry->act->edge->is_file) { ++ } else if(etry->act->edge->is_file && !(etry->act->is_symlink)) { + in_handleFileEvent(ev, etry); // esentially poll_file()! + } else { + fs_node_walk(etry->act->edge->node, poll_tree); diff --git a/SOURCES/rsyslog-8.37.0-rhbz1614181-imtcp-imudp-preservecase-option.patch b/SOURCES/rsyslog-8.37.0-rhbz1614181-imtcp-imudp-preservecase-option.patch new file mode 100644 index 0000000..e4c6e50 --- /dev/null +++ b/SOURCES/rsyslog-8.37.0-rhbz1614181-imtcp-imudp-preservecase-option.patch @@ -0,0 +1,286 @@ +From 9ac54f0d7d70b8a9879889b4522a1d552fca1100 Mon Sep 17 00:00:00 2001 +From: Noriko Hosoi +Date: Thu, 12 Jul 2018 11:52:04 -0700 +Subject: [PATCH] Introducing an option preservecase to imudp and imtcp module + for managing the case of FROMHOST value. + +Usage: +module(load="imudp" [preservecase="on"|"off"]) +module(load="imtdp" [preservecase="on"|"off"]) + +If preservecase="on", FROMHOST value is handled in the case sensitive manner. +If preservecase="off", FROMHOST value is handled in the case insensitive manner. + +To maintain the current behaviour, the default value of preservecase is +"on" for imtcp and "off" for imudp. + +Incremented tcpsrvCURR_IF_VERSION by 1. + +References: +https://github.com/rsyslog/rsyslog/pull/2774 +https://bugzilla.redhat.com/show_bug.cgi?id=1309698 +--- + plugins/imtcp/imtcp.c | 14 ++++++++++++-- + plugins/imudp/imudp.c | 15 ++++++++++++--- + runtime/msg.c | 6 +++++- + runtime/msg.h | 2 ++ + runtime/net.c | 2 +- + runtime/tcpsrv.c | 21 +++++++++++++++++++++ + runtime/tcpsrv.h | 5 ++++- + 7 files changed, 57 insertions(+), 8 deletions(-) + +diff --git a/plugins/imtcp/imtcp.c b/plugins/imtcp/imtcp.c +index 8e3dcc0a2..45fa240b5 100644 +--- a/plugins/imtcp/imtcp.c ++++ b/plugins/imtcp/imtcp.c +@@ -100,6 +100,7 @@ static struct configSettings_s { + int bDisableLFDelim; + int discardTruncatedMsg; + int bUseFlowControl; ++ int bPreserveCase; + uchar *gnutlsPriorityString; + uchar *pszStrmDrvrAuthMode; + uchar *pszInputName; +@@ -144,6 +145,7 @@ struct modConfData_s { + uchar *pszStrmDrvrAuthMode; /* authentication mode to use */ + struct cnfarray *permittedPeers; + sbool configSetViaV2Method; ++ sbool bPreserveCase; /* preserve case of fromhost; true by default */ + }; + + static modConfData_t *loadModConf = NULL;/* modConf ptr to use for the current load process */ +@@ -169,7 +171,8 @@ static struct cnfparamdescr modpdescr[] = { + { "keepalive.probes", eCmdHdlrPositiveInt, 0 }, + { "keepalive.time", eCmdHdlrPositiveInt, 0 }, + { "keepalive.interval", eCmdHdlrPositiveInt, 0 }, +- { "gnutlsprioritystring", eCmdHdlrString, 0 } ++ { "gnutlsprioritystring", eCmdHdlrString, 0 }, ++ { "preservecase", eCmdHdlrBinary, 0 } + }; + static struct cnfparamblk modpblk = + { CNFPARAMBLK_VERSION, +@@ -375,6 +378,7 @@ addListner(modConfData_t *modConf, instanceConf_t *inst) + if(pPermPeersRoot != NULL) { + CHKiRet(tcpsrv.SetDrvrPermPeers(pOurTcpsrv, pPermPeersRoot)); + } ++ CHKiRet(tcpsrv.SetPreserveCase(pOurTcpsrv, modConf->bPreserveCase)); + } + + /* initialized, now add socket and listener params */ +@@ -473,6 +477,7 @@ CODESTARTbeginCnfLoad + loadModConf->pszStrmDrvrAuthMode = NULL; + loadModConf->permittedPeers = NULL; + loadModConf->configSetViaV2Method = 0; ++ loadModConf->bPreserveCase = 1; /* default to true */ + bLegacyCnfModGlobalsPermitted = 1; + /* init legacy config variables */ + cs.pszStrmDrvrAuthMode = NULL; +@@ -543,6 +548,8 @@ CODESTARTsetModCnf + loadModConf->pszStrmDrvrName = (uchar*)es_str2cstr(pvals[i].val.d.estr, NULL); + } else if(!strcmp(modpblk.descr[i].name, "permittedpeer")) { + loadModConf->permittedPeers = cnfarrayDup(pvals[i].val.d.ar); ++ } else if(!strcmp(modpblk.descr[i].name, "preservecase")) { ++ loadModConf->bPreserveCase = (int) pvals[i].val.d.n; + } else { + dbgprintf("imtcp: program error, non-handled " + "param '%s' in beginCnfLoad\n", modpblk.descr[i].name); +@@ -584,6 +591,7 @@ CODESTARTendCnfLoad + loadModConf->pszStrmDrvrAuthMode = cs.pszStrmDrvrAuthMode; + cs.pszStrmDrvrAuthMode = NULL; + } ++ pModConf->bPreserveCase = cs.bPreserveCase; + } + free(cs.pszStrmDrvrAuthMode); + cs.pszStrmDrvrAuthMode = NULL; +@@ -731,6 +739,7 @@ resetConfigVariables(uchar __attribute__((unused)) *pp, void __attribute__((unus + cs.pszInputName = NULL; + free(cs.pszStrmDrvrAuthMode); + cs.pszStrmDrvrAuthMode = NULL; ++ cs.bPreserveCase = 1; + return RS_RET_OK; + } + +@@ -797,7 +806,8 @@ CODEmodInit_QueryRegCFSLineHdlr + NULL, &cs.bEmitMsgOnClose, STD_LOADABLE_MODULE_ID, &bLegacyCnfModGlobalsPermitted)); + CHKiRet(regCfSysLineHdlr2(UCHAR_CONSTANT("inputtcpserverstreamdrivermode"), 0, eCmdHdlrInt, + NULL, &cs.iStrmDrvrMode, STD_LOADABLE_MODULE_ID, &bLegacyCnfModGlobalsPermitted)); +- ++ CHKiRet(regCfSysLineHdlr2(UCHAR_CONSTANT("inputtcpserverpreservecase"), 1, eCmdHdlrBinary, ++ NULL, &cs.bPreserveCase, STD_LOADABLE_MODULE_ID, &bLegacyCnfModGlobalsPermitted)); + CHKiRet(omsdRegCFSLineHdlr(UCHAR_CONSTANT("resetconfigvariables"), 1, eCmdHdlrCustomHandler, + resetConfigVariables, NULL, STD_LOADABLE_MODULE_ID)); + ENDmodInit +diff --git a/plugins/imudp/imudp.c b/plugins/imudp/imudp.c +index 51a9d712a..74437781c 100644 +--- a/plugins/imudp/imudp.c ++++ b/plugins/imudp/imudp.c +@@ -152,6 +152,7 @@ struct modConfData_s { + int batchSize; /* max nbr of input batch --> also recvmmsg() max count */ + int8_t wrkrMax; /* max nbr of worker threads */ + sbool configSetViaV2Method; ++ sbool bPreserveCase; /* preserves the case of fromhost; "off" by default */ + }; + static modConfData_t *loadModConf = NULL;/* modConf ptr to use for the current load process */ + static modConfData_t *runModConf = NULL;/* modConf ptr to use for the current load process */ +@@ -162,7 +163,8 @@ static struct cnfparamdescr modpdescr[] = { + { "schedulingpriority", eCmdHdlrInt, 0 }, + { "batchsize", eCmdHdlrInt, 0 }, + { "threads", eCmdHdlrPositiveInt, 0 }, +- { "timerequery", eCmdHdlrInt, 0 } ++ { "timerequery", eCmdHdlrInt, 0 }, ++ { "preservecase", eCmdHdlrBinary, 0 } + }; + static struct cnfparamblk modpblk = + { CNFPARAMBLK_VERSION, +@@ -447,8 +449,12 @@ processPacket(struct lstn_s *lstn, struct sockaddr_storage *frominetPrev, int *p + if(lstn->dfltTZ != NULL) + MsgSetDfltTZ(pMsg, (char*) lstn->dfltTZ); + pMsg->msgFlags = NEEDS_PARSING | PARSE_HOSTNAME | NEEDS_DNSRESOL; +- if(*pbIsPermitted == 2) +- pMsg->msgFlags |= NEEDS_ACLCHK_U; /* request ACL check after resolution */ ++ if(*pbIsPermitted == 2) { ++ pMsg->msgFlags |= NEEDS_ACLCHK_U; /* request ACL check after resolution */ ++ } ++ if(runModConf->bPreserveCase) { ++ pMsg->msgFlags |= PRESERVE_CASE; /* preserve case of fromhost */ ++ } + CHKiRet(msgSetFromSockinfo(pMsg, frominet)); + CHKiRet(ratelimitAddMsg(lstn->ratelimiter, multiSub, pMsg)); + STATSCOUNTER_INC(lstn->ctrSubmit, lstn->mutCtrSubmit); +@@ -1030,6 +1036,7 @@ CODESTARTbeginCnfLoad + loadModConf->iTimeRequery = TIME_REQUERY_DFLT; + loadModConf->iSchedPrio = SCHED_PRIO_UNSET; + loadModConf->pszSchedPolicy = NULL; ++ loadModConf->bPreserveCase = 0; /* off */ + bLegacyCnfModGlobalsPermitted = 1; + /* init legacy config vars */ + cs.pszBindRuleset = NULL; +@@ -1079,6 +1086,8 @@ CODESTARTsetModCnf + } else { + loadModConf->wrkrMax = wrkrMax; + } ++ } else if(!strcmp(modpblk.descr[i].name, "preservecase")) { ++ loadModConf->bPreserveCase = (int) pvals[i].val.d.n; + } else { + dbgprintf("imudp: program error, non-handled " + "param '%s' in beginCnfLoad\n", modpblk.descr[i].name); +diff --git a/runtime/msg.c b/runtime/msg.c +index c43f81314..9ed4eaf84 100644 +--- a/runtime/msg.c ++++ b/runtime/msg.c +@@ -506,7 +506,11 @@ resolveDNS(smsg_t * const pMsg) { + MsgLock(pMsg); + CHKiRet(objUse(net, CORE_COMPONENT)); + if(pMsg->msgFlags & NEEDS_DNSRESOL) { +- localRet = net.cvthname(pMsg->rcvFrom.pfrominet, &localName, NULL, &ip); ++ if (pMsg->msgFlags & PRESERVE_CASE) { ++ localRet = net.cvthname(pMsg->rcvFrom.pfrominet, NULL, &localName, &ip); ++ } else { ++ localRet = net.cvthname(pMsg->rcvFrom.pfrominet, &localName, NULL, &ip); ++ } + if(localRet == RS_RET_OK) { + /* we pass down the props, so no need for AddRef */ + MsgSetRcvFromWithoutAddRef(pMsg, localName); +diff --git a/runtime/msg.h b/runtime/msg.h +index cd530aca3..1287cb7a4 100644 +--- a/runtime/msg.h ++++ b/runtime/msg.h +@@ -156,6 +156,8 @@ struct msg { + /* check UDP ACLs after DNS resolution has been done in main queue consumer */ + #define NO_PRI_IN_RAW 0x100 + /* rawmsg does not include a PRI (Solaris!), but PRI is already set correctly in the msg object */ ++#define PRESERVE_CASE 0x200 ++/* preserve case in fromhost */ + + /* (syslog) protocol types */ + #define MSG_LEGACY_PROTOCOL 0 +diff --git a/runtime/net.c b/runtime/net.c +index d6ff8a3d4..aef906601 100644 +--- a/runtime/net.c ++++ b/runtime/net.c +@@ -1152,7 +1152,7 @@ cvthname(struct sockaddr_storage *f, prop_t **localName, prop_t **fqdn, prop_t * + { + DEFiRet; + assert(f != NULL); +- iRet = dnscacheLookup(f, NULL, fqdn, localName, ip); ++ iRet = dnscacheLookup(f, fqdn, NULL, localName, ip); + RETiRet; + } + +diff --git a/runtime/tcpsrv.c b/runtime/tcpsrv.c +index 61e9ff4d2..d5993b4f0 100644 +--- a/runtime/tcpsrv.c ++++ b/runtime/tcpsrv.c +@@ -495,6 +495,15 @@ SessAccept(tcpsrv_t *pThis, tcpLstnPortList_t *pLstnInfo, tcps_sess_t **ppSess, + + /* get the host name */ + CHKiRet(netstrm.GetRemoteHName(pNewStrm, &fromHostFQDN)); ++ if (!pThis->bPreserveCase) { ++ /* preserve_case = off */ ++ uchar *p; ++ for(p = fromHostFQDN; *p; p++) { ++ if (isupper((int) *p)) { ++ *p = tolower((int) *p); ++ } ++ } ++ } + CHKiRet(netstrm.GetRemoteIP(pNewStrm, &fromHostIP)); + CHKiRet(netstrm.GetRemAddr(pNewStrm, &addr)); + /* TODO: check if we need to strip the domain name here -- rgerhards, 2008-04-24 */ +@@ -1001,6 +1010,7 @@ BEGINobjConstruct(tcpsrv) /* be sure to specify the object type also in END macr + pThis->ratelimitBurst = 10000; + pThis->bUseFlowControl = 1; + pThis->pszDrvrName = NULL; ++ pThis->bPreserveCase = 1; /* preserve case in fromhost; default to true. */ + ENDobjConstruct(tcpsrv) + + +@@ -1433,6 +1443,16 @@ SetSessMax(tcpsrv_t *pThis, int iMax) + } + + ++static rsRetVal ++SetPreserveCase(tcpsrv_t *pThis, int bPreserveCase) ++{ ++ DEFiRet; ++ ISOBJ_TYPE_assert(pThis, tcpsrv); ++ pThis-> bPreserveCase = bPreserveCase; ++ RETiRet; ++} ++ ++ + /* queryInterface function + * rgerhards, 2008-02-29 + */ +@@ -1491,6 +1511,7 @@ CODESTARTobjQueryInterface(tcpsrv) + pIf->SetRuleset = SetRuleset; + pIf->SetLinuxLikeRatelimiters = SetLinuxLikeRatelimiters; + pIf->SetNotificationOnRemoteClose = SetNotificationOnRemoteClose; ++ pIf->SetPreserveCase = SetPreserveCase; + + finalize_it: + ENDobjQueryInterface(tcpsrv) +diff --git a/runtime/tcpsrv.h b/runtime/tcpsrv.h +index 22a65c20a..f17b1b438 100644 +--- a/runtime/tcpsrv.h ++++ b/runtime/tcpsrv.h +@@ -85,6 +85,7 @@ struct tcpsrv_s { + int maxFrameSize; /**< max frame size for octet counted*/ + int bDisableLFDelim; /**< if 1, standard LF frame delimiter is disabled (*very dangerous*) */ + int discardTruncatedMsg;/**< discard msg part that has been truncated*/ ++ sbool bPreserveCase; /**< preserve case in fromhost */ + int ratelimitInterval; + int ratelimitBurst; + tcps_sess_t **pSessions;/**< array of all of our sessions */ +@@ -177,8 +178,10 @@ BEGINinterface(tcpsrv) /* name must also be changed in ENDinterface macro! */ + rsRetVal (*SetbSPFramingFix)(tcpsrv_t*, sbool); + /* added v19 -- PascalWithopf, 2017-08-08 */ + rsRetVal (*SetGnutlsPriorityString)(tcpsrv_t*, uchar*); ++ /* added v21 -- Preserve case in fromhost, 2018-08-16 */ ++ rsRetVal (*SetPreserveCase)(tcpsrv_t *pThis, int bPreserveCase); + ENDinterface(tcpsrv) +-#define tcpsrvCURR_IF_VERSION 20 /* increment whenever you change the interface structure! */ ++#define tcpsrvCURR_IF_VERSION 21 /* increment whenever you change the interface structure! */ + /* change for v4: + * - SetAddtlFrameDelim() added -- rgerhards, 2008-12-10 + * - SetInputName() added -- rgerhards, 2008-12-10 diff --git a/SOURCES/rsyslog-8.37.0-rhbz1622768-kubernetes-404-handling.patch b/SOURCES/rsyslog-8.37.0-rhbz1622768-kubernetes-404-handling.patch new file mode 100644 index 0000000..9b6db7b --- /dev/null +++ b/SOURCES/rsyslog-8.37.0-rhbz1622768-kubernetes-404-handling.patch @@ -0,0 +1,761 @@ +From 3987cd929d859f900318b393133c3bdde8dfffd5 Mon Sep 17 00:00:00 2001 +From: Rich Megginson +Date: Tue, 28 Aug 2018 12:44:23 -0600 +Subject: [PATCH] mmkubertnetes: action fails preparation cycle if kubernetes + API destroys resource during bootup sequence + +The plugin was not handling 404 Not Found correctly when looking +up pods and namespaces. In this case, we assume the pod/namespace +was deleted, annotate the record with whatever metadata we have, +and cache the fact that the pod/namespace is missing so we don't +attempt to look it up again. +In addition, the plugin was not handling error 429 Busy correctly. +In this case, it should also annotate the record with whatever +metadata it has, and _not_ cache anything. By default the plugin +will retry every 5 seconds to connect to Kubernetes. This +behavior is controlled by the new config param `busyretryinterval`. +This commit also adds impstats counters so that admins can +view the state of the plugin to see if the lookups are working +or are returning errors. The stats are reported per-instance +or per-action to facilitate using multiple different actions +for different Kubernetes servers. +This commit also adds support for client cert auth to +Kubernetes via the two new config params `tls.mycert` and +`tls.myprivkey`. +--- + contrib/mmkubernetes/mmkubernetes.c | 296 ++++++++++++++++++++++++---- + 7 files changed, 160 insertions(+), 36 deletions(-) + +diff --git a/contrib/mmkubernetes/mmkubernetes.c b/contrib/mmkubernetes/mmkubernetes.c +index 422cb2577..5bf5b049d 100644 +--- a/contrib/mmkubernetes/mmkubernetes.c ++++ b/contrib/mmkubernetes/mmkubernetes.c +@@ -52,9 +52,12 @@ + #include "syslogd-types.h" + #include "module-template.h" + #include "errmsg.h" ++#include "statsobj.h" + #include "regexp.h" + #include "hashtable.h" + #include "srUtils.h" ++#include "unicode-helper.h" ++#include "datetime.h" + + /* static data */ + MODULE_TYPE_OUTPUT /* this is technically an output plugin */ +@@ -62,6 +65,8 @@ MODULE_TYPE_KEEP /* releasing the module would cause a leak through libcurl */ + MODULE_CNFNAME("mmkubernetes") + DEF_OMOD_STATIC_DATA + DEFobjCurrIf(regexp) ++DEFobjCurrIf(statsobj) ++DEFobjCurrIf(datetime) + + #define HAVE_LOADSAMPLESFROMSTRING 1 + #if defined(NO_LOADSAMPLESFROMSTRING) +@@ -95,12 +100,14 @@ DEFobjCurrIf(regexp) + #define DFLT_CONTAINER_NAME "$!CONTAINER_NAME" /* name of variable holding CONTAINER_NAME value */ + #define DFLT_CONTAINER_ID_FULL "$!CONTAINER_ID_FULL" /* name of variable holding CONTAINER_ID_FULL value */ + #define DFLT_KUBERNETES_URL "https://kubernetes.default.svc.cluster.local:443" ++#define DFLT_BUSY_RETRY_INTERVAL 5 /* retry every 5 seconds */ + + static struct cache_s { + const uchar *kbUrl; + struct hashtable *mdHt; + struct hashtable *nsHt; + pthread_mutex_t *cacheMtx; ++ int lastBusyTime; + } **caches; + + typedef struct { +@@ -116,6 +123,8 @@ struct modConfData_s { + uchar *srcMetadataPath; /* where to get data for kubernetes queries */ + uchar *dstMetadataPath; /* where to put metadata obtained from kubernetes */ + uchar *caCertFile; /* File holding the CA cert (+optional chain) of CA that issued the Kubernetes server cert */ ++ uchar *myCertFile; /* File holding cert corresponding to private key used for client cert auth */ ++ uchar *myPrivKeyFile; /* File holding private key corresponding to cert used for client cert auth */ + sbool allowUnsignedCerts; /* For testing/debugging - do not check for CA certs (CURLOPT_SSL_VERIFYPEER FALSE) */ + uchar *token; /* The token value to use to authenticate to Kubernetes - takes precedence over tokenFile */ + uchar *tokenFile; /* The file whose contents is the token value to use to authenticate to Kubernetes */ +@@ -127,6 +136,7 @@ struct modConfData_s { + uchar *fnRulebase; /* lognorm rulebase filename for container log filename match */ + char *contRules; /* lognorm rules for CONTAINER_NAME value match */ + uchar *contRulebase; /* lognorm rulebase filename for CONTAINER_NAME value match */ ++ int busyRetryInterval; /* how to handle 429 response - 0 means error, non-zero means retry every N seconds */ + }; + + /* action (instance) configuration data */ +@@ -135,6 +145,8 @@ typedef struct _instanceData { + msgPropDescr_t *srcMetadataDescr; /* where to get data for kubernetes queries */ + uchar *dstMetadataPath; /* where to put metadata obtained from kubernetes */ + uchar *caCertFile; /* File holding the CA cert (+optional chain) of CA that issued the Kubernetes server cert */ ++ uchar *myCertFile; /* File holding cert corresponding to private key used for client cert auth */ ++ uchar *myPrivKeyFile; /* File holding private key corresponding to cert used for client cert auth */ + sbool allowUnsignedCerts; /* For testing/debugging - do not check for CA certs (CURLOPT_SSL_VERIFYPEER FALSE) */ + uchar *token; /* The token value to use to authenticate to Kubernetes - takes precedence over tokenFile */ + uchar *tokenFile; /* The file whose contents is the token value to use to authenticate to Kubernetes */ +@@ -151,6 +163,7 @@ typedef struct _instanceData { + msgPropDescr_t *contNameDescr; /* CONTAINER_NAME field */ + msgPropDescr_t *contIdFullDescr; /* CONTAINER_ID_FULL field */ + struct cache_s *cache; ++ int busyRetryInterval; /* how to handle 429 response - 0 means error, non-zero means retry every N seconds */ + } instanceData; + + typedef struct wrkrInstanceData { +@@ -159,6 +172,16 @@ typedef struct wrkrInstanceData { + struct curl_slist *curlHdr; + char *curlRply; + size_t curlRplyLen; ++ statsobj_t *stats; /* stats for this instance */ ++ STATSCOUNTER_DEF(k8sRecordSeen, mutK8sRecordSeen) ++ STATSCOUNTER_DEF(namespaceMetadataSuccess, mutNamespaceMetadataSuccess) ++ STATSCOUNTER_DEF(namespaceMetadataNotFound, mutNamespaceMetadataNotFound) ++ STATSCOUNTER_DEF(namespaceMetadataBusy, mutNamespaceMetadataBusy) ++ STATSCOUNTER_DEF(namespaceMetadataError, mutNamespaceMetadataError) ++ STATSCOUNTER_DEF(podMetadataSuccess, mutPodMetadataSuccess) ++ STATSCOUNTER_DEF(podMetadataNotFound, mutPodMetadataNotFound) ++ STATSCOUNTER_DEF(podMetadataBusy, mutPodMetadataBusy) ++ STATSCOUNTER_DEF(podMetadataError, mutPodMetadataError) + } wrkrInstanceData_t; + + /* module parameters (v6 config format) */ +@@ -167,6 +190,8 @@ static struct cnfparamdescr modpdescr[] = { + { "srcmetadatapath", eCmdHdlrString, 0 }, + { "dstmetadatapath", eCmdHdlrString, 0 }, + { "tls.cacert", eCmdHdlrString, 0 }, ++ { "tls.mycert", eCmdHdlrString, 0 }, ++ { "tls.myprivkey", eCmdHdlrString, 0 }, + { "allowunsignedcerts", eCmdHdlrBinary, 0 }, + { "token", eCmdHdlrString, 0 }, + { "tokenfile", eCmdHdlrString, 0 }, +@@ -174,7 +199,8 @@ static struct cnfparamdescr modpdescr[] = { + { "de_dot", eCmdHdlrBinary, 0 }, + { "de_dot_separator", eCmdHdlrString, 0 }, + { "filenamerulebase", eCmdHdlrString, 0 }, +- { "containerrulebase", eCmdHdlrString, 0 } ++ { "containerrulebase", eCmdHdlrString, 0 }, ++ { "busyretryinterval", eCmdHdlrInt, 0 } + #if HAVE_LOADSAMPLESFROMSTRING == 1 + , + { "filenamerules", eCmdHdlrArray, 0 }, +@@ -193,6 +219,8 @@ static struct cnfparamdescr actpdescr[] = { + { "srcmetadatapath", eCmdHdlrString, 0 }, + { "dstmetadatapath", eCmdHdlrString, 0 }, + { "tls.cacert", eCmdHdlrString, 0 }, ++ { "tls.mycert", eCmdHdlrString, 0 }, ++ { "tls.myprivkey", eCmdHdlrString, 0 }, + { "allowunsignedcerts", eCmdHdlrBinary, 0 }, + { "token", eCmdHdlrString, 0 }, + { "tokenfile", eCmdHdlrString, 0 }, +@@ -200,7 +228,8 @@ static struct cnfparamdescr actpdescr[] = { + { "de_dot", eCmdHdlrBinary, 0 }, + { "de_dot_separator", eCmdHdlrString, 0 }, + { "filenamerulebase", eCmdHdlrString, 0 }, +- { "containerrulebase", eCmdHdlrString, 0 } ++ { "containerrulebase", eCmdHdlrString, 0 }, ++ { "busyretryinterval", eCmdHdlrInt, 0 } + #if HAVE_LOADSAMPLESFROMSTRING == 1 + , + { "filenamerules", eCmdHdlrArray, 0 }, +@@ -493,8 +522,9 @@ ENDbeginCnfLoad + BEGINsetModCnf + struct cnfparamvals *pvals = NULL; + int i; +- FILE *fp; ++ FILE *fp = NULL; + int ret; ++ char errStr[1024]; + CODESTARTsetModCnf + pvals = nvlstGetParams(lst, &modpblk, NULL); + if(pvals == NULL) { +@@ -509,6 +539,7 @@ CODESTARTsetModCnf + } + + loadModConf->de_dot = DFLT_DE_DOT; ++ loadModConf->busyRetryInterval = DFLT_BUSY_RETRY_INTERVAL; + for(i = 0 ; i < modpblk.nParams ; ++i) { + if(!pvals[i].bUsed) { + continue; +@@ -528,15 +559,42 @@ CODESTARTsetModCnf + loadModConf->caCertFile = (uchar *) es_str2cstr(pvals[i].val.d.estr, NULL); + fp = fopen((const char*)loadModConf->caCertFile, "r"); + if(fp == NULL) { +- char errStr[1024]; + rs_strerror_r(errno, errStr, sizeof(errStr)); + iRet = RS_RET_NO_FILE_ACCESS; + LogError(0, iRet, +- "error: certificate file %s couldn't be accessed: %s\n", ++ "error: 'tls.cacert' file %s couldn't be accessed: %s\n", + loadModConf->caCertFile, errStr); + ABORT_FINALIZE(iRet); + } else { + fclose(fp); ++ fp = NULL; ++ } ++ } else if(!strcmp(modpblk.descr[i].name, "tls.mycert")) { ++ free(loadModConf->myCertFile); ++ loadModConf->myCertFile = (uchar*)es_str2cstr(pvals[i].val.d.estr, NULL); ++ fp = fopen((const char*)loadModConf->myCertFile, "r"); ++ if(fp == NULL) { ++ rs_strerror_r(errno, errStr, sizeof(errStr)); ++ iRet = RS_RET_NO_FILE_ACCESS; ++ LogError(0, iRet, ++ "error: 'tls.mycert' file %s couldn't be accessed: %s\n", ++ loadModConf->myCertFile, errStr); ++ } else { ++ fclose(fp); ++ fp = NULL; ++ } ++ } else if(!strcmp(modpblk.descr[i].name, "tls.myprivkey")) { ++ loadModConf->myPrivKeyFile = (uchar*)es_str2cstr(pvals[i].val.d.estr, NULL); ++ fp = fopen((const char*)loadModConf->myPrivKeyFile, "r"); ++ if(fp == NULL) { ++ rs_strerror_r(errno, errStr, sizeof(errStr)); ++ iRet = RS_RET_NO_FILE_ACCESS; ++ LogError(0, iRet, ++ "error: 'tls.myprivkey' file %s couldn't be accessed: %s\n", ++ loadModConf->myPrivKeyFile, errStr); ++ } else { ++ fclose(fp); ++ fp = NULL; + } + } else if(!strcmp(modpblk.descr[i].name, "allowunsignedcerts")) { + loadModConf->allowUnsignedCerts = pvals[i].val.d.n; +@@ -548,7 +606,6 @@ CODESTARTsetModCnf + loadModConf->tokenFile = (uchar *) es_str2cstr(pvals[i].val.d.estr, NULL); + fp = fopen((const char*)loadModConf->tokenFile, "r"); + if(fp == NULL) { +- char errStr[1024]; + rs_strerror_r(errno, errStr, sizeof(errStr)); + iRet = RS_RET_NO_FILE_ACCESS; + LogError(0, iRet, +@@ -557,6 +614,7 @@ CODESTARTsetModCnf + ABORT_FINALIZE(iRet); + } else { + fclose(fp); ++ fp = NULL; + } + } else if(!strcmp(modpblk.descr[i].name, "annotation_match")) { + free_annotationmatch(&loadModConf->annotation_match); +@@ -577,7 +635,6 @@ CODESTARTsetModCnf + loadModConf->fnRulebase = (uchar *) es_str2cstr(pvals[i].val.d.estr, NULL); + fp = fopen((const char*)loadModConf->fnRulebase, "r"); + if(fp == NULL) { +- char errStr[1024]; + rs_strerror_r(errno, errStr, sizeof(errStr)); + iRet = RS_RET_NO_FILE_ACCESS; + LogError(0, iRet, +@@ -586,6 +643,7 @@ CODESTARTsetModCnf + ABORT_FINALIZE(iRet); + } else { + fclose(fp); ++ fp = NULL; + } + #if HAVE_LOADSAMPLESFROMSTRING == 1 + } else if(!strcmp(modpblk.descr[i].name, "containerrules")) { +@@ -597,7 +655,6 @@ CODESTARTsetModCnf + loadModConf->contRulebase = (uchar *) es_str2cstr(pvals[i].val.d.estr, NULL); + fp = fopen((const char*)loadModConf->contRulebase, "r"); + if(fp == NULL) { +- char errStr[1024]; + rs_strerror_r(errno, errStr, sizeof(errStr)); + iRet = RS_RET_NO_FILE_ACCESS; + LogError(0, iRet, +@@ -606,7 +663,10 @@ CODESTARTsetModCnf + ABORT_FINALIZE(iRet); + } else { + fclose(fp); ++ fp = NULL; + } ++ } else if(!strcmp(modpblk.descr[i].name, "busyretryinterval")) { ++ loadModConf->busyRetryInterval = pvals[i].val.d.n; + } else { + dbgprintf("mmkubernetes: program error, non-handled " + "param '%s' in module() block\n", modpblk.descr[i].name); +@@ -650,6 +710,8 @@ CODESTARTsetModCnf + caches = calloc(1, sizeof(struct cache_s *)); + + finalize_it: ++ if (fp) ++ fclose(fp); + if(pvals != NULL) + cnfparamvalsDestruct(pvals, &modpblk); + ENDsetModCnf +@@ -667,6 +729,8 @@ CODESTARTfreeInstance + free(pData->srcMetadataDescr); + free(pData->dstMetadataPath); + free(pData->caCertFile); ++ free(pData->myCertFile); ++ free(pData->myPrivKeyFile); + free(pData->token); + free(pData->tokenFile); + free(pData->fnRules); +@@ -710,6 +774,45 @@ CODESTARTcreateWrkrInstance + char *tokenHdr = NULL; + FILE *fp = NULL; + char *token = NULL; ++ char *statsName = NULL; ++ ++ CHKiRet(statsobj.Construct(&(pWrkrData->stats))); ++ if ((-1 == asprintf(&statsName, "mmkubernetes(%s)", pWrkrData->pData->kubernetesUrl)) || ++ (!statsName)) { ++ ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); ++ } ++ CHKiRet(statsobj.SetName(pWrkrData->stats, (uchar *)statsName)); ++ free(statsName); ++ statsName = NULL; ++ CHKiRet(statsobj.SetOrigin(pWrkrData->stats, UCHAR_CONSTANT("mmkubernetes"))); ++ STATSCOUNTER_INIT(pWrkrData->k8sRecordSeen, pWrkrData->mutK8sRecordSeen); ++ CHKiRet(statsobj.AddCounter(pWrkrData->stats, UCHAR_CONSTANT("recordseen"), ++ ctrType_IntCtr, CTR_FLAG_RESETTABLE, &(pWrkrData->k8sRecordSeen))); ++ STATSCOUNTER_INIT(pWrkrData->namespaceMetadataSuccess, pWrkrData->mutNamespaceMetadataSuccess); ++ CHKiRet(statsobj.AddCounter(pWrkrData->stats, UCHAR_CONSTANT("namespacemetadatasuccess"), ++ ctrType_IntCtr, CTR_FLAG_RESETTABLE, &(pWrkrData->namespaceMetadataSuccess))); ++ STATSCOUNTER_INIT(pWrkrData->namespaceMetadataNotFound, pWrkrData->mutNamespaceMetadataNotFound); ++ CHKiRet(statsobj.AddCounter(pWrkrData->stats, UCHAR_CONSTANT("namespacemetadatanotfound"), ++ ctrType_IntCtr, CTR_FLAG_RESETTABLE, &(pWrkrData->namespaceMetadataNotFound))); ++ STATSCOUNTER_INIT(pWrkrData->namespaceMetadataBusy, pWrkrData->mutNamespaceMetadataBusy); ++ CHKiRet(statsobj.AddCounter(pWrkrData->stats, UCHAR_CONSTANT("namespacemetadatabusy"), ++ ctrType_IntCtr, CTR_FLAG_RESETTABLE, &(pWrkrData->namespaceMetadataBusy))); ++ STATSCOUNTER_INIT(pWrkrData->namespaceMetadataError, pWrkrData->mutNamespaceMetadataError); ++ CHKiRet(statsobj.AddCounter(pWrkrData->stats, UCHAR_CONSTANT("namespacemetadataerror"), ++ ctrType_IntCtr, CTR_FLAG_RESETTABLE, &(pWrkrData->namespaceMetadataError))); ++ STATSCOUNTER_INIT(pWrkrData->podMetadataSuccess, pWrkrData->mutPodMetadataSuccess); ++ CHKiRet(statsobj.AddCounter(pWrkrData->stats, UCHAR_CONSTANT("podmetadatasuccess"), ++ ctrType_IntCtr, CTR_FLAG_RESETTABLE, &(pWrkrData->podMetadataSuccess))); ++ STATSCOUNTER_INIT(pWrkrData->podMetadataNotFound, pWrkrData->mutPodMetadataNotFound); ++ CHKiRet(statsobj.AddCounter(pWrkrData->stats, UCHAR_CONSTANT("podmetadatanotfound"), ++ ctrType_IntCtr, CTR_FLAG_RESETTABLE, &(pWrkrData->podMetadataNotFound))); ++ STATSCOUNTER_INIT(pWrkrData->podMetadataBusy, pWrkrData->mutPodMetadataBusy); ++ CHKiRet(statsobj.AddCounter(pWrkrData->stats, UCHAR_CONSTANT("podmetadatabusy"), ++ ctrType_IntCtr, CTR_FLAG_RESETTABLE, &(pWrkrData->podMetadataBusy))); ++ STATSCOUNTER_INIT(pWrkrData->podMetadataError, pWrkrData->mutPodMetadataError); ++ CHKiRet(statsobj.AddCounter(pWrkrData->stats, UCHAR_CONSTANT("podmetadataerror"), ++ ctrType_IntCtr, CTR_FLAG_RESETTABLE, &(pWrkrData->podMetadataError))); ++ CHKiRet(statsobj.ConstructFinalize(pWrkrData->stats)); + + hdr = curl_slist_append(hdr, "Content-Type: text/json; charset=utf-8"); + if (pWrkrData->pData->token) { +@@ -749,12 +852,20 @@ CODESTARTcreateWrkrInstance + curl_easy_setopt(ctx, CURLOPT_WRITEDATA, pWrkrData); + if(pWrkrData->pData->caCertFile) + curl_easy_setopt(ctx, CURLOPT_CAINFO, pWrkrData->pData->caCertFile); ++ if(pWrkrData->pData->myCertFile) ++ curl_easy_setopt(ctx, CURLOPT_SSLCERT, pWrkrData->pData->myCertFile); ++ if(pWrkrData->pData->myPrivKeyFile) ++ curl_easy_setopt(ctx, CURLOPT_SSLKEY, pWrkrData->pData->myPrivKeyFile); + if(pWrkrData->pData->allowUnsignedCerts) + curl_easy_setopt(ctx, CURLOPT_SSL_VERIFYPEER, 0); + + pWrkrData->curlCtx = ctx; + finalize_it: + free(token); ++ free(statsName); ++ if ((iRet != RS_RET_OK) && pWrkrData->stats) { ++ statsobj.Destruct(&(pWrkrData->stats)); ++ } + if (fp) { + fclose(fp); + } +@@ -765,6 +876,7 @@ BEGINfreeWrkrInstance + CODESTARTfreeWrkrInstance + curl_easy_cleanup(pWrkrData->curlCtx); + curl_slist_free_all(pWrkrData->curlHdr); ++ statsobj.Destruct(&(pWrkrData->stats)); + ENDfreeWrkrInstance + + +@@ -790,6 +902,8 @@ cacheNew(const uchar *const url) + key_equals_string, hashtable_json_object_put); + cache->nsHt = create_hashtable(100, hash_from_string, + key_equals_string, hashtable_json_object_put); ++ dbgprintf("mmkubernetes: created cache mdht [%p] nsht [%p]\n", ++ cache->mdHt, cache->nsHt); + cache->cacheMtx = malloc(sizeof(pthread_mutex_t)); + if (!cache->mdHt || !cache->nsHt || !cache->cacheMtx) { + free (cache); +@@ -797,6 +911,7 @@ cacheNew(const uchar *const url) + FINALIZE; + } + pthread_mutex_init(cache->cacheMtx, NULL); ++ cache->lastBusyTime = 0; + + finalize_it: + return cache; +@@ -816,9 +931,10 @@ static void cacheFree(struct cache_s *cache) + BEGINnewActInst + struct cnfparamvals *pvals = NULL; + int i; +- FILE *fp; ++ FILE *fp = NULL; + char *rxstr = NULL; + char *srcMetadataPath = NULL; ++ char errStr[1024]; + CODESTARTnewActInst + DBGPRINTF("newActInst (mmkubernetes)\n"); + +@@ -840,6 +956,7 @@ CODESTARTnewActInst + + pData->de_dot = loadModConf->de_dot; + pData->allowUnsignedCerts = loadModConf->allowUnsignedCerts; ++ pData->busyRetryInterval = loadModConf->busyRetryInterval; + for(i = 0 ; i < actpblk.nParams ; ++i) { + if(!pvals[i].bUsed) { + continue; +@@ -863,7 +980,6 @@ CODESTARTnewActInst + pData->caCertFile = (uchar *) es_str2cstr(pvals[i].val.d.estr, NULL); + fp = fopen((const char*)pData->caCertFile, "r"); + if(fp == NULL) { +- char errStr[1024]; + rs_strerror_r(errno, errStr, sizeof(errStr)); + iRet = RS_RET_NO_FILE_ACCESS; + LogError(0, iRet, +@@ -872,6 +988,33 @@ CODESTARTnewActInst + ABORT_FINALIZE(iRet); + } else { + fclose(fp); ++ fp = NULL; ++ } ++ } else if(!strcmp(actpblk.descr[i].name, "tls.mycert")) { ++ pData->myCertFile = (uchar*)es_str2cstr(pvals[i].val.d.estr, NULL); ++ fp = fopen((const char*)pData->myCertFile, "r"); ++ if(fp == NULL) { ++ rs_strerror_r(errno, errStr, sizeof(errStr)); ++ iRet = RS_RET_NO_FILE_ACCESS; ++ LogError(0, iRet, ++ "error: 'tls.mycert' file %s couldn't be accessed: %s\n", ++ pData->myCertFile, errStr); ++ } else { ++ fclose(fp); ++ fp = NULL; ++ } ++ } else if(!strcmp(actpblk.descr[i].name, "tls.myprivkey")) { ++ pData->myPrivKeyFile = (uchar*)es_str2cstr(pvals[i].val.d.estr, NULL); ++ fp = fopen((const char*)pData->myPrivKeyFile, "r"); ++ if(fp == NULL) { ++ rs_strerror_r(errno, errStr, sizeof(errStr)); ++ iRet = RS_RET_NO_FILE_ACCESS; ++ LogError(0, iRet, ++ "error: 'tls.myprivkey' file %s couldn't be accessed: %s\n", ++ pData->myPrivKeyFile, errStr); ++ } else { ++ fclose(fp); ++ fp = NULL; + } + } else if(!strcmp(actpblk.descr[i].name, "allowunsignedcerts")) { + pData->allowUnsignedCerts = pvals[i].val.d.n; +@@ -883,7 +1026,6 @@ CODESTARTnewActInst + pData->tokenFile = (uchar *) es_str2cstr(pvals[i].val.d.estr, NULL); + fp = fopen((const char*)pData->tokenFile, "r"); + if(fp == NULL) { +- char errStr[1024]; + rs_strerror_r(errno, errStr, sizeof(errStr)); + iRet = RS_RET_NO_FILE_ACCESS; + LogError(0, iRet, +@@ -892,6 +1034,7 @@ CODESTARTnewActInst + ABORT_FINALIZE(iRet); + } else { + fclose(fp); ++ fp = NULL; + } + } else if(!strcmp(actpblk.descr[i].name, "annotation_match")) { + free_annotationmatch(&pData->annotation_match); +@@ -912,7 +1055,6 @@ CODESTARTnewActInst + pData->fnRulebase = (uchar *) es_str2cstr(pvals[i].val.d.estr, NULL); + fp = fopen((const char*)pData->fnRulebase, "r"); + if(fp == NULL) { +- char errStr[1024]; + rs_strerror_r(errno, errStr, sizeof(errStr)); + iRet = RS_RET_NO_FILE_ACCESS; + LogError(0, iRet, +@@ -921,6 +1063,7 @@ CODESTARTnewActInst + ABORT_FINALIZE(iRet); + } else { + fclose(fp); ++ fp = NULL; + } + #if HAVE_LOADSAMPLESFROMSTRING == 1 + } else if(!strcmp(modpblk.descr[i].name, "containerrules")) { +@@ -932,7 +1075,6 @@ CODESTARTnewActInst + pData->contRulebase = (uchar *) es_str2cstr(pvals[i].val.d.estr, NULL); + fp = fopen((const char*)pData->contRulebase, "r"); + if(fp == NULL) { +- char errStr[1024]; + rs_strerror_r(errno, errStr, sizeof(errStr)); + iRet = RS_RET_NO_FILE_ACCESS; + LogError(0, iRet, +@@ -941,7 +1083,10 @@ CODESTARTnewActInst + ABORT_FINALIZE(iRet); + } else { + fclose(fp); ++ fp = NULL; + } ++ } else if(!strcmp(actpblk.descr[i].name, "busyretryinterval")) { ++ pData->busyRetryInterval = pvals[i].val.d.n; + } else { + dbgprintf("mmkubernetes: program error, non-handled " + "param '%s' in action() block\n", actpblk.descr[i].name); +@@ -982,6 +1127,10 @@ CODESTARTnewActInst + pData->dstMetadataPath = (uchar *) strdup((char *) loadModConf->dstMetadataPath); + if(pData->caCertFile == NULL && loadModConf->caCertFile) + pData->caCertFile = (uchar *) strdup((char *) loadModConf->caCertFile); ++ if(pData->myCertFile == NULL && loadModConf->myCertFile) ++ pData->myCertFile = (uchar *) strdup((char *) loadModConf->myCertFile); ++ if(pData->myPrivKeyFile == NULL && loadModConf->myPrivKeyFile) ++ pData->myPrivKeyFile = (uchar *) strdup((char *) loadModConf->myPrivKeyFile); + if(pData->token == NULL && loadModConf->token) + pData->token = (uchar *) strdup((char *) loadModConf->token); + if(pData->tokenFile == NULL && loadModConf->tokenFile) +@@ -1018,6 +1167,8 @@ CODESTARTnewActInst + CODE_STD_FINALIZERnewActInst + if(pvals != NULL) + cnfparamvalsDestruct(pvals, &actpblk); ++ if(fp) ++ fclose(fp); + free(rxstr); + free(srcMetadataPath); + ENDnewActInst +@@ -1061,6 +1212,8 @@ CODESTARTfreeCnf + free(pModConf->srcMetadataPath); + free(pModConf->dstMetadataPath); + free(pModConf->caCertFile); ++ free(pModConf->myCertFile); ++ free(pModConf->myPrivKeyFile); + free(pModConf->token); + free(pModConf->tokenFile); + free(pModConf->de_dot_separator); +@@ -1069,8 +1222,11 @@ CODESTARTfreeCnf + free(pModConf->contRules); + free(pModConf->contRulebase); + free_annotationmatch(&pModConf->annotation_match); +- for(i = 0; caches[i] != NULL; i++) ++ for(i = 0; caches[i] != NULL; i++) { ++ dbgprintf("mmkubernetes: freeing cache [%d] mdht [%p] nsht [%p]\n", ++ i, caches[i]->mdHt, caches[i]->nsHt); + cacheFree(caches[i]); ++ } + free(caches); + ENDfreeCnf + +@@ -1082,6 +1238,8 @@ CODESTARTdbgPrintInstInfo + dbgprintf("\tsrcMetadataPath='%s'\n", pData->srcMetadataDescr->name); + dbgprintf("\tdstMetadataPath='%s'\n", pData->dstMetadataPath); + dbgprintf("\ttls.cacert='%s'\n", pData->caCertFile); ++ dbgprintf("\ttls.mycert='%s'\n", pData->myCertFile); ++ dbgprintf("\ttls.myprivkey='%s'\n", pData->myPrivKeyFile); + dbgprintf("\tallowUnsignedCerts='%d'\n", pData->allowUnsignedCerts); + dbgprintf("\ttoken='%s'\n", pData->token); + dbgprintf("\ttokenFile='%s'\n", pData->tokenFile); +@@ -1093,6 +1251,7 @@ CODESTARTdbgPrintInstInfo + dbgprintf("\tfilenamerules='%s'\n", pData->fnRules); + dbgprintf("\tcontainerrules='%s'\n", pData->contRules); + #endif ++ dbgprintf("\tbusyretryinterval='%d'\n", pData->busyRetryInterval); + ENDdbgPrintInstInfo + + +@@ -1206,6 +1365,24 @@ queryKB(wrkrInstanceData_t *pWrkrData, char *url, struct json_object **rply) + struct json_object *jo; + long resp_code = 400; + ++ if (pWrkrData->pData->cache->lastBusyTime) { ++ time_t now; ++ datetime.GetTime(&now); ++ now -= pWrkrData->pData->cache->lastBusyTime; ++ if (now < pWrkrData->pData->busyRetryInterval) { ++ LogMsg(0, RS_RET_RETRY, LOG_DEBUG, ++ "mmkubernetes: Waited [%ld] of [%d] seconds for the requested url [%s]\n", ++ now, pWrkrData->pData->busyRetryInterval, url); ++ ABORT_FINALIZE(RS_RET_RETRY); ++ } else { ++ LogMsg(0, RS_RET_OK, LOG_DEBUG, ++ "mmkubernetes: Cleared busy status after [%d] seconds - " ++ "will retry the requested url [%s]\n", ++ pWrkrData->pData->busyRetryInterval, url); ++ pWrkrData->pData->cache->lastBusyTime = 0; ++ } ++ } ++ + /* query kubernetes for pod info */ + ccode = curl_easy_setopt(pWrkrData->curlCtx, CURLOPT_URL, url); + if(ccode != CURLE_OK) +@@ -1238,17 +1415,23 @@ queryKB(wrkrInstanceData_t *pWrkrData, char *url, struct json_object **rply) + ABORT_FINALIZE(RS_RET_ERR); + } + if(resp_code == 404) { +- LogMsg(0, RS_RET_ERR, LOG_ERR, ++ LogMsg(0, RS_RET_NOT_FOUND, LOG_INFO, + "mmkubernetes: Not Found: the resource does not exist at url [%s]\n", + url); +- ABORT_FINALIZE(RS_RET_ERR); ++ ABORT_FINALIZE(RS_RET_NOT_FOUND); + } + if(resp_code == 429) { +- LogMsg(0, RS_RET_ERR, LOG_ERR, ++ if (pWrkrData->pData->busyRetryInterval) { ++ time_t now; ++ datetime.GetTime(&now); ++ pWrkrData->pData->cache->lastBusyTime = now; ++ } ++ ++ LogMsg(0, RS_RET_RETRY, LOG_INFO, + "mmkubernetes: Too Many Requests: the server is too heavily loaded " + "to provide the data for the requested url [%s]\n", + url); +- ABORT_FINALIZE(RS_RET_ERR); ++ ABORT_FINALIZE(RS_RET_RETRY); + } + if(resp_code != 200) { + LogMsg(0, RS_RET_ERR, LOG_ERR, +@@ -1299,12 +1482,14 @@ BEGINdoAction + char *mdKey = NULL; + struct json_object *jMetadata = NULL, *jMetadataCopy = NULL, *jMsgMeta = NULL, + *jo = NULL; +- int add_ns_metadata = 0; ++ int add_pod_metadata = 1; + CODESTARTdoAction + CHKiRet_Hdlr(extractMsgMetadata(pMsg, pWrkrData->pData, &jMsgMeta)) { + ABORT_FINALIZE((iRet == RS_RET_NOT_FOUND) ? RS_RET_OK : iRet); + } + ++ STATSCOUNTER_INC(pWrkrData->k8sRecordSeen, pWrkrData->mutK8sRecordSeen); ++ + if (fjson_object_object_get_ex(jMsgMeta, "pod_name", &jo)) + podName = json_object_get_string(jo); + if (fjson_object_object_get_ex(jMsgMeta, "namespace_name", &jo)) +@@ -1347,28 +1532,49 @@ CODESTARTdoAction + } + iRet = queryKB(pWrkrData, url, &jReply); + free(url); +- /* todo: implement support for the .orphaned namespace */ +- if (iRet != RS_RET_OK) { ++ if (iRet == RS_RET_NOT_FOUND) { ++ /* negative cache namespace - make a dummy empty namespace metadata object */ ++ jNsMeta = json_object_new_object(); ++ STATSCOUNTER_INC(pWrkrData->namespaceMetadataNotFound, ++ pWrkrData->mutNamespaceMetadataNotFound); ++ } else if (iRet == RS_RET_RETRY) { ++ /* server is busy - retry or error */ ++ STATSCOUNTER_INC(pWrkrData->namespaceMetadataBusy, ++ pWrkrData->mutNamespaceMetadataBusy); ++ if (0 == pWrkrData->pData->busyRetryInterval) { ++ pthread_mutex_unlock(pWrkrData->pData->cache->cacheMtx); ++ ABORT_FINALIZE(RS_RET_ERR); ++ } ++ add_pod_metadata = 0; /* don't cache pod metadata either - retry both */ ++ } else if (iRet != RS_RET_OK) { ++ /* hard error - something the admin needs to fix e.g. network, config, auth */ + json_object_put(jReply); + jReply = NULL; ++ STATSCOUNTER_INC(pWrkrData->namespaceMetadataError, ++ pWrkrData->mutNamespaceMetadataError); + pthread_mutex_unlock(pWrkrData->pData->cache->cacheMtx); + FINALIZE; +- } +- +- if(fjson_object_object_get_ex(jReply, "metadata", &jNsMeta)) { ++ } else if (fjson_object_object_get_ex(jReply, "metadata", &jNsMeta)) { + jNsMeta = json_object_get(jNsMeta); + parse_labels_annotations(jNsMeta, &pWrkrData->pData->annotation_match, + pWrkrData->pData->de_dot, + (const char *)pWrkrData->pData->de_dot_separator, + pWrkrData->pData->de_dot_separator_len); +- add_ns_metadata = 1; ++ STATSCOUNTER_INC(pWrkrData->namespaceMetadataSuccess, ++ pWrkrData->mutNamespaceMetadataSuccess); + } else { + /* namespace with no metadata??? */ + LogMsg(0, RS_RET_ERR, LOG_INFO, + "mmkubernetes: namespace [%s] has no metadata!\n", ns); +- jNsMeta = NULL; ++ /* negative cache namespace - make a dummy empty namespace metadata object */ ++ jNsMeta = json_object_new_object(); ++ STATSCOUNTER_INC(pWrkrData->namespaceMetadataSuccess, ++ pWrkrData->mutNamespaceMetadataSuccess); + } + ++ if(jNsMeta) { ++ hashtable_insert(pWrkrData->pData->cache->nsHt, strdup(ns), jNsMeta); ++ } + json_object_put(jReply); + jReply = NULL; + } +@@ -1381,14 +1587,28 @@ CODESTARTdoAction + } + iRet = queryKB(pWrkrData, url, &jReply); + free(url); +- if(iRet != RS_RET_OK) { +- if(jNsMeta && add_ns_metadata) { +- hashtable_insert(pWrkrData->pData->cache->nsHt, strdup(ns), jNsMeta); ++ if (iRet == RS_RET_NOT_FOUND) { ++ /* negative cache pod - make a dummy empty pod metadata object */ ++ iRet = RS_RET_OK; ++ STATSCOUNTER_INC(pWrkrData->podMetadataNotFound, pWrkrData->mutPodMetadataNotFound); ++ } else if (iRet == RS_RET_RETRY) { ++ /* server is busy - retry or error */ ++ STATSCOUNTER_INC(pWrkrData->podMetadataBusy, pWrkrData->mutPodMetadataBusy); ++ if (0 == pWrkrData->pData->busyRetryInterval) { ++ pthread_mutex_unlock(pWrkrData->pData->cache->cacheMtx); ++ ABORT_FINALIZE(RS_RET_ERR); + } ++ add_pod_metadata = 0; /* do not cache so that we can retry */ ++ iRet = RS_RET_OK; ++ } else if(iRet != RS_RET_OK) { ++ /* hard error - something the admin needs to fix e.g. network, config, auth */ + json_object_put(jReply); + jReply = NULL; ++ STATSCOUNTER_INC(pWrkrData->podMetadataError, pWrkrData->mutPodMetadataError); + pthread_mutex_unlock(pWrkrData->pData->cache->cacheMtx); + FINALIZE; ++ } else { ++ STATSCOUNTER_INC(pWrkrData->podMetadataSuccess, pWrkrData->mutPodMetadataSuccess); + } + + jo = json_object_new_object(); +@@ -1435,11 +1655,9 @@ CODESTARTdoAction + json_object_object_add(jo, "container_id", json_object_get(jo2)); + json_object_object_add(jMetadata, "docker", jo); + +- hashtable_insert(pWrkrData->pData->cache->mdHt, mdKey, jMetadata); +- mdKey = NULL; +- if(jNsMeta && add_ns_metadata) { +- hashtable_insert(pWrkrData->pData->cache->nsHt, strdup(ns), jNsMeta); +- ns = NULL; ++ if (add_pod_metadata) { ++ hashtable_insert(pWrkrData->pData->cache->mdHt, mdKey, jMetadata); ++ mdKey = NULL; + } + } + +@@ -1450,6 +1668,11 @@ CODESTARTdoAction + * outside of the cache lock + */ + jMetadataCopy = json_tokener_parse(json_object_get_string(jMetadata)); ++ if (!add_pod_metadata) { ++ /* jMetadata object was created from scratch and not cached */ ++ json_object_put(jMetadata); ++ jMetadata = NULL; ++ } + pthread_mutex_unlock(pWrkrData->pData->cache->cacheMtx); + /* the +1 is there to skip the leading '$' */ + msgAddJSON(pMsg, (uchar *) pWrkrData->pData->dstMetadataPath + 1, jMetadataCopy, 0, 0); +@@ -1470,7 +1693,9 @@ BEGINmodExit + CODESTARTmodExit + curl_global_cleanup(); + ++ objRelease(datetime, CORE_COMPONENT); + objRelease(regexp, LM_REGEXP_FILENAME); ++ objRelease(statsobj, CORE_COMPONENT); + ENDmodExit + + +@@ -1489,8 +1714,9 @@ CODESTARTmodInit + *ipIFVersProvided = CURR_MOD_IF_VERSION; /* we only support the current interface specification */ + CODEmodInit_QueryRegCFSLineHdlr + DBGPRINTF("mmkubernetes: module compiled with rsyslog version %s.\n", VERSION); ++ CHKiRet(objUse(statsobj, CORE_COMPONENT)); + CHKiRet(objUse(regexp, LM_REGEXP_FILENAME)); +- ++ CHKiRet(objUse(datetime, CORE_COMPONENT)); + /* CURL_GLOBAL_ALL initializes more than is needed but the + * libcurl documentation discourages use of other values + */ diff --git a/SOURCES/rsyslog-8.37.0-rhbz1627941-imfile-support-for-endmsg.regex.patch b/SOURCES/rsyslog-8.37.0-rhbz1627941-imfile-support-for-endmsg.regex.patch new file mode 100644 index 0000000..0a4acc7 --- /dev/null +++ b/SOURCES/rsyslog-8.37.0-rhbz1627941-imfile-support-for-endmsg.regex.patch @@ -0,0 +1,263 @@ +From e8d64cbd15fa84907dc23f8b52d6f2f847b46fec Mon Sep 17 00:00:00 2001 +From: Rich Megginson +Date: Mon, 10 Sep 2018 17:25:38 -0600 +Subject: [PATCH] imfile: support for endmsg.regex + +This adds support for endmsg.regex. It is similar to +startmsg.regex except that it matches the line that denotes +the end of the message, rather than the start of the next message. +This is primarily for container log file use cases such as this: + + date stdout P start of message + date stdout P middle of message + date stdout F end of message + +The `F` means this is the line which contains the final part of +the message. The fully assembled message should be +`start of message middle of message end of message`. +`startmsg.regex="^[^ ]+ stdout F "` will match. + +(cherry picked from commit c902a0938fe163b5351829d2b72001d024895c16) +(cherry picked from commit dd4a72c4d52d8da98ed6b86114868e1a450ccb41) +--- + plugins/imfile/imfile.c | 44 ++++-- + plugins/imptcp/imptcp.c | 10 +- + runtime/stream.c | 28 +++- + runtime/stream.h | 2 +- + 4 files changed, 62 insertions(+), 20 deletions(-) + +diff --git a/plugins/imfile/imfile.c b/plugins/imfile/imfile.c +index 7767c9f02..87706082f 100644 +--- a/plugins/imfile/imfile.c ++++ b/plugins/imfile/imfile.c +@@ -126,7 +126,9 @@ struct instanceConf_s { + sbool bRMStateOnDel; + uint8_t readMode; + uchar *startRegex; +- regex_t end_preg; /* compiled version of startRegex */ ++ uchar *endRegex; ++ regex_t start_preg; /* compiled version of startRegex */ ++ regex_t end_preg; /* compiled version of endRegex */ + sbool discardTruncatedMsg; + sbool msgDiscardingError; + sbool escapeLF; +@@ -281,6 +283,7 @@ static struct cnfparamdescr inppdescr[] = { + { "ruleset", eCmdHdlrString, 0 }, + { "readmode", eCmdHdlrInt, 0 }, + { "startmsg.regex", eCmdHdlrString, 0 }, ++ { "endmsg.regex", eCmdHdlrString, 0 }, + { "discardtruncatedmsg", eCmdHdlrBinary, 0 }, + { "msgdiscardingerror", eCmdHdlrBinary, 0 }, + { "escapelf", eCmdHdlrBinary, 0 }, +@@ -1421,6 +1424,7 @@ pollFileReal(act_obj_t *act, cstr_t **pCStr) + int64 strtOffs; + DEFiRet; + int nProcessed = 0; ++ regex_t *start_preg = NULL, *end_preg = NULL; + + DBGPRINTF("pollFileReal enter, pStrm %p, name '%s'\n", act->pStrm, act->name); + DBGPRINTF("pollFileReal enter, edge %p\n", act->edge); +@@ -1432,15 +1436,18 @@ pollFileReal(act_obj_t *act, cstr_t **pCStr) + CHKiRet(openFile(act)); /* open file */ + } + ++ start_preg = (inst->startRegex == NULL) ? NULL : &inst->start_preg; ++ end_preg = (inst->endRegex == NULL) ? NULL : &inst->end_preg; ++ + /* loop below will be exited when strmReadLine() returns EOF */ + while(glbl.GetGlobalInputTermState() == 0) { + if(inst->maxLinesAtOnce != 0 && nProcessed >= inst->maxLinesAtOnce) + break; +- if(inst->startRegex == NULL) { ++ if((start_preg == NULL) && (end_preg == NULL)) { + CHKiRet(strm.ReadLine(act->pStrm, pCStr, inst->readMode, inst->escapeLF, + inst->trimLineOverBytes, &strtOffs)); + } else { +- CHKiRet(strmReadMultiLine(act->pStrm, pCStr, &inst->end_preg, ++ CHKiRet(strmReadMultiLine(act->pStrm, pCStr, start_preg, end_preg, + inst->escapeLF, inst->discardTruncatedMsg, inst->msgDiscardingError, &strtOffs)); + } + ++nProcessed; +@@ -1506,6 +1513,7 @@ createInstance(instanceConf_t **const pinst) + inst->iPersistStateInterval = 0; + inst->readMode = 0; + inst->startRegex = NULL; ++ inst->endRegex = NULL; + inst->discardTruncatedMsg = 0; + inst->msgDiscardingError = 1; + inst->bRMStateOnDel = 1; +@@ -1713,6 +1721,8 @@ CODESTARTnewInpInst + inst->readMode = (sbool) pvals[i].val.d.n; + } else if(!strcmp(inppblk.descr[i].name, "startmsg.regex")) { + inst->startRegex = (uchar*)es_str2cstr(pvals[i].val.d.estr, NULL); ++ } else if(!strcmp(inppblk.descr[i].name, "endmsg.regex")) { ++ inst->endRegex = (uchar*)es_str2cstr(pvals[i].val.d.estr, NULL); + } else if(!strcmp(inppblk.descr[i].name, "discardtruncatedmsg")) { + inst->discardTruncatedMsg = (sbool) pvals[i].val.d.n; + } else if(!strcmp(inppblk.descr[i].name, "msgdiscardingerror")) { +@@ -1753,19 +1763,31 @@ CODESTARTnewInpInst + "param '%s'\n", inppblk.descr[i].name); + } + } +- if(inst->readMode != 0 && inst->startRegex != NULL) { ++ i = (inst->readMode > 0) ? 1 : 0; ++ i = (NULL != inst->startRegex) ? (i+1) : i; ++ i = (NULL != inst->endRegex) ? (i+1) : i; ++ if(i > 1) { + LogError(0, RS_RET_PARAM_NOT_PERMITTED, +- "readMode and startmsg.regex cannot be set " +- "at the same time --- remove one of them"); ++ "only one of readMode or startmsg.regex or endmsg.regex can be set " ++ "at the same time"); + ABORT_FINALIZE(RS_RET_PARAM_NOT_PERMITTED); + } + + if(inst->startRegex != NULL) { +- const int errcode = regcomp(&inst->end_preg, (char*)inst->startRegex, REG_EXTENDED); ++ const int errcode = regcomp(&inst->start_preg, (char*)inst->startRegex, REG_EXTENDED); ++ if(errcode != 0) { ++ char errbuff[512]; ++ regerror(errcode, &inst->start_preg, errbuff, sizeof(errbuff)); ++ parser_errmsg("imfile: error in startmsg.regex expansion: %s", errbuff); ++ ABORT_FINALIZE(RS_RET_ERR); ++ } ++ } ++ if(inst->endRegex != NULL) { ++ const int errcode = regcomp(&inst->end_preg, (char*)inst->endRegex, REG_EXTENDED); + if(errcode != 0) { + char errbuff[512]; + regerror(errcode, &inst->end_preg, errbuff, sizeof(errbuff)); +- parser_errmsg("imfile: error in regex expansion: %s", errbuff); ++ parser_errmsg("imfile: error in endmsg.regex expansion: %s", errbuff); + ABORT_FINALIZE(RS_RET_ERR); + } + } +@@ -1970,9 +1992,13 @@ CODESTARTfreeCnf + free(inst->pszStateFile); + free(inst->pszFileName_forOldStateFile); + if(inst->startRegex != NULL) { +- regfree(&inst->end_preg); ++ regfree(&inst->start_preg); + free(inst->startRegex); + } ++ if(inst->endRegex != NULL) { ++ regfree(&inst->end_preg); ++ free(inst->endRegex); ++ } + del = inst; + inst = inst->next; + free(del); +diff --git a/plugins/imptcp/imptcp.c b/plugins/imptcp/imptcp.c +index 9b6be0f40..a94b97f41 100644 +--- a/plugins/imptcp/imptcp.c ++++ b/plugins/imptcp/imptcp.c +@@ -162,7 +162,7 @@ struct instanceConf_s { + int ratelimitInterval; + int ratelimitBurst; + uchar *startRegex; +- regex_t end_preg; /* compiled version of startRegex */ ++ regex_t start_preg; /* compiled version of startRegex */ + struct instanceConf_s *next; + }; + +@@ -961,7 +961,7 @@ processDataRcvd_regexFraming(ptcpsess_t *const __restrict__ pThis, + if(c == '\n') { + pThis->iCurrLine = pThis->iMsg; + } else { +- const int isMatch = !regexec(&inst->end_preg, (char*)pThis->pMsg+pThis->iCurrLine, 0, NULL, 0); ++ const int isMatch = !regexec(&inst->start_preg, (char*)pThis->pMsg+pThis->iCurrLine, 0, NULL, 0); + if(isMatch) { + DBGPRINTF("regex match (%d), framing line: %s\n", pThis->iCurrLine, pThis->pMsg); + strcpy((char*)pThis->pMsg_save, (char*) pThis->pMsg+pThis->iCurrLine); +@@ -2188,10 +2188,10 @@ CODESTARTnewInpInst + } + + if(inst->startRegex != NULL) { +- const int errcode = regcomp(&inst->end_preg, (char*)inst->startRegex, REG_EXTENDED); ++ const int errcode = regcomp(&inst->start_preg, (char*)inst->startRegex, REG_EXTENDED); + if(errcode != 0) { + char errbuff[512]; +- regerror(errcode, &inst->end_preg, errbuff, sizeof(errbuff)); ++ regerror(errcode, &inst->start_preg, errbuff, sizeof(errbuff)); + parser_errmsg("imptcp: error in framing.delimiter.regex expansion: %s", errbuff); + ABORT_FINALIZE(RS_RET_ERR); + } +@@ -2348,7 +2348,7 @@ CODESTARTfreeCnf + free(inst->pszInputName); + free(inst->dfltTZ); + if(inst->startRegex != NULL) { +- regfree(&inst->end_preg); ++ regfree(&inst->start_preg); + free(inst->startRegex); + } + del = inst; +diff --git a/runtime/stream.c b/runtime/stream.c +index 6b7e7028e..0f4197103 100644 +--- a/runtime/stream.c ++++ b/runtime/stream.c +@@ -942,12 +942,12 @@ strmReadMultiLine_isTimedOut(const strm_t *const __restrict__ pThis) + + /* read a multi-line message from a strm file. + * The multi-line message is terminated based on the user-provided +- * startRegex (Posix ERE). For performance reasons, the regex ++ * startRegex or endRegex (Posix ERE). For performance reasons, the regex + * must already have been compiled by the user. + * added 2015-05-12 rgerhards + */ + rsRetVal +-strmReadMultiLine(strm_t *pThis, cstr_t **ppCStr, regex_t *preg, const sbool bEscapeLF, ++strmReadMultiLine(strm_t *pThis, cstr_t **ppCStr, regex_t *start_preg, regex_t *end_preg, const sbool bEscapeLF, + const sbool discardTruncatedMsg, const sbool msgDiscardingError, int64 *const strtOffs) + { + uchar c; +@@ -979,9 +979,14 @@ strmReadMultiLine(strm_t *pThis, cstr_t **ppCStr, regex_t *preg, const sbool bEs + cstrFinalize(thisLine); + + /* we have a line, now let's assemble the message */ +- const int isMatch = !regexec(preg, (char*)rsCStrGetSzStrNoNULL(thisLine), 0, NULL, 0); +- +- if(isMatch) { ++ const int isStartMatch = start_preg ? ++ !regexec(start_preg, (char*)rsCStrGetSzStrNoNULL(thisLine), 0, NULL, 0) : ++ 0; ++ const int isEndMatch = end_preg ? ++ !regexec(end_preg, (char*)rsCStrGetSzStrNoNULL(thisLine), 0, NULL, 0) : ++ 0; ++ ++ if(isStartMatch) { + /* in this case, the *previous* message is complete and we are + * at the start of a new one. + */ +@@ -1047,6 +1052,19 @@ strmReadMultiLine(strm_t *pThis, cstr_t **ppCStr, regex_t *preg, const sbool bEs + } + } + } ++ if(isEndMatch) { ++ /* in this case, the *current* message is complete and we are ++ * at the end of it. ++ */ ++ if(pThis->ignoringMsg == 0) { ++ if(pThis->prevMsgSegment != NULL) { ++ finished = 1; ++ *ppCStr = pThis->prevMsgSegment; ++ pThis->prevMsgSegment= NULL; ++ } ++ } ++ pThis->ignoringMsg = 0; ++ } + cstrDestruct(&thisLine); + } while(finished == 0); + +diff --git a/runtime/stream.h b/runtime/stream.h +index 71596879e..7dc597ff5 100644 +--- a/runtime/stream.h ++++ b/runtime/stream.h +@@ -225,7 +225,7 @@ ENDinterface(strm) + /* prototypes */ + PROTOTYPEObjClassInit(strm); + rsRetVal strmMultiFileSeek(strm_t *pThis, unsigned int fileNum, off64_t offs, off64_t *bytesDel); +-rsRetVal strmReadMultiLine(strm_t *pThis, cstr_t **ppCStr, regex_t *preg, ++rsRetVal strmReadMultiLine(strm_t *pThis, cstr_t **ppCStr, regex_t *start_preg, regex_t *end_preg, + sbool bEscapeLF, sbool discardTruncatedMsg, sbool msgDiscardingError, int64 *const strtOffs); + int strmReadMultiLine_isTimedOut(const strm_t *const __restrict__ pThis); + void strmDebugOutBuf(const strm_t *const pThis); diff --git a/SOURCES/rsyslog-8.37.0-rhbz1659898-imjournal-default-tag.patch b/SOURCES/rsyslog-8.37.0-rhbz1659898-imjournal-default-tag.patch new file mode 100644 index 0000000..cab8873 --- /dev/null +++ b/SOURCES/rsyslog-8.37.0-rhbz1659898-imjournal-default-tag.patch @@ -0,0 +1,93 @@ +diff -up ./plugins/imjournal/imjournal.c.default-tag ./plugins/imjournal/imjournal.c +--- ./plugins/imjournal/imjournal.c.default-tag 2018-05-17 08:50:11.416418022 -0400 ++++ ./plugins/imjournal/imjournal.c 2018-05-17 08:53:02.884418022 -0400 +@@ -78,6 +78,7 @@ static struct configSettings_s { + int bUseJnlPID; + char *usePid; + int bWorkAroundJournalBug; ++ char *dfltTag; + } cs; + + static rsRetVal facilityHdlr(uchar **pp, void *pVal); +@@ -93,7 +94,8 @@ static struct cnfparamdescr modpdescr[] + { "defaultfacility", eCmdHdlrString, 0 }, + { "usepidfromsystem", eCmdHdlrBinary, 0 }, + { "usepid", eCmdHdlrString, 0 }, +- { "workaroundjournalbug", eCmdHdlrBinary, 0 } ++ { "workaroundjournalbug", eCmdHdlrBinary, 0 }, ++ { "defaulttag", eCmdHdlrGetWord, 0 } + }; + static struct cnfparamblk modpblk = + { CNFPARAMBLK_VERSION, +@@ -104,6 +106,7 @@ static struct cnfparamblk modpblk = + #define DFLT_persiststateinterval 10 + #define DFLT_SEVERITY pri2sev(LOG_NOTICE) + #define DFLT_FACILITY pri2fac(LOG_USER) ++#define DFLT_TAG "journal" + + static int bLegacyCnfModGlobalsPermitted = 1;/* are legacy module-global config parameters permitted? */ + +@@ -268,7 +271,7 @@ readjournal(void) + + /* Information from messages */ + char *message = NULL; +- char *sys_iden; ++ char *sys_iden = NULL; + char *sys_iden_help = NULL; + char *c = NULL; + +@@ -331,7 +334,7 @@ readjournal(void) + if (sd_journal_get_data(j, "SYSLOG_IDENTIFIER", &get, &length) >= 0) { + CHKiRet(sanitizeValue(((const char *)get) + 18, length - 18, &sys_iden)); + } else { +- CHKmalloc(sys_iden = strdup("journal")); ++ CHKmalloc(sys_iden = strdup(cs.dfltTag)); + } + + /* trying to get PID, default is "SYSLOG_PID" property */ +@@ -654,6 +657,11 @@ CODESTARTrunInput + "\"usepidfromsystem\" is depricated, use \"usepid\" instead"); + } + ++ if (cs.dfltTag == NULL) { ++ cs.dfltTag = strdup(DFLT_TAG); ++ } ++ ++ + if (cs.usePid && (strcmp(cs.usePid, "system") == 0)) { + pidFieldName = "_PID"; + bPidFallBack = 0; +@@ -732,6 +740,7 @@ CODESTARTbeginCnfLoad + cs.bUseJnlPID = -1; + cs.usePid = NULL; + cs.bWorkAroundJournalBug = 0; ++ cs.dfltTag = NULL; + ENDbeginCnfLoad + + +@@ -754,6 +763,7 @@ BEGINfreeCnf + CODESTARTfreeCnf + free(cs.stateFile); + free(cs.usePid); ++ free(cs.dfltTag); + statsobj.Destruct(&(statsCounter.stats)); + ENDfreeCnf + +@@ -832,6 +842,8 @@ CODESTARTsetModCnf + cs.usePid = (char *)es_str2cstr(pvals[i].val.d.estr, NULL); + } else if (!strcmp(modpblk.descr[i].name, "workaroundjournalbug")) { + cs.bWorkAroundJournalBug = (int) pvals[i].val.d.n; ++ } else if (!strcmp(modpblk.descr[i].name, "defaulttag")) { ++ cs.dfltTag = (char *)es_str2cstr(pvals[i].val.d.estr, NULL); + } else { + dbgprintf("imjournal: program error, non-handled " + "param '%s' in beginCnfLoad\n", modpblk.descr[i].name); +@@ -799,6 +820,8 @@ CODEmodInit_QueryRegCFSLineHdlr + facilityHdlr, &cs.iDfltFacility, STD_LOADABLE_MODULE_ID)); + CHKiRet(omsdRegCFSLineHdlr((uchar *)"imjournalusepidfromsystem", 0, eCmdHdlrBinary, + NULL, &cs.bUseJnlPID, STD_LOADABLE_MODULE_ID)); ++ CHKiRet(omsdRegCFSLineHdlr((uchar *)"imjournaldefaulttag", 0, eCmdHdlrGetWord, ++ NULL, &cs.dfltTag, STD_LOADABLE_MODULE_ID)); + ENDmodInit + /* vim:set ai: + */ diff --git a/SOURCES/rsyslog-8.37.0-rhbz1674471-imfile-log-rotation.patch b/SOURCES/rsyslog-8.37.0-rhbz1674471-imfile-log-rotation.patch new file mode 100644 index 0000000..e721782 --- /dev/null +++ b/SOURCES/rsyslog-8.37.0-rhbz1674471-imfile-log-rotation.patch @@ -0,0 +1,368 @@ +From f85ef7aabcec84497a5eaf9670616b3402c79d9c Mon Sep 17 00:00:00 2001 +From: Rainer Gerhards +Date: Sun, 23 Sep 2018 13:19:31 +0200 +Subject: File rotation with imfile broken + +Previously, truncation was only detected at end of file. Especially with +busy files that could cause loss of data and possibly also stall imfile +reading. The new code now also checks during each read. Obviously, there +is some additional overhead associated with that, but this is unavoidable. + +It still is highly recommended NOT to turn on "reopenOnTruncate" in imfile. +Note that there are also inherant reliability issues. There is no way to +"fix" these, as they are caused by races between the process(es) who truncate +and rsyslog reading the file. But with the new code, the "problem window" +should be much smaller and, more importantly, imfile should not stall. + +A change in the inode was not detected under all circumstances, +most importantly not in some logrotate cases. + +Includes new tests made by Andre Lorbach. They now use the +logrotate tool natively to reproduce the issue. +--- + runtime/rsyslog.h | 6 ++-- + plugins/imfile/imfile.c | 17 +++- + runtime/stream.c | 122 ++++++++---- + runtime/stream.h | 7 ++ + 4 files changed, 126 insertions(+), 36 deletions(-) + +diff --git a/runtime/rsyslog.h b/runtime/rsyslog.h +index 61d0af623..22a1c46d1 100644 +--- a/runtime/rsyslog.h ++++ b/runtime/rsyslog.h +@@ -221,9 +221,9 @@ enum rsRetVal_ /** return value. All methods return this if not specified oth + /* begin regular error codes */ + RS_RET_NOT_IMPLEMENTED = -7, /**< implementation is missing (probably internal error or lazyness ;)) */ + RS_RET_OUT_OF_MEMORY = -6, /**< memory allocation failed */ +- RS_RET_PROVIDED_BUFFER_TOO_SMALL = -50, +-/*< the caller provided a buffer, but the called function sees the size of this buffer is too small - +-operation not carried out */ ++ RS_RET_PROVIDED_BUFFER_TOO_SMALL = -50, /*< the caller provided a buffer, but the called function sees ++ the size of this buffer is too small - operation not carried out */ ++ RS_RET_FILE_TRUNCATED = -51, /**< (input) file was truncated, not an error but a status */ + RS_RET_TRUE = -3, /**< to indicate a true state (can be used as TRUE, legacy) */ + RS_RET_FALSE = -2, /**< to indicate a false state (can be used as FALSE, legacy) */ + RS_RET_NO_IRET = -8, /**< This is a trick for the debuging system - it means no iRet is provided */ +diff --git a/plugins/imfile/imfile.c b/plugins/imfile/imfile.c +index f4a4ef9b7..6be8b2999 100644 +--- a/plugins/imfile/imfile.c ++++ b/plugins/imfile/imfile.c +@@ -740,8 +740,19 @@ detect_updates(fs_edge_t *const edge) + act_obj_unlink(act); + restart = 1; + break; ++ } else if(fileInfo.st_ino != act->ino) { ++ DBGPRINTF("file '%s' inode changed from %llu to %llu, unlinking from " ++ "internal lists\n", act->name, (long long unsigned) act->ino, ++ (long long unsigned) fileInfo.st_ino); ++ if(act->pStrm != NULL) { ++ /* we do no need to re-set later, as act_obj_unlink ++ * will destroy the strm obj */ ++ strmSet_checkRotation(act->pStrm, STRM_ROTATION_DO_NOT_CHECK); ++ } ++ act_obj_unlink(act); ++ restart = 1; ++ break; + } +- // TODO: add inode check for change notification! + + } + +@@ -993,10 +1004,10 @@ chk_active(const act_obj_t *act, const act_obj_t *const deleted) + /* unlink act object from linked list and then + * destruct it. + */ +-static void //ATTR_NONNULL() ++static void ATTR_NONNULL() + act_obj_unlink(act_obj_t *act) + { +- DBGPRINTF("act_obj_unlink %p: %s\n", act, act->name); ++ DBGPRINTF("act_obj_unlink %p: %s, pStrm %p\n", act, act->name, act->pStrm); + if(act->prev == NULL) { + act->edge->active = act->next; + } else { +diff --git a/runtime/stream.c b/runtime/stream.c +index 0f4197103..32a12b256 100644 +--- a/runtime/stream.c ++++ b/runtime/stream.c +@@ -400,6 +400,7 @@ static rsRetVal strmOpenFile(strm_t *pThis) + CHKiRet(doPhysOpen(pThis)); + + pThis->iCurrOffs = 0; ++ pThis->iBufPtrMax = 0; + CHKiRet(getFileSize(pThis->pszCurrFName, &offset)); + if(pThis->tOperationsMode == STREAMMODE_WRITE_APPEND) { + pThis->iCurrOffs = offset; +@@ -636,6 +637,78 @@ strmHandleEOF(strm_t *pThis) + RETiRet; + } + ++ ++/* helper to checkTruncation */ ++static rsRetVal ATTR_NONNULL() ++rereadTruncated(strm_t *const pThis, const char *const reason) ++{ ++ DEFiRet; ++ ++ LogMsg(errno, RS_RET_FILE_TRUNCATED, LOG_WARNING, "file '%s': truncation detected, " ++ "(%s) - re-start reading from beginning", ++ pThis->pszCurrFName, reason); ++ DBGPRINTF("checkTruncation, file %s last buffer CHANGED\n", pThis->pszCurrFName); ++ CHKiRet(strmCloseFile(pThis)); ++ CHKiRet(strmOpenFile(pThis)); ++ iRet = RS_RET_FILE_TRUNCATED; ++ ++finalize_it: ++ RETiRet; ++} ++/* helper to read: ++ * Check if file has been truncated since last read and, if so, re-set reading ++ * to begin of file. To detect truncation, we try to re-read the last block. ++ * If that does not succeed or different data than from the original read is ++ * returned, truncation is assumed. ++ * NOTE: this function must be called only if truncation is enabled AND ++ * when the previous read buffer still is valid (aka "before the next read"). ++ * It is ok to call with a 0-size buffer, which we than assume as begin of ++ * reading. In that case, no truncation will be detected. ++ * rgerhards, 2018-09-20 ++ */ ++static rsRetVal ATTR_NONNULL() ++checkTruncation(strm_t *const pThis) ++{ ++ DEFiRet; ++ int ret; ++ off64_t backseek; ++ assert(pThis->bReopenOnTruncate); ++ ++ DBGPRINTF("checkTruncation, file %s, iBufPtrMax %zd\n", pThis->pszCurrFName, pThis->iBufPtrMax); ++ if(pThis->iBufPtrMax == 0) { ++ FINALIZE; ++ } ++ ++ int currpos = lseek64(pThis->fd, 0, SEEK_CUR); ++ backseek = -1 * (off64_t) pThis->iBufPtrMax; ++ dbgprintf("checkTruncation in actual processing, currpos %d, backseek is %d\n", (int)currpos, (int) backseek); ++ ret = lseek64(pThis->fd, backseek, SEEK_CUR); ++ if(ret < 0) { ++ iRet = rereadTruncated(pThis, "cannot seek backward to begin of last block"); ++ FINALIZE; ++ } ++dbgprintf("checkTruncation seek backwrds: %d\n", ret); ++currpos = lseek64(pThis->fd, 0, SEEK_CUR); ++dbgprintf("checkTruncation in actual processing, NEW currpos %d, backseek is %d\n", (int)currpos, (int) backseek); ++ ++ const ssize_t lenRead = read(pThis->fd, pThis->pIOBuf_truncation, pThis->iBufPtrMax); ++ dbgprintf("checkTruncation proof-read: %d bytes\n", (int) lenRead); ++ if(lenRead < 0) { ++ iRet = rereadTruncated(pThis, "last block could not be re-read"); ++ FINALIZE; ++ } ++ ++ if(!memcmp(pThis->pIOBuf_truncation, pThis->pIOBuf, pThis->iBufPtrMax)) { ++ DBGPRINTF("checkTruncation, file %s last buffer unchanged\n", pThis->pszCurrFName); ++ } else { ++ iRet = rereadTruncated(pThis, "last block data different"); ++ } ++ ++finalize_it: ++ RETiRet; ++} ++ ++ + /* read the next buffer from disk + * rgerhards, 2008-02-13 + */ +@@ -668,6 +741,13 @@ strmReadBuf(strm_t *pThis, int *padBytes) + toRead = (size_t) bytesLeft; + } + } ++ if(pThis->bReopenOnTruncate) { ++ rsRetVal localRet = checkTruncation(pThis); ++ if(localRet == RS_RET_FILE_TRUNCATED) { ++ continue; ++ } ++ CHKiRet(localRet); ++ } + iLenRead = read(pThis->fd, pThis->pIOBuf, toRead); + DBGOPRINT((obj_t*) pThis, "file %d read %ld bytes\n", pThis->fd, iLenRead); + /* end crypto */ +@@ -1184,6 +1264,7 @@ static rsRetVal strmConstructFinalize(strm_t *pThis) + } else { + /* we work synchronously, so we need to alloc a fixed pIOBuf */ + CHKmalloc(pThis->pIOBuf = (uchar*) MALLOC(pThis->sIOBufSize)); ++ CHKmalloc(pThis->pIOBuf_truncation = (char*) MALLOC(pThis->sIOBufSize)); + } + + finalize_it: +@@ -1231,6 +1312,7 @@ CODESTARTobjDestruct(strm) + } + } else { + free(pThis->pIOBuf); ++ free(pThis->pIOBuf_truncation); + } + + /* Finally, we can free the resources. +diff --git a/runtime/stream.c b/runtime/stream.c +index 2d494c612..5b52591ef 100644 +--- a/runtime/stream.c ++++ b/runtime/stream.c +@@ -360,8 +360,8 @@ CheckFileChange(strm_t *pThis) + CHKiRet(strmSetCurrFName(pThis)); + if(stat((char*) pThis->pszCurrFName, &statName) == -1) + ABORT_FINALIZE(RS_RET_IO_ERROR); +- DBGPRINTF("stream/after deserialize checking for file change on '%s', " +- "inode %u/%u, size/currOffs %llu/%llu\n", ++ DBGPRINTF("CheckFileChange: stream/after deserialize checking for file change " ++ "on '%s', inode %u/%u, size/currOffs %llu/%llu\n", + pThis->pszCurrFName, (unsigned) pThis->inode, + (unsigned) statName.st_ino, + (long long unsigned) statName.st_size, +@@ -574,8 +574,8 @@ strmNextFile(strm_t *pThis) + * circumstances). So starting as of now, we only check the inode number and + * a file change is detected only if the inode changes. -- rgerhards, 2011-01-10 + */ +-static rsRetVal +-strmHandleEOFMonitor(strm_t *pThis) ++static rsRetVal ATTR_NONNULL() ++strmHandleEOFMonitor(strm_t *const pThis) + { + DEFiRet; + struct stat statName; +@@ -611,8 +611,8 @@ strmHandleEOFMonitor(strm_t *pThis) + * try to open the next one. + * rgerhards, 2008-02-13 + */ +-static rsRetVal +-strmHandleEOF(strm_t *pThis) ++static rsRetVal ATTR_NONNULL() ++strmHandleEOF(strm_t *const pThis) + { + DEFiRet; + +@@ -629,7 +629,13 @@ strmHandleEOF(strm_t *pThis) + CHKiRet(strmNextFile(pThis)); + break; + case STREAMTYPE_FILE_MONITOR: +- CHKiRet(strmHandleEOFMonitor(pThis)); ++ DBGOPRINT((obj_t*) pThis, "file '%s' (%d) EOF, rotationCheck %d\n", ++ pThis->pszCurrFName, pThis->fd, pThis->rotationCheck); ++ if(pThis->rotationCheck == STRM_ROTATION_DO_CHECK) { ++ CHKiRet(strmHandleEOFMonitor(pThis)); ++ } else { ++ ABORT_FINALIZE(RS_RET_EOF); ++ } + break; + } + +@@ -687,9 +693,6 @@ checkTruncation(strm_t *const pThis) + iRet = rereadTruncated(pThis, "cannot seek backward to begin of last block"); + FINALIZE; + } +-dbgprintf("checkTruncation seek backwrds: %d\n", ret); +-currpos = lseek64(pThis->fd, 0, SEEK_CUR); +-dbgprintf("checkTruncation in actual processing, NEW currpos %d, backseek is %d\n", (int)currpos, (int) backseek); + + const ssize_t lenRead = read(pThis->fd, pThis->pIOBuf_truncation, pThis->iBufPtrMax); + dbgprintf("checkTruncation proof-read: %d bytes\n", (int) lenRead); +@@ -861,7 +864,7 @@ static rsRetVal strmUnreadChar(strm_t *pThis, uchar c) + * a line, but following lines that are indented are part of the same log entry + */ + static rsRetVal +-strmReadLine(strm_t *pThis, cstr_t **ppCStr, uint8_t mode, sbool bEscapeLF, ++strmReadLine(strm_t *const pThis, cstr_t **ppCStr, uint8_t mode, sbool bEscapeLF, + uint32_t trimLineOverBytes, int64 *const strtOffs) + { + uchar c; +@@ -2147,14 +2150,25 @@ DEFpropSetMeth(strm, cryprov, cryprov_if_t*) + DEFpropSetMeth(strm, cryprovData, void*) + + /* sets timeout in seconds */ +-void ++void ATTR_NONNULL() + strmSetReadTimeout(strm_t *const __restrict__ pThis, const int val) + { ++ ISOBJ_TYPE_assert(pThis, strm); + pThis->readTimeout = val; + } + +-static rsRetVal strmSetbDeleteOnClose(strm_t *pThis, int val) ++void ATTR_NONNULL() ++strmSet_checkRotation(strm_t *const pThis, const int val) { ++ ISOBJ_TYPE_assert(pThis, strm); ++ assert(val == STRM_ROTATION_DO_CHECK || val == STRM_ROTATION_DO_NOT_CHECK); ++ pThis->rotationCheck = val; ++} ++ ++ ++static rsRetVal ATTR_NONNULL() ++strmSetbDeleteOnClose(strm_t *const pThis, const int val) + { ++ ISOBJ_TYPE_assert(pThis, strm); + pThis->bDeleteOnClose = val; + if(pThis->cryprov != NULL) { + pThis->cryprov->SetDeleteOnClose(pThis->cryprovFileData, pThis->bDeleteOnClose); +@@ -2162,15 +2176,19 @@ static rsRetVal strmSetbDeleteOnClose(strm_t *pThis, int val) + return RS_RET_OK; + } + +-static rsRetVal strmSetiMaxFiles(strm_t *pThis, int iNewVal) ++static rsRetVal ATTR_NONNULL() ++strmSetiMaxFiles(strm_t *const pThis, const int iNewVal) + { ++ ISOBJ_TYPE_assert(pThis, strm); + pThis->iMaxFiles = iNewVal; + pThis->iFileNumDigits = getNumberDigits(iNewVal); + return RS_RET_OK; + } + +-static rsRetVal strmSetFileNotFoundError(strm_t *pThis, int pFileNotFoundError) ++static rsRetVal ATTR_NONNULL() ++strmSetFileNotFoundError(strm_t *const pThis, const int pFileNotFoundError) + { ++ ISOBJ_TYPE_assert(pThis, strm); + pThis->fileNotFoundError = pFileNotFoundError; + return RS_RET_OK; + } +diff --git a/runtime/stream.h b/runtime/stream.h +index 7dc597ff5..e3d6c2372 100644 +--- a/runtime/stream.h ++++ b/runtime/stream.h +@@ -124,6 +124,7 @@ typedef struct strm_s { + ino_t inode; /* current inode for files being monitored (undefined else) */ + uchar *pszCurrFName; /* name of current file (if open) */ + uchar *pIOBuf; /* the iobuffer currently in use to gather data */ ++ char *pIOBuf_truncation; /* iobuffer used during trucation detection block re-reads */ + size_t iBufPtrMax; /* current max Ptr in Buffer (if partial read!) */ + size_t iBufPtr; /* pointer into current buffer */ + int iUngetC; /* char set via UngetChar() call or -1 if none set */ + +diff --git a/runtime/stream.h b/runtime/stream.h +index e3d6c2372..f6f48378a 100644 +--- a/runtime/stream.h ++++ b/runtime/stream.h +@@ -91,6 +91,10 @@ typedef enum { /* when extending, do NOT change existing modes! */ + STREAMMODE_WRITE_APPEND = 4 + } strmMode_t; + ++/* settings for stream rotation (applies not to all processing modes!) */ ++#define STRM_ROTATION_DO_CHECK 0 ++#define STRM_ROTATION_DO_NOT_CHECK 1 ++ + #define STREAM_ASYNC_NUMBUFS 2 /* must be a power of 2 -- TODO: make configurable */ + /* The strm_t data structure */ + typedef struct strm_s { +@@ -114,6 +118,7 @@ typedef struct strm_s { + sbool bDisabled; /* should file no longer be written to? (currently set only if omfile file size limit fails) */ + sbool bSync; /* sync this file after every write? */ + sbool bReopenOnTruncate; ++ int rotationCheck; /* rotation check mode */ + size_t sIOBufSize;/* size of IO buffer */ + uchar *pszDir; /* Directory */ + int lenDir; +@@ -234,5 +239,6 @@ void strmSetReadTimeout(strm_t *const __restrict__ pThis, const int val); + const uchar * ATTR_NONNULL() strmGetPrevLineSegment(strm_t *const pThis); + const uchar * ATTR_NONNULL() strmGetPrevMsgSegment(strm_t *const pThis); + int ATTR_NONNULL() strmGetPrevWasNL(const strm_t *const pThis); ++void ATTR_NONNULL() strmSet_checkRotation(strm_t *const pThis, const int val); + + #endif /* #ifndef STREAM_H_INCLUDED */ diff --git a/SOURCES/rsyslog-8.37.0-rhbz1677037-short-offMsg-overrun-crash.patch b/SOURCES/rsyslog-8.37.0-rhbz1677037-short-offMsg-overrun-crash.patch new file mode 100644 index 0000000..c9f7455 --- /dev/null +++ b/SOURCES/rsyslog-8.37.0-rhbz1677037-short-offMsg-overrun-crash.patch @@ -0,0 +1,86 @@ +From 1255a67fdec2fc44cd49b6ea8c463f4319910812 Mon Sep 17 00:00:00 2001 +From: Jiri Vymazal +Date: Wed, 27 Feb 2019 11:57:49 +0100 +Subject: [PATCH] Enlarged msg offset types for bigger structured messages + +using a large enough (dozens of kBs) structured message +it is possible to overflow the signed short type which leads +to rsyslog crash. +--- + runtime/msg.c | 12 ++++++------ + runtime/msg.h | 8 ++++---- + 2 files changed, 10 insertions(+), 10 deletions(-) + +diff --git a/runtime/msg.c b/runtime/msg.c +index b82c38b9ee..96306bbeab 100644 +--- a/runtime/msg.c ++++ b/runtime/msg.c +@@ -839,7 +839,7 @@ msgBaseConstruct(smsg_t **ppThis) + pM->iFacility = LOG_INVLD; + pM->iLenPROGNAME = -1; + pM->offAfterPRI = 0; +- pM->offMSG = -1; ++ pM->offMSG = 0; + pM->iProtocolVersion = 0; + pM->msgFlags = 0; + pM->iLenRawMsg = 0; +@@ -2167,7 +2167,7 @@ MsgSetFlowControlType(smsg_t * const pMsg, flowControl_t eFlowCtl) + * rgerhards, 2009-06-16 + */ + rsRetVal +-MsgSetAfterPRIOffs(smsg_t * const pMsg, short offs) ++MsgSetAfterPRIOffs(smsg_t * const pMsg, uint32_t offs) + { + assert(pMsg != NULL); + pMsg->offAfterPRI = offs; +@@ -2819,12 +2819,12 @@ void MsgSetHOSTNAME(smsg_t *pThis, const uchar* pszHOSTNAME, const int lenHOSTNA + * (exactly by one). This can happen if we have a message that does not + * contain any MSG part. + */ +-void MsgSetMSGoffs(smsg_t * const pMsg, short offs) ++void MsgSetMSGoffs(smsg_t * const pMsg, uint32_t offs) + { + ISOBJ_TYPE_assert(pMsg, msg); + pMsg->offMSG = offs; +- if(offs > pMsg->iLenRawMsg) { +- assert(offs - 1 == pMsg->iLenRawMsg); ++ if(offs > (uint32_t)pMsg->iLenRawMsg) { ++ assert((int)offs - 1 == pMsg->iLenRawMsg); + pMsg->iLenMSG = 0; + } else { + pMsg->iLenMSG = pMsg->iLenRawMsg - offs; +@@ -2920,7 +2920,7 @@ MsgSetRawMsg(smsg_t *const pThis, const char*const pszRawMsg, const size_t lenMs + memcpy(pThis->pszRawMsg, pszRawMsg, pThis->iLenRawMsg); + pThis->pszRawMsg[pThis->iLenRawMsg] = '\0'; /* this also works with truncation! */ + /* correct other information */ +- if(pThis->iLenRawMsg > pThis->offMSG) ++ if((uint32_t)pThis->iLenRawMsg > pThis->offMSG) + pThis->iLenMSG += deltaSize; + else + pThis->iLenMSG = 0; +diff --git a/runtime/msg.h b/runtime/msg.h +index 74439275b1..722cca6e8a 100644 +--- a/runtime/msg.h ++++ b/runtime/msg.h +@@ -67,8 +67,8 @@ struct msg { + sbool bParseSuccess; /* set to reflect state of last executed higher level parser */ + unsigned short iSeverity;/* the severity */ + unsigned short iFacility;/* Facility code */ +- short offAfterPRI; /* offset, at which raw message WITHOUT PRI part starts in pszRawMsg */ +- short offMSG; /* offset at which the MSG part starts in pszRawMsg */ ++ uint32_t offAfterPRI; /* offset, at which raw message WITHOUT PRI part starts in pszRawMsg */ ++ uint32_t offMSG; /* offset at which the MSG part starts in pszRawMsg */ + short iProtocolVersion;/* protocol version of message received 0 - legacy, 1 syslog-protocol) */ + int msgFlags; /* flags associated with this message */ + int iLenRawMsg; /* length of raw message */ +@@ -194,8 +194,8 @@ void MsgSetRcvFromStr(smsg_t *const pMsg, const uchar* pszRcvFrom, const int, pr + rsRetVal MsgSetRcvFromIP(smsg_t *pMsg, prop_t*); + rsRetVal MsgSetRcvFromIPStr(smsg_t *const pThis, const uchar *psz, const int len, prop_t **ppProp); + void MsgSetHOSTNAME(smsg_t *pMsg, const uchar* pszHOSTNAME, const int lenHOSTNAME); +-rsRetVal MsgSetAfterPRIOffs(smsg_t *pMsg, short offs); +-void MsgSetMSGoffs(smsg_t *pMsg, short offs); ++rsRetVal MsgSetAfterPRIOffs(smsg_t *pMsg, uint32_t offs); ++void MsgSetMSGoffs(smsg_t *pMsg, uint32_t offs); + void MsgSetRawMsgWOSize(smsg_t *pMsg, char* pszRawMsg); + void ATTR_NONNULL() MsgSetRawMsg(smsg_t *const pThis, const char*const pszRawMsg, const size_t lenMsg); + rsRetVal MsgReplaceMSG(smsg_t *pThis, const uchar* pszMSG, int lenMSG); diff --git a/SOURCES/rsyslog-8.37.0-rhbz1716867-imjournal-memleak.patch b/SOURCES/rsyslog-8.37.0-rhbz1716867-imjournal-memleak.patch new file mode 100644 index 0000000..d0bc972 --- /dev/null +++ b/SOURCES/rsyslog-8.37.0-rhbz1716867-imjournal-memleak.patch @@ -0,0 +1,58 @@ +From 920c28ff705aac74f389b4613815b14b9482e497 Mon Sep 17 00:00:00 2001 +From: Jiri Vymazal +Date: Mon, 21 Jan 2019 10:58:03 +0100 +Subject: [PATCH] Added missing free() calls of received journal cursor + +--- + plugins/imjournal/imjournal.c | 5 +++++ + 1 file changed, 5 insertions(+) + +diff --git a/plugins/imjournal/imjournal.c b/plugins/imjournal/imjournal.c +index a85e521003..f5c2be4b6e 100644 +--- a/plugins/imjournal/imjournal.c ++++ b/plugins/imjournal/imjournal.c +@@ -449,6 +449,7 @@ readjournal(void) + finalize_it: + free(sys_iden_help); + free(message); ++ free(c); + RETiRet; + } + +@@ -470,6 +471,7 @@ persistJournalState(void) + } + } else { + int ret; ++ free(last_cursor); + if ((ret = sd_journal_get_cursor(j, &last_cursor))) { + LogError(-ret, RS_RET_ERR, "imjournal: sd_journal_get_cursor() failed"); + ABORT_FINALIZE(RS_RET_ERR); +@@ -630,6 +632,7 @@ loadJournalState(void) + iRet = RS_RET_ERR; + } + } ++ free(tmp_cursor); + } + } else { + LogError(0, RS_RET_IO_ERROR, "imjournal: " +@@ -843,6 +846,7 @@ BEGINfreeCnf + free(cs.stateFile); + free(cs.usePid); + free(cs.dfltTag); ++ free(last_cursor); + statsobj.Destruct(&(statsCounter.stats)); + ENDfreeCnf + +diff --git a/plugins/imjournal/imjournal.c b/plugins/imjournal/imjournal.c +index f5c2be4b6e..7225fae1ab 100644 +--- a/plugins/imjournal/imjournal.c ++++ b/plugins/imjournal/imjournal.c +@@ -474,6 +474,7 @@ persistJournalState(void) + free(last_cursor); + if ((ret = sd_journal_get_cursor(j, &last_cursor))) { + LogError(-ret, RS_RET_ERR, "imjournal: sd_journal_get_cursor() failed"); ++ last_cursor = NULL; + ABORT_FINALIZE(RS_RET_ERR); + } + } + diff --git a/SOURCES/rsyslog.conf b/SOURCES/rsyslog.conf new file mode 100644 index 0000000..c4ac740 --- /dev/null +++ b/SOURCES/rsyslog.conf @@ -0,0 +1,79 @@ +# rsyslog configuration file + +# For more information see /usr/share/doc/rsyslog-*/rsyslog_conf.html +# or latest version online at http://www.rsyslog.com/doc/rsyslog_conf.html +# If you experience problems, see http://www.rsyslog.com/doc/troubleshoot.html + +#### MODULES #### + +module(load="imuxsock" # provides support for local system logging (e.g. via logger command) + SysSock.Use="off") # Turn off message reception via local log socket; + # local messages are retrieved through imjournal now. +module(load="imjournal" # provides access to the systemd journal + StateFile="imjournal.state") # File to store the position in the journal +#module(load="imklog") # reads kernel messages (the same are read from journald) +#module(load"immark") # provides --MARK-- message capability + +# Provides UDP syslog reception +# for parameters see http://www.rsyslog.com/doc/imudp.html +#module(load="imudp") # needs to be done just once +#input(type="imudp" port="514") + +# Provides TCP syslog reception +# for parameters see http://www.rsyslog.com/doc/imtcp.html +#module(load="imtcp") # needs to be done just once +#input(type="imtcp" port="514") + +#### GLOBAL DIRECTIVES #### + +# Where to place auxiliary files +global(workDirectory="/var/lib/rsyslog") + +# Use default timestamp format +module(load="builtin:omfile" Template="RSYSLOG_TraditionalFileFormat") + +# Include all config files in /etc/rsyslog.d/ +include(file="/etc/rsyslog.d/*.conf" mode="optional") + +#### RULES #### + +# Log all kernel messages to the console. +# Logging much else clutters up the screen. +#kern.* /dev/console + +# Log anything (except mail) of level info or higher. +# Don't log private authentication messages! +*.info;mail.none;authpriv.none;cron.none /var/log/messages + +# The authpriv file has restricted access. +authpriv.* /var/log/secure + +# Log all the mail messages in one place. +mail.* -/var/log/maillog + + +# Log cron stuff +cron.* /var/log/cron + +# Everybody gets emergency messages +*.emerg :omusrmsg:* + +# Save news errors of level crit and higher in a special file. +uucp,news.crit /var/log/spooler + +# Save boot messages also to boot.log +local7.* /var/log/boot.log + + +# ### sample forwarding rule ### +#action(type="omfwd" +# An on-disk queue is created for this action. If the remote host is +# down, messages are spooled to disk and sent when it is up again. +#queue.filename="fwdRule1" # unique name prefix for spool files +#queue.maxdiskspace="1g" # 1gb space limit (use as much as possible) +#queue.saveonshutdown="on" # save messages to disk on shutdown +#queue.type="LinkedList" # run asynchronously +#action.resumeRetryCount="-1" # infinite retries if host is down +# Remote Logging (we use TCP for reliable delivery) +# remote_host is: name/ip, e.g. 192.168.0.1, port optional e.g. 10514 +#Target="remote_host" Port="XXX" Protocol="tcp") diff --git a/SOURCES/rsyslog.log b/SOURCES/rsyslog.log new file mode 100644 index 0000000..b101e32 --- /dev/null +++ b/SOURCES/rsyslog.log @@ -0,0 +1,12 @@ +/var/log/cron +/var/log/maillog +/var/log/messages +/var/log/secure +/var/log/spooler +{ + missingok + sharedscripts + postrotate + /usr/bin/systemctl kill -s HUP rsyslog.service >/dev/null 2>&1 || true + endscript +} diff --git a/SOURCES/rsyslog.sysconfig b/SOURCES/rsyslog.sysconfig new file mode 100644 index 0000000..bc65731 --- /dev/null +++ b/SOURCES/rsyslog.sysconfig @@ -0,0 +1,5 @@ +# Options for rsyslogd +# Syslogd options are deprecated since rsyslog v3. +# If you want to use them, switch to compatibility mode 2 by "-c 2" +# See rsyslogd(8) for more details +SYSLOGD_OPTIONS="" diff --git a/SPECS/rsyslog.spec b/SPECS/rsyslog.spec new file mode 100644 index 0000000..9e9f90f --- /dev/null +++ b/SPECS/rsyslog.spec @@ -0,0 +1,552 @@ +%define rsyslog_statedir %{_sharedstatedir}/%{name} +%define rsyslog_pkidir %{_sysconfdir}/pki/%{name} +%define rsyslog_docdir %{_docdir}/%{name} + + +Summary: Enhanced system logging and kernel message trapping daemon +Name: rsyslog +Version: 8.37.0 +Release: 11%{?dist} +License: (GPLv3+ and ASL 2.0) +Group: System Environment/Daemons +ExcludeArch: i686 +URL: http://www.rsyslog.com/ +Source0: http://www.rsyslog.com/files/download/rsyslog/%{name}-%{version}.tar.gz +Source1: http://www.rsyslog.com/files/download/rsyslog/%{name}-doc-%{version}.tar.gz +Source2: rsyslog.conf +Source3: rsyslog.sysconfig +Source4: rsyslog.log + +BuildRequires: autoconf +BuildRequires: automake +BuildRequires: bison +BuildRequires: flex +BuildRequires: libcurl-devel +BuildRequires: libgcrypt-devel +BuildRequires: libfastjson-devel >= 0.99.8 +BuildRequires: libestr-devel >= 0.1.9 +BuildRequires: libtool +BuildRequires: libuuid-devel +BuildRequires: pkgconfig +BuildRequires: python3-docutils +# it depens on rhbz#1419228 +BuildRequires: systemd-devel >= 219-39 +BuildRequires: zlib-devel + +Requires: logrotate >= 3.5.2 +Requires: bash >= 2.0 +Requires: libestr >= 0.1.9 +Requires(post): systemd +Requires(preun): systemd +Requires(postun): systemd + +Provides: syslog +Obsoletes: sysklogd < 1.5-11 + +# tweak the upstream service file to honour configuration from /etc/sysconfig/rsyslog +Patch0: rsyslog-8.32.0-service.patch +# imjournal: adds "journal" when tag/process name is missing +Patch1: rsyslog-8.37.0-rhbz1659898-imjournal-default-tag.patch +Patch2: rsyslog-8.37.0-rhbz1614179-imfile-symlink-support.patch +Patch3: rsyslog-8.37.0-rhbz1622768-kubernetes-404-handling.patch +Patch4: rsyslog-8.37.0-rhbz1627941-imfile-support-for-endmsg.regex.patch +Patch5: rsyslog-8.37.0-rhbz1674471-imfile-log-rotation.patch +Patch6: rsyslog-8.37.0-rhbz1677037-short-offMsg-overrun-crash.patch +Patch7: rsyslog-8.37.0-rhbz1614181-imtcp-imudp-preservecase-option.patch +Patch8: rsyslog-8.37.0-rhbz1716867-imjournal-memleak.patch + +%package crypto +Summary: Encryption support +Group: System Environment/Daemons +Requires: %name = %version-%release + +%package doc +Summary: HTML Documentation for rsyslog +Group: Documentation +#no reason to have arched documentation +BuildArch: noarch + +%package elasticsearch +Summary: ElasticSearch output module for rsyslog +Group: System Environment/Daemons +Requires: %name = %version-%release + +%package gnutls +Summary: TLS protocol support for rsyslog +Group: System Environment/Daemons +Requires: %name = %version-%release +BuildRequires: gnutls-devel + +%package gssapi +Summary: GSSAPI authentication and encryption support for rsyslog +Group: System Environment/Daemons +Requires: %name = %version-%release +BuildRequires: krb5-devel + +%package kafka +Summary: Provides kafka support for rsyslog +Group: System Environment/Daemons +Requires: %name = %version-%release +BuildRequires: librdkafka-devel + +%package mmaudit +Summary: Message modification module supporting Linux audit format +Group: System Environment/Daemons +Requires: %name = %version-%release + +%package mmjsonparse +Summary: JSON enhanced logging support +Group: System Environment/Daemons +Requires: %name = %version-%release + +%package mmkubernetes +Summary: Provides the mmkubernetes module +Group: System Environment/Daemons +Requires: %name = %version-%release + +%package mmnormalize +Summary: Log normalization support for rsyslog +Group: System Environment/Daemons +Requires: %name = %version-%release +BuildRequires: liblognorm-devel + +%package mmsnmptrapd +Summary: Message modification module for snmptrapd generated messages +Group: System Environment/Daemons +Requires: %name = %version-%release + +%package mysql +Summary: MySQL support for rsyslog +Group: System Environment/Daemons +Requires: %name = %version-%release +BuildRequires: mariadb-connector-c-devel + +%package pgsql +Summary: PostgresSQL support for rsyslog +Group: System Environment/Daemons +Requires: %name = %version-%release +BuildRequires: postgresql-devel + +%package relp +Summary: RELP protocol support for rsyslog +Group: System Environment/Daemons +Requires: %name = %version-%release +Requires: librelp >= 1.0.3 +BuildRequires: librelp-devel >= 1.2.16 + +%package snmp +Summary: SNMP protocol support for rsyslog +Group: System Environment/Daemons +Requires: %name = %version-%release +BuildRequires: net-snmp-devel + +%description +Rsyslog is an enhanced, multi-threaded syslog daemon. It supports MySQL, +syslog/TCP, RFC 3195, permitted sender lists, filtering on any message part, +and fine grain output format control. It is compatible with stock sysklogd +and can be used as a drop-in replacement. Rsyslog is simple to set up, with +advanced features suitable for enterprise-class, encryption-protected syslog +relay chains. + +%description crypto +This package contains a module providing log file encryption and a +command line tool to process encrypted logs. + +%description doc +This subpackage contains documentation for rsyslog. + +%description elasticsearch +This module provides the capability for rsyslog to feed logs directly into +Elasticsearch. + +%description gnutls +The rsyslog-gnutls package contains the rsyslog plugins that provide the +ability to receive syslog messages via upcoming syslog-transport-tls +IETF standard protocol. + +%description gssapi +The rsyslog-gssapi package contains the rsyslog plugins which support GSSAPI +authentication and secure connections. GSSAPI is commonly used for Kerberos +authentication. + +%description kafka +The rsyslog-kafka package provides modules for Apache Kafka input and output. + +%description mmaudit +This module provides message modification supporting Linux audit format +in various settings. + +%description mmjsonparse +This module provides the capability to recognize and parse JSON enhanced +syslog messages. + +%description mmkubernetes +The rsyslog-mmkubernetes package provides module for adding kubernetes +container metadata. + +%description mmnormalize +This module provides the capability to normalize log messages via liblognorm. + +%description mmsnmptrapd +This message modification module takes messages generated from snmptrapd and +modifies them so that they look like they originated from the read originator. + +%description mysql +The rsyslog-mysql package contains a dynamic shared object that will add +MySQL database support to rsyslog. + +%description pgsql +The rsyslog-pgsql package contains a dynamic shared object that will add +PostgreSQL database support to rsyslog. + +%description relp +The rsyslog-relp package contains the rsyslog plugins that provide +the ability to receive syslog messages via the reliable RELP +protocol. + +%description snmp +The rsyslog-snmp package contains the rsyslog plugin that provides the +ability to send syslog messages as SNMPv1 and SNMPv2c traps. + +%prep +# set up rsyslog-doc sources +%setup -q -a 1 -T -c + +#regenerate the docs + +#mv build/searchindex.js searchindex_backup.js +#sphinx-build -b html source build +#clean up +#mv searchindex_backup.js build/searchindex.js + +rm -r LICENSE README.md source build/objects.inv +mv build doc + +# set up rsyslog sources +%setup -q -D + +%patch0 -p1 -b .service +%patch1 -p1 -b .default-tag +%patch2 -p1 -b .imfile-symlink +%patch3 -p1 -b .mmkubernetes-404 +%patch4 -p1 -b .endmsg-regex +%patch5 -p1 -b .rotation-detection +%patch6 -p1 -b .short-offmsg-crash +%patch7 -p1 -b .preservecase-option +%patch8 -p1 -b .imjournal-memleak + +%build +%ifarch sparc64 +#sparc64 need big PIE +export CFLAGS="$RPM_OPT_FLAGS -fPIE" +%else +export CFLAGS="$RPM_OPT_FLAGS -fpie" +%endif +export LDFLAGS="-pie -Wl,-z,relro -Wl,-z,now" + +sed -i 's/%{version}/%{version}-%{release}/g' configure.ac +autoreconf -if +%configure \ + --prefix=/usr \ + --disable-static \ + --disable-testbench \ + --enable-elasticsearch \ + --enable-generate-man-pages \ + --enable-gnutls \ + --enable-gssapi-krb5 \ + --enable-imdiag \ + --enable-imfile \ + --enable-imjournal \ + --enable-imkafka \ + --enable-impstats \ + --enable-imptcp \ + --enable-mail \ + --enable-mmanon \ + --enable-mmaudit \ + --enable-mmcount \ + --enable-mmjsonparse \ + --enable-mmkubernetes \ + --enable-mmnormalize \ + --enable-mmsnmptrapd \ + --enable-mmutf8fix \ + --enable-mysql \ + --enable-omjournal \ + --enable-omkafka \ + --enable-omprog \ + --enable-omstdout \ + --enable-omuxsock \ + --enable-pgsql \ + --enable-pmaixforwardedfrom \ + --enable-pmcisconames \ + --enable-pmlastmsg \ + --enable-pmsnare \ + --enable-relp \ + --enable-snmp \ + --enable-unlimited-select \ + --enable-usertools + +make + +%install +make DESTDIR=%{buildroot} install + +install -d -m 755 %{buildroot}%{_sysconfdir}/sysconfig +install -d -m 755 %{buildroot}%{_sysconfdir}/logrotate.d +install -d -m 755 %{buildroot}%{_sysconfdir}/rsyslog.d +install -d -m 700 %{buildroot}%{rsyslog_statedir} +install -d -m 700 %{buildroot}%{rsyslog_pkidir} +install -d -m 755 %{buildroot}%{rsyslog_docdir}/html + +install -p -m 644 %{SOURCE2} %{buildroot}%{_sysconfdir}/rsyslog.conf +install -p -m 644 %{SOURCE3} %{buildroot}%{_sysconfdir}/sysconfig/rsyslog +install -p -m 644 %{SOURCE4} %{buildroot}%{_sysconfdir}/logrotate.d/syslog +install -p -m 644 plugins/ommysql/createDB.sql %{buildroot}%{rsyslog_docdir}/mysql-createDB.sql +install -p -m 644 plugins/ompgsql/createDB.sql %{buildroot}%{rsyslog_docdir}/pgsql-createDB.sql +install -p -m 644 contrib/mmkubernetes/*.rulebase %{buildroot}%{rsyslog_docdir} +# extract documentation +cp -r doc/* %{buildroot}%{rsyslog_docdir}/html +# get rid of libtool libraries +rm -f %{buildroot}%{_libdir}/rsyslog/*.la +# get rid of socket activation by default +sed -i '/^Alias/s/^/;/;/^Requires=syslog.socket/s/^/;/' %{buildroot}%{_unitdir}/rsyslog.service + +# convert line endings from "\r\n" to "\n" +cat tools/recover_qi.pl | tr -d '\r' > %{buildroot}%{_bindir}/rsyslog-recover-qi.pl + +%post +for n in /var/log/{messages,secure,maillog,spooler} +do + [ -f $n ] && continue + umask 066 && touch $n +done +%systemd_post rsyslog.service + +%preun +%systemd_preun rsyslog.service + +%postun +%systemd_postun_with_restart rsyslog.service + +%files +%doc AUTHORS COPYING* ChangeLog +%exclude %{rsyslog_docdir}/html +%exclude %{rsyslog_docdir}/mysql-createDB.sql +%exclude %{rsyslog_docdir}/pgsql-createDB.sql +%dir %{_libdir}/rsyslog +%dir %{_sysconfdir}/rsyslog.d +%dir %{rsyslog_statedir} +%dir %{rsyslog_pkidir} +%{_sbindir}/rsyslogd +%attr(755,root,root) %{_bindir}/rsyslog-recover-qi.pl +%{_mandir}/man5/rsyslog.conf.5.gz +%{_mandir}/man8/rsyslogd.8.gz +%{_unitdir}/rsyslog.service +%config(noreplace) %{_sysconfdir}/rsyslog.conf +%config(noreplace) %{_sysconfdir}/sysconfig/rsyslog +%config(noreplace) %{_sysconfdir}/logrotate.d/syslog +# plugins +%{_libdir}/rsyslog/fmhash.so +%{_libdir}/rsyslog/fmhttp.so +%{_libdir}/rsyslog/imdiag.so +%{_libdir}/rsyslog/imfile.so +%{_libdir}/rsyslog/imjournal.so +%{_libdir}/rsyslog/imklog.so +%{_libdir}/rsyslog/immark.so +%{_libdir}/rsyslog/impstats.so +%{_libdir}/rsyslog/imptcp.so +%{_libdir}/rsyslog/imtcp.so +%{_libdir}/rsyslog/imudp.so +%{_libdir}/rsyslog/imuxsock.so +%{_libdir}/rsyslog/lmnet.so +%{_libdir}/rsyslog/lmnetstrms.so +%{_libdir}/rsyslog/lmnsd_ptcp.so +%{_libdir}/rsyslog/lmregexp.so +%{_libdir}/rsyslog/lmstrmsrv.so +%{_libdir}/rsyslog/lmtcpclt.so +%{_libdir}/rsyslog/lmtcpsrv.so +%{_libdir}/rsyslog/lmzlibw.so +%{_libdir}/rsyslog/mmanon.so +%{_libdir}/rsyslog/mmcount.so +%{_libdir}/rsyslog/mmexternal.so +%{_libdir}/rsyslog/mmutf8fix.so +%{_libdir}/rsyslog/omjournal.so +%{_libdir}/rsyslog/ommail.so +%{_libdir}/rsyslog/omprog.so +%{_libdir}/rsyslog/omstdout.so +%{_libdir}/rsyslog/omtesting.so +%{_libdir}/rsyslog/omuxsock.so +%{_libdir}/rsyslog/pmaixforwardedfrom.so +%{_libdir}/rsyslog/pmcisconames.so +%{_libdir}/rsyslog/pmlastmsg.so +%{_libdir}/rsyslog/pmsnare.so + +%files crypto +%{_bindir}/rscryutil +%{_mandir}/man1/rscryutil.1.gz +%{_libdir}/rsyslog/lmcry_gcry.so + +%files doc +%doc %{rsyslog_docdir}/html + +%files elasticsearch +%{_libdir}/rsyslog/omelasticsearch.so + +%files gssapi +%{_libdir}/rsyslog/lmgssutil.so +%{_libdir}/rsyslog/imgssapi.so +%{_libdir}/rsyslog/omgssapi.so + +%files gnutls +%{_libdir}/rsyslog/lmnsd_gtls.so + +%files kafka +%{_libdir}/rsyslog/imkafka.so +%{_libdir}/rsyslog/omkafka.so + +%files mmaudit +%{_libdir}/rsyslog/mmaudit.so + +%files mmjsonparse +%{_libdir}/rsyslog/mmjsonparse.so + +%files mmkubernetes +%{_libdir}/rsyslog/mmkubernetes.so +%doc %{rsyslog_docdir}/k8s_filename.rulebase +%doc %{rsyslog_docdir}/k8s_container_name.rulebase + +%files mmnormalize +%{_libdir}/rsyslog/mmnormalize.so + +%files mmsnmptrapd +%{_libdir}/rsyslog/mmsnmptrapd.so + +%files mysql +%doc %{rsyslog_docdir}/mysql-createDB.sql +%{_libdir}/rsyslog/ommysql.so + +%files pgsql +%doc %{rsyslog_docdir}/pgsql-createDB.sql +%{_libdir}/rsyslog/ompgsql.so + +%files relp +%{_libdir}/rsyslog/imrelp.so +%{_libdir}/rsyslog/omrelp.so + +%files snmp +%{_libdir}/rsyslog/omsnmp.so + + +%changelog +* Wed Jun 05 2019 Jiri Vymazal - 8.37.0-11 + RHEL 8.1.0 ERRATUM +- fixed memory leak in imjournal by proper cursor releasing + resolves: rhbz#1716867 + +* Fri May 10 2019 Jiri Vymazal - 8.37.0-10 + RHEL 8.1.0 ERRATUM +- added option for imfile endmsg.regex + resolves: rhbz#1627941 +- added patch enhancing imfile rotation detection + resolves: rhbz#1674471 +- added patch fixing msgOffset datatype preventing crash on + message with too long other fields + resolves: rhbz#1677037 +- added patch introducing "preservecase" option for imudp/imtcp + resolves: rhbz#1614181 + +* Mon Dec 17 2018 Jiri Vymazal - 8.37.0-9 + RHEL 8.0.0 ERRATUM +- added back legacy option for imjournal default tag + resolves: rhbz#1659898 + +* Fri Dec 14 2018 Jiri Vymazal - 8.37.0-8 + RHEL 8.0.0 ERRATUM +- fixes mmkubenetes handling 404 and 429 errors + resolves: rhbz#1622768 + +* Fri Oct 19 2018 Jiri Vymazal - 8.37.0-7 +- removed version from docdir macro + resolves: rhbz#1638023 + +* Mon Aug 27 2018 Jiri Vymazal - 8.37.0-6 +- updated patch for enhanced imfile symlink support + resolves: rhbz#1614179 + +* Fri Aug 10 2018 Jiri Vymazal - 8.37.0-5 +- rebuild for rebased dependencies +- dependency cleanup and sorted sub-packages in spec + resolves: rhbz#1613880 + +* Fri Aug 10 2018 Jiri Vymazal - 8.37.0-4 +- enabled mmkubernetes module + resolves: rhbz#1614432 + resolves: rhbz#1614441 + +* Thu Aug 09 2018 Josef Ridky - 8.37.0-3 +- Rebuild for Net-SNMP + +* Thu Aug 09 2018 Jiri Vymazal - 8.37.0-2 +- added patch for enhanced imfile symlink support + resolves: rhbz#1614179 + +* Wed Aug 08 2018 Jiri Vymazal - 8.37.0-1 +- rebase to 8.37.0 + resolves: rhbz#1613880 + resolves: rhbz#1564054 + resolves: rhbz#1598218 + - dropped invalid statefile patch - upstreamed + - dropped imjournal duplicates patch - upstreamed + resolves: rhbz#1544394 +- renumbered default tag patch and fitted onto rebased version + +* Fri Aug 03 2018 Jiri Vymazal - 8.36.0-4 +- removed dependency on libee + resolves: rhbz#1612032 + +* Wed Aug 01 2018 Jiri Vymazal - 8.36.0-3 +- dropped json_nonoverwrite patch as there is no reason for + keeping it +- renumbered rest of patches +- added release number to AC_INIT to have it in package error logs + +* Mon Jul 16 2018 Charalampos Stratakis - 8.36.0-2 +- Depend on python3-docutils + +* Mon Jul 02 2018 Jiri Vymazal - 8.36.0-1 +- changed PID file name to follow upstream +- removed config option to disable stdlog as it is now + disabled by default + +* Thu Jun 28 2018 Jiri Vymazal - 8.36.0-1 +- rebase to 8.36 +- removed hiredis module +- removed omudpspoof module + resolves: rhbz#1593762 +- finished converting config to new-style syntax + +* Mon May 21 2018 Jiri Vymazal - 8.35.0-1 +- spec file cleanup +- enabled kafka and hiredis modules + resolves: rhbz#1542497 + resolves: rhbz#1542504 +- renamed patch fixing imjournal duplicating messages + resolves: rhbz#1544394 + +* Thu May 17 2018 Marek Tamaskovic - 8.35.0-1 +- rebase to 8.35 +- rebased patches from 8.32 to 8.35 + - fixed imjournal-duplicates + - fixed imjournal-default-tag + - fixed service patch + - fixed in upstream deserialize-property-name + +* Fri Mar 23 2018 Radovan Sroka - 8.32.0-2 +- rebuild, bumped release number + +* Tue Feb 06 2018 Radovan Sroka - 8.32.0-1 +- initial clean build with plugins from rhel7 +- removed plugins: + - libdbi + - omruleset + - pmrfc3164sd +- imported from fedora26