A System and Service Manager
CentOS Sources
2018-03-06 de541a7e59f844872838e6a4697ad9419d53374e
import systemd-219-42.el7_4.10
10 files added
1 files modified
1495 ■■■■■ changed files
SOURCES/0508-shared-cgroup-utils-_CGROUP_CONTROLLER_MASK_ALL-does.patch 28 ●●●●● patch | view | raw | blame | history
SOURCES/0509-core-execute-fix-fork-fail-handling-in-exec_spawn.patch 30 ●●●●● patch | view | raw | blame | history
SOURCES/0510-journal-remove-error-check-that-never-happens.patch 53 ●●●●● patch | view | raw | blame | history
SOURCES/0511-sd-journal-various-clean-ups-and-modernizations.patch 503 ●●●●● patch | view | raw | blame | history
SOURCES/0512-journalctl-continue-operation-even-if-we-run-into-an.patch 35 ●●●●● patch | view | raw | blame | history
SOURCES/0513-journalctl-when-we-fail-to-open-a-journal-file-print.patch 204 ●●●●● patch | view | raw | blame | history
SOURCES/0514-sd-journal-properly-handle-inotify-queue-overflow.patch 442 ●●●●● patch | view | raw | blame | history
SOURCES/0515-sd-journal-make-sure-it-s-safe-to-call-sd_journal_pr.patch 36 ●●●●● patch | view | raw | blame | history
SOURCES/0516-sd-journal-when-picking-up-a-new-file-compare-inode-.patch 77 ●●●●● patch | view | raw | blame | history
SOURCES/0517-journalctl-Periodically-call-sd_journal_process-in-j.patch 59 ●●●●● patch | view | raw | blame | history
SPECS/systemd.spec 28 ●●●●● patch | view | raw | blame | history
SOURCES/0508-shared-cgroup-utils-_CGROUP_CONTROLLER_MASK_ALL-does.patch
New file
@@ -0,0 +1,28 @@
From 839a1935c5433fe973008f231aeb247df0344eca Mon Sep 17 00:00:00 2001
From: Lukas Nykryn <lnykryn@redhat.com>
Date: Tue, 9 Jan 2018 12:59:19 +0100
Subject: [PATCH] shared/cgroup-utils: _CGROUP_CONTROLLER_MASK_ALL does not
 cover CGROUP_PIDS
7d44d0d43465892d4753ff50592588f49d56cf95 added a CGROUP_PIDS but
did not bump _CGROUP_CONTROLLER_MASK_ALL.
RHEL-only
Resolves: #1532586
---
 src/shared/cgroup-util.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/src/shared/cgroup-util.h b/src/shared/cgroup-util.h
index 31bd8d311..e76cd334d 100644
--- a/src/shared/cgroup-util.h
+++ b/src/shared/cgroup-util.h
@@ -36,7 +36,7 @@ typedef enum CGroupControllerMask {
         CGROUP_MEMORY = 8,
         CGROUP_DEVICE = 16,
         CGROUP_PIDS = 32,
-        _CGROUP_CONTROLLER_MASK_ALL = 31
+        _CGROUP_CONTROLLER_MASK_ALL = 63
 } CGroupControllerMask;
 /*
SOURCES/0509-core-execute-fix-fork-fail-handling-in-exec_spawn.patch
New file
@@ -0,0 +1,30 @@
From 7ebf4a0faffecb3a0c8abe8bea47502044038d66 Mon Sep 17 00:00:00 2001
From: lc85446 <lc85446@alibaba-inc.com>
Date: Thu, 26 Nov 2015 11:46:40 +0800
Subject: [PATCH] core:execute: fix fork() fail handling in exec_spawn()
If pid < 0 after fork(), 0 is always returned because r =
exec_context_load_environment() has exited successfully.
This will make the caller of exec_spawn() not able to handle
the fork() error case and make systemd abort assert() possibly.
Cherry-picked from: 74129a127676e4f0edac0db4296c103e76ec6694
Resolves: #1437114
---
 src/core/execute.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/src/core/execute.c b/src/core/execute.c
index 4265b9c34..e68276973 100644
--- a/src/core/execute.c
+++ b/src/core/execute.c
@@ -1977,7 +1977,7 @@ int exec_spawn(ExecCommand *command,
                         NULL);
         pid = fork();
         if (pid < 0)
-                return log_unit_error_errno(params->unit_id, r, "Failed to fork: %m");
+                return log_unit_error_errno(params->unit_id, errno, "Failed to fork: %m");
         if (pid == 0) {
                 int exit_status;
SOURCES/0510-journal-remove-error-check-that-never-happens.patch
New file
@@ -0,0 +1,53 @@
From cc8cc45e4b7799ac1dad7701de2df3db4fbb790c Mon Sep 17 00:00:00 2001
From: Thomas Hindoe Paaboel Andersen <phomes@gmail.com>
Date: Fri, 14 Aug 2015 23:40:27 +0200
Subject: [PATCH] journal: remove error check that never happens
remove_directory will always return 0 so this can never happen.
Besides that, d->path and d are freed so we would end up with
a null pointer dereference anyway.
(cherry picked from commit b2b46f91dbb71676cb981907c68521e4b1e80af1)
Related: #1465759
---
 src/journal/sd-journal.c | 12 +++---------
 1 file changed, 3 insertions(+), 9 deletions(-)
diff --git a/src/journal/sd-journal.c b/src/journal/sd-journal.c
index 72f312b67..3749f9e89 100644
--- a/src/journal/sd-journal.c
+++ b/src/journal/sd-journal.c
@@ -1487,7 +1487,7 @@ static int add_root_directory(sd_journal *j, const char *p) {
         return 0;
 }
-static int remove_directory(sd_journal *j, Directory *d) {
+static void remove_directory(sd_journal *j, Directory *d) {
         assert(j);
         if (d->wd > 0) {
@@ -1506,8 +1506,6 @@ static int remove_directory(sd_journal *j, Directory *d) {
         free(d->path);
         free(d);
-
-        return 0;
 }
 static int add_search_paths(sd_journal *j) {
@@ -2145,12 +2143,8 @@ static void process_inotify_event(sd_journal *j, struct inotify_event *e) {
                         /* Event for a subdirectory */
-                        if (e->mask & (IN_DELETE_SELF|IN_MOVE_SELF|IN_UNMOUNT)) {
-                                r = remove_directory(j, d);
-                                if (r < 0)
-                                        log_debug_errno(r, "Failed to remove directory %s: %m", d->path);
-                        }
-
+                        if (e->mask & (IN_DELETE_SELF|IN_MOVE_SELF|IN_UNMOUNT))
+                                remove_directory(j, d);
                 } else if (d->is_root && (e->mask & IN_ISDIR) && e->len > 0 && sd_id128_from_string(e->name, &id) >= 0) {
SOURCES/0511-sd-journal-various-clean-ups-and-modernizations.patch
New file
@@ -0,0 +1,503 @@
From ee0e6e54479f8ad1991e531bb1e931b696d67aaf Mon Sep 17 00:00:00 2001
From: Lennart Poettering <lennart@poettering.net>
Date: Mon, 2 Nov 2015 23:14:30 +0100
Subject: [PATCH] sd-journal: various clean-ups and modernizations
- Always print a debug log message about files and directories we cannot
  open right when it happens instead of the caller, thus reducing the
  number of places where we need to generate the debug message.
- Always push the errors we encounter immediately into the error set,
  when we run into them, instead of in the caller. Thus, we never forget
  to push them in.
- Use stack instead of heap memory where we can.
- Make remove_file() void, since it cannot fail anyway and always
  returned 0.
- Make local machine check of journal directories explicit in a
  function, to make things more readable.
- Port to all directory listing loops FOREACH_DIRENT_ALL()
- sd-daemon is library code, hence never log at higher log levels than
  LOG_DEBUG.
(cherry picked from commit d617408ecbe69db69aefddfcb10a6c054ea46ba0)
Related: #1465759
---
 src/journal/sd-journal.c | 242 ++++++++++++++++++++++-------------------------
 1 file changed, 111 insertions(+), 131 deletions(-)
diff --git a/src/journal/sd-journal.c b/src/journal/sd-journal.c
index 3749f9e89..9895d9608 100644
--- a/src/journal/sd-journal.c
+++ b/src/journal/sd-journal.c
@@ -1171,6 +1171,8 @@ static bool file_has_type_prefix(const char *prefix, const char *filename) {
 }
 static bool file_type_wanted(int flags, const char *filename) {
+        assert(filename);
+
         if (!endswith(filename, ".journal") && !endswith(filename, ".journal~"))
                 return false;
@@ -1195,7 +1197,7 @@ static bool file_type_wanted(int flags, const char *filename) {
 static int add_any_file(sd_journal *j, const char *path) {
         JournalFile *f = NULL;
-        int r;
+        int r, k;
         assert(j);
         assert(path);
@@ -1204,20 +1206,23 @@ static int add_any_file(sd_journal *j, const char *path) {
                 return 0;
         if (ordered_hashmap_size(j->files) >= JOURNAL_FILES_MAX) {
-                log_warning("Too many open journal files, not adding %s.", path);
-                return set_put_error(j, -ETOOMANYREFS);
+                log_debug("Too many open journal files, not adding %s.", path);
+                r = -ETOOMANYREFS;
+                goto fail;
         }
         r = journal_file_open(path, O_RDONLY, 0, false, false, NULL, j->mmap, NULL, &f);
-        if (r < 0)
-                return r;
+        if (r < 0) {
+                log_debug_errno(r, "Failed to open journal file %s: %m", path);
+                goto fail;
+        }
         /* journal_file_dump(f); */
         r = ordered_hashmap_put(j->files, f->path, f);
         if (r < 0) {
                 journal_file_close(f);
-                return r;
+                goto fail;
         }
         log_debug("File %s added.", f->path);
@@ -1227,10 +1232,17 @@ static int add_any_file(sd_journal *j, const char *path) {
         j->current_invalidate_counter ++;
         return 0;
+
+fail:
+        k = set_put_error(j, r);
+        if (k < 0)
+                return k;
+
+        return r;
 }
 static int add_file(sd_journal *j, const char *prefix, const char *filename) {
-        char *path = NULL;
+        const char *path;
         assert(j);
         assert(prefix);
@@ -1250,24 +1262,20 @@ static int add_file(sd_journal *j, const char *prefix, const char *filename) {
         return add_any_file(j, path);
 }
-static int remove_file(sd_journal *j, const char *prefix, const char *filename) {
-        _cleanup_free_ char *path;
+static void remove_file(sd_journal *j, const char *prefix, const char *filename) {
+        const char *path;
         JournalFile *f;
         assert(j);
         assert(prefix);
         assert(filename);
-        path = strjoin(prefix, "/", filename, NULL);
-        if (!path)
-                return -ENOMEM;
-
+        path = strjoina(prefix, "/", filename);
         f = ordered_hashmap_get(j->files, path);
         if (!f)
-                return 0;
+                return;
         remove_file_real(j, f);
-        return 0;
 }
 static void remove_file_real(sd_journal *j, JournalFile *f) {
@@ -1296,12 +1304,27 @@ static void remove_file_real(sd_journal *j, JournalFile *f) {
         j->current_invalidate_counter ++;
 }
+static int dirname_is_machine_id(const char *fn) {
+        sd_id128_t id, machine;
+        int r;
+
+        r = sd_id128_get_machine(&machine);
+        if (r < 0)
+                return r;
+
+        r = sd_id128_from_string(fn, &id);
+        if (r < 0)
+                return r;
+
+        return sd_id128_equal(id, machine);
+}
+
 static int add_directory(sd_journal *j, const char *prefix, const char *dirname) {
         _cleanup_free_ char *path = NULL;
-        int r;
         _cleanup_closedir_ DIR *d = NULL;
-        sd_id128_t id, mid;
+        struct dirent *de = NULL;
         Directory *m;
+        int r, k;
         assert(j);
         assert(prefix);
@@ -1310,35 +1333,36 @@ static int add_directory(sd_journal *j, const char *prefix, const char *dirname)
         log_debug("Considering %s/%s.", prefix, dirname);
         if ((j->flags & SD_JOURNAL_LOCAL_ONLY) &&
-            (sd_id128_from_string(dirname, &id) < 0 ||
-             sd_id128_get_machine(&mid) < 0 ||
-             !(sd_id128_equal(id, mid) || path_startswith(prefix, "/run"))))
+            !(dirname_is_machine_id(dirname) > 0 || path_startswith(prefix, "/run")))
             return 0;
         path = strjoin(prefix, "/", dirname, NULL);
-        if (!path)
-                return -ENOMEM;
+        if (!path) {
+                r = -ENOMEM;
+                goto fail;
+        }
         d = opendir(path);
         if (!d) {
-                log_debug_errno(errno, "Failed to open %s: %m", path);
-                if (errno == ENOENT)
-                        return 0;
-                return -errno;
+                r = log_debug_errno(errno, "Failed to open directory %s: %m", path);
+                goto fail;
         }
         m = hashmap_get(j->directories_by_path, path);
         if (!m) {
                 m = new0(Directory, 1);
-                if (!m)
-                        return -ENOMEM;
+                if (!m) {
+                        r = -ENOMEM;
+                        goto fail;
+                }
                 m->is_root = false;
                 m->path = path;
                 if (hashmap_put(j->directories_by_path, m->path, m) < 0) {
                         free(m);
-                        return -ENOMEM;
+                        r = -ENOMEM;
+                        goto fail;
                 }
                 path = NULL; /* avoid freeing in cleanup */
@@ -1360,41 +1384,30 @@ static int add_directory(sd_journal *j, const char *prefix, const char *dirname)
                         inotify_rm_watch(j->inotify_fd, m->wd);
         }
-        for (;;) {
-                struct dirent *de;
-
-                errno = 0;
-                de = readdir(d);
-                if (!de && errno != 0) {
-                        r = -errno;
-                        log_debug_errno(errno, "Failed to read directory %s: %m", m->path);
-                        return r;
-                }
-                if (!de)
-                        break;
+        FOREACH_DIRENT_ALL(de, d, return log_debug_errno(errno, "Failed to read directory %s: %m", m->path)) {
                 if (dirent_is_file_with_suffix(de, ".journal") ||
-                    dirent_is_file_with_suffix(de, ".journal~")) {
-                        r = add_file(j, m->path, de->d_name);
-                        if (r < 0) {
-                                log_debug_errno(r, "Failed to add file %s/%s: %m",
-                                                m->path, de->d_name);
-                                r = set_put_error(j, r);
-                                if (r < 0)
-                                        return r;
-                        }
-                }
+                    dirent_is_file_with_suffix(de, ".journal~"))
+                        (void) add_file(j, m->path, de->d_name);
         }
         check_network(j, dirfd(d));
         return 0;
+
+fail:
+        k = set_put_error(j, r);
+        if (k < 0)
+                return k;
+
+        return r;
 }
-static int add_root_directory(sd_journal *j, const char *p) {
+static int add_root_directory(sd_journal *j, const char *p, bool missing_ok) {
         _cleanup_closedir_ DIR *d = NULL;
+        struct dirent *de;
         Directory *m;
-        int r;
+        int r, k;
         assert(j);
         assert(p);
@@ -1407,26 +1420,35 @@ static int add_root_directory(sd_journal *j, const char *p) {
                 p = strjoina(j->prefix, p);
         d = opendir(p);
-        if (!d)
-                return -errno;
+        if (!d) {
+                if (errno == ENOENT && missing_ok)
+                        return 0;
+
+                r = log_debug_errno(errno, "Failed to open root directory %s: %m", p);
+                goto fail;
+        }
         m = hashmap_get(j->directories_by_path, p);
         if (!m) {
                 m = new0(Directory, 1);
-                if (!m)
-                        return -ENOMEM;
+                if (!m) {
+                        r = -ENOMEM;
+                        goto fail;
+                }
                 m->is_root = true;
                 m->path = strdup(p);
                 if (!m->path) {
                         free(m);
-                        return -ENOMEM;
+                        r = -ENOMEM;
+                        goto fail;
                 }
                 if (hashmap_put(j->directories_by_path, m->path, m) < 0) {
                         free(m->path);
                         free(m);
-                        return -ENOMEM;
+                        r = -ENOMEM;
+                        goto fail;
                 }
                 j->current_invalidate_counter ++;
@@ -1449,42 +1471,27 @@ static int add_root_directory(sd_journal *j, const char *p) {
         if (j->no_new_files)
                 return 0;
-        for (;;) {
-                struct dirent *de;
+        FOREACH_DIRENT_ALL(de, d, return log_debug_errno(errno, "Failed to read directory %s: %m", m->path)) {
                 sd_id128_t id;
-                errno = 0;
-                de = readdir(d);
-                if (!de && errno != 0) {
-                        r = -errno;
-                        log_debug_errno(errno, "Failed to read directory %s: %m", m->path);
-                        return r;
-                }
-                if (!de)
-                        break;
-
                 if (dirent_is_file_with_suffix(de, ".journal") ||
-                    dirent_is_file_with_suffix(de, ".journal~")) {
-                        r = add_file(j, m->path, de->d_name);
-                        if (r < 0) {
-                                log_debug_errno(r, "Failed to add file %s/%s: %m",
-                                                m->path, de->d_name);
-                                r = set_put_error(j, r);
-                                if (r < 0)
-                                        return r;
-                        }
-                } else if ((de->d_type == DT_DIR || de->d_type == DT_LNK || de->d_type == DT_UNKNOWN) &&
-                           sd_id128_from_string(de->d_name, &id) >= 0) {
-
-                        r = add_directory(j, m->path, de->d_name);
-                        if (r < 0)
-                                log_debug_errno(r, "Failed to add directory %s/%s: %m", m->path, de->d_name);
-                }
+                    dirent_is_file_with_suffix(de, ".journal~"))
+                        (void) add_file(j, m->path, de->d_name);
+                else if (IN_SET(de->d_type, DT_DIR, DT_LNK, DT_UNKNOWN) &&
+                         sd_id128_from_string(de->d_name, &id) >= 0)
+                        (void) add_directory(j, m->path, de->d_name);
         }
         check_network(j, dirfd(d));
         return 0;
+
+fail:
+        k = set_put_error(j, r);
+        if (k < 0)
+                return k;
+
+        return r;
 }
 static void remove_directory(sd_journal *j, Directory *d) {
@@ -1509,8 +1516,8 @@ static void remove_directory(sd_journal *j, Directory *d) {
 }
 static int add_search_paths(sd_journal *j) {
-        int r;
-        const char search_paths[] =
+
+        static const char search_paths[] =
                 "/run/log/journal\0"
                 "/var/log/journal\0";
         const char *p;
@@ -1520,14 +1527,8 @@ static int add_search_paths(sd_journal *j) {
         /* We ignore most errors here, since the idea is to only open
          * what's actually accessible, and ignore the rest. */
-        NULSTR_FOREACH(p, search_paths) {
-                r = add_root_directory(j, p);
-                if (r < 0 && r != -ENOENT) {
-                        r = set_put_error(j, r);
-                        if (r < 0)
-                                return r;
-                }
-        }
+        NULSTR_FOREACH(p, search_paths)
+                (void) add_root_directory(j, p, true);
         return 0;
 }
@@ -1551,17 +1552,14 @@ static int add_current_paths(sd_journal *j) {
                 if (!dir)
                         return -ENOMEM;
-                r = add_root_directory(j, dir);
-                if (r < 0) {
-                        set_put_error(j, r);
+                r = add_root_directory(j, dir, true);
+                if (r < 0)
                         return r;
-                }
         }
         return 0;
 }
-
 static int allocate_inotify(sd_journal *j) {
         assert(j);
@@ -1689,11 +1687,9 @@ _public_ int sd_journal_open_directory(sd_journal **ret, const char *path, int f
         if (!j)
                 return -ENOMEM;
-        r = add_root_directory(j, path);
-        if (r < 0) {
-                set_put_error(j, r);
+        r = add_root_directory(j, path, false);
+        if (r < 0)
                 goto fail;
-        }
         *ret = j;
         return 0;
@@ -1718,10 +1714,8 @@ _public_ int sd_journal_open_files(sd_journal **ret, const char **paths, int fla
         STRV_FOREACH(path, paths) {
                 r = add_any_file(j, *path);
-                if (r < 0) {
-                        log_error_errno(r, "Failed to open %s: %m", *path);
+                if (r < 0)
                         goto fail;
-                }
         }
         j->no_new_files = true;
@@ -2061,7 +2055,7 @@ _public_ int sd_journal_get_fd(sd_journal *j) {
         if (j->no_new_files)
                 r = add_current_paths(j);
         else if (j->path)
-                r = add_root_directory(j, j->path);
+                r = add_root_directory(j, j->path, true);
         else
                 r = add_search_paths(j);
         if (r < 0)
@@ -2108,7 +2102,6 @@ _public_ int sd_journal_get_timeout(sd_journal *j, uint64_t *timeout_usec) {
 static void process_inotify_event(sd_journal *j, struct inotify_event *e) {
         Directory *d;
-        int r;
         assert(j);
         assert(e);
@@ -2124,20 +2117,10 @@ static void process_inotify_event(sd_journal *j, struct inotify_event *e) {
                         /* Event for a journal file */
-                        if (e->mask & (IN_CREATE|IN_MOVED_TO|IN_MODIFY|IN_ATTRIB)) {
-                                r = add_file(j, d->path, e->name);
-                                if (r < 0) {
-                                        log_debug_errno(r, "Failed to add file %s/%s: %m",
-                                                        d->path, e->name);
-                                        set_put_error(j, r);
-                                }
-
-                        } else if (e->mask & (IN_DELETE|IN_MOVED_FROM|IN_UNMOUNT)) {
-
-                                r = remove_file(j, d->path, e->name);
-                                if (r < 0)
-                                        log_debug_errno(r, "Failed to remove file %s/%s: %m", d->path, e->name);
-                        }
+                        if (e->mask & (IN_CREATE|IN_MOVED_TO|IN_MODIFY|IN_ATTRIB))
+                                (void) add_file(j, d->path, e->name);
+                        else if (e->mask & (IN_DELETE|IN_MOVED_FROM|IN_UNMOUNT))
+                                remove_file(j, d->path, e->name);
                 } else if (!d->is_root && e->len == 0) {
@@ -2150,11 +2133,8 @@ static void process_inotify_event(sd_journal *j, struct inotify_event *e) {
                         /* Event for root directory */
-                        if (e->mask & (IN_CREATE|IN_MOVED_TO|IN_MODIFY|IN_ATTRIB)) {
-                                r = add_directory(j, d->path, e->name);
-                                if (r < 0)
-                                        log_debug_errno(r, "Failed to add directory %s/%s: %m", d->path, e->name);
-                        }
+                        if (e->mask & (IN_CREATE|IN_MOVED_TO|IN_MODIFY|IN_ATTRIB))
+                                (void) add_directory(j, d->path, e->name);
                 }
                 return;
@@ -2163,7 +2143,7 @@ static void process_inotify_event(sd_journal *j, struct inotify_event *e) {
         if (e->mask & IN_IGNORED)
                 return;
-        log_warning("Unknown inotify event.");
+        log_debug("Unknown inotify event.");
 }
 static int determine_change(sd_journal *j) {
SOURCES/0512-journalctl-continue-operation-even-if-we-run-into-an.patch
New file
@@ -0,0 +1,35 @@
From f5a2a7b3630d0ba9a25f8a81cdeaf5336b745e3e Mon Sep 17 00:00:00 2001
From: Lennart Poettering <lennart@poettering.net>
Date: Mon, 2 Nov 2015 23:13:01 +0100
Subject: [PATCH] journalctl: continue operation, even if we run into an
 invalid file
(cherry picked from commit 4f52b822b05c373f40fea1a41ae3ade5d5ff558e)
Related: #1465759
---
 src/journal/journalctl.c | 7 ++-----
 1 file changed, 2 insertions(+), 5 deletions(-)
diff --git a/src/journal/journalctl.c b/src/journal/journalctl.c
index c771cff8b..8c8379732 100644
--- a/src/journal/journalctl.c
+++ b/src/journal/journalctl.c
@@ -1804,15 +1804,12 @@ static int access_check(sd_journal *j) {
         SET_FOREACH(code, j->errors, it) {
                 int err;
-                err = -PTR_TO_INT(code);
-                assert(err > 0);
+                err = abs(PTR_TO_INT(code));
                 if (err == EACCES)
                         continue;
-                log_warning_errno(err, "Error was encountered while opening journal files: %m");
-                if (r == 0)
-                        r = -err;
+                log_warning_errno(err, "An error was encountered while opening journal files, ignoring: %m");
         }
         return r;
SOURCES/0513-journalctl-when-we-fail-to-open-a-journal-file-print.patch
New file
@@ -0,0 +1,204 @@
From 8a5944217f444929ba6a4124a4ee7fbd2c5a3fc3 Mon Sep 17 00:00:00 2001
From: Lennart Poettering <lennart@poettering.net>
Date: Mon, 2 Nov 2015 23:37:05 +0100
Subject: [PATCH] journalctl: when we fail to open a journal file, print why
When we enumerate journal files and encounter an invalid one, remember
which this, and show it to the user.
Note the possibly slightly surprising logic here: we store only one path
per error code. This means we show all error kinds but not every actual
error we encounter. This has the benefit of not requiring us to keep a
potentially unbounded list of errors with their sources around, but can
still provide a pretty complete overview on the errors we encountered.
Fixes #1669.
(cherry picked from commit 5768d2594940668506bb4cafa078f654cc20dc5a)
Resolves: #1465759
---
 src/journal/journal-internal.h |  2 +-
 src/journal/journalctl.c       | 27 ++++++++++++++++++-----
 src/journal/sd-journal.c       | 49 ++++++++++++++++++++++++++++++++++--------
 3 files changed, 63 insertions(+), 15 deletions(-)
diff --git a/src/journal/journal-internal.h b/src/journal/journal-internal.h
index 115d7776d..eb23ac28a 100644
--- a/src/journal/journal-internal.h
+++ b/src/journal/journal-internal.h
@@ -123,7 +123,7 @@ struct sd_journal {
         Hashmap *directories_by_path;
         Hashmap *directories_by_wd;
-        Set *errors;
+        Hashmap *errors;
 };
 char *journal_make_match_string(sd_journal *j);
diff --git a/src/journal/journalctl.c b/src/journal/journalctl.c
index 8c8379732..0be70764e 100644
--- a/src/journal/journalctl.c
+++ b/src/journal/journalctl.c
@@ -1783,33 +1783,50 @@ static int access_check_var_log_journal(sd_journal *j) {
 static int access_check(sd_journal *j) {
         Iterator it;
         void *code;
+        char *path;
         int r = 0;
         assert(j);
-        if (set_isempty(j->errors)) {
+        if (hashmap_isempty(j->errors)) {
                 if (ordered_hashmap_isempty(j->files))
                         log_notice("No journal files were found.");
                 return 0;
         }
-        if (set_contains(j->errors, INT_TO_PTR(-EACCES))) {
+        if (hashmap_contains(j->errors, INT_TO_PTR(-EACCES))) {
                 (void) access_check_var_log_journal(j);
                 if (ordered_hashmap_isempty(j->files))
                         r = log_error_errno(EACCES, "No journal files were opened due to insufficient permissions.");
         }
-        SET_FOREACH(code, j->errors, it) {
+        HASHMAP_FOREACH_KEY(path, code, j->errors, it) {
                 int err;
                 err = abs(PTR_TO_INT(code));
-                if (err == EACCES)
+                switch (err) {
+                case EACCES:
                         continue;
-                log_warning_errno(err, "An error was encountered while opening journal files, ignoring: %m");
+                case ENODATA:
+                        log_warning_errno(err, "Journal file %s is truncated, ignoring file.", path);
+                        break;
+
+                case EPROTONOSUPPORT:
+                        log_warning_errno(err, "Journal file %s uses an unsupported feature, ignoring file.", path);
+                        break;
+
+                case EBADMSG:
+                        log_warning_errno(err, "Journal file %s corrupted, ignoring file.", path);
+                        break;
+
+                default:
+                        log_warning_errno(err, "An error was encountered while opening journal file %s, ignoring file.", path);
+                        break;
+                }
         }
         return r;
diff --git a/src/journal/sd-journal.c b/src/journal/sd-journal.c
index 9895d9608..14b65cfed 100644
--- a/src/journal/sd-journal.c
+++ b/src/journal/sd-journal.c
@@ -62,19 +62,46 @@ static bool journal_pid_changed(sd_journal *j) {
         return j->original_pid != getpid();
 }
-/* We return an error here only if we didn't manage to
-   memorize the real error. */
-static int set_put_error(sd_journal *j, int r) {
+static int journal_put_error(sd_journal *j, int r, const char *path) {
+        char *copy;
         int k;
+        /* Memorize an error we encountered, and store which
+         * file/directory it was generated from. Note that we store
+         * only *one* path per error code, as the error code is the
+         * key into the hashmap, and the path is the value. This means
+         * we keep track only of all error kinds, but not of all error
+         * locations. This has the benefit that the hashmap cannot
+         * grow beyond bounds.
+         *
+         * We return an error here only if we didn't manage to
+         * memorize the real error. */
+
         if (r >= 0)
                 return r;
-        k = set_ensure_allocated(&j->errors, NULL);
+        k = hashmap_ensure_allocated(&j->errors, NULL);
         if (k < 0)
                 return k;
-        return set_put(j->errors, INT_TO_PTR(r));
+        if (path) {
+                copy = strdup(path);
+                if (!copy)
+                        return -ENOMEM;
+        } else
+                copy = NULL;
+
+        k = hashmap_put(j->errors, INT_TO_PTR(r), copy);
+        if (k < 0) {
+                free(copy);
+
+                if (k == -EEXIST)
+                        return 0;
+
+                return k;
+        }
+
+        return 0;
 }
 static void detach_location(sd_journal *j) {
@@ -1234,7 +1261,7 @@ static int add_any_file(sd_journal *j, const char *path) {
         return 0;
 fail:
-        k = set_put_error(j, r);
+        k = journal_put_error(j, r, path);
         if (k < 0)
                 return k;
@@ -1396,7 +1423,7 @@ static int add_directory(sd_journal *j, const char *prefix, const char *dirname)
         return 0;
 fail:
-        k = set_put_error(j, r);
+        k = journal_put_error(j, r, path ?: dirname);
         if (k < 0)
                 return k;
@@ -1487,7 +1514,7 @@ static int add_root_directory(sd_journal *j, const char *p, bool missing_ok) {
         return 0;
 fail:
-        k = set_put_error(j, r);
+        k = journal_put_error(j, r, p);
         if (k < 0)
                 return k;
@@ -1732,6 +1759,7 @@ fail:
 _public_ void sd_journal_close(sd_journal *j) {
         Directory *d;
         JournalFile *f;
+        char *p;
         if (!j)
                 return;
@@ -1759,10 +1787,13 @@ _public_ void sd_journal_close(sd_journal *j) {
                 mmap_cache_unref(j->mmap);
         }
+        while ((p = hashmap_steal_first(j->errors)))
+                free(p);
+        hashmap_free(j->errors);
+
         free(j->path);
         free(j->prefix);
         free(j->unique_field);
-        set_free(j->errors);
         free(j);
 }
SOURCES/0514-sd-journal-properly-handle-inotify-queue-overflow.patch
New file
@@ -0,0 +1,442 @@
From afb8109dd1968e6353dbdda13e6216e12f2dec03 Mon Sep 17 00:00:00 2001
From: Lennart Poettering <lennart@poettering.net>
Date: Mon, 12 Feb 2018 16:14:58 +0100
Subject: [PATCH] sd-journal: properly handle inotify queue overflow
This adds proper handling of IN_Q_OVERFLOW: when the inotify queue runs
over we'll reiterate all directories we are looking at. At the same time
we'll mark all files and directories we encounter that way with a
generation counter we first increased. All files and directories not
marked like this are then unloaded.
With this logic we do the best when the inotify queue overflows: we
synchronize our in-memory state again with what's on disk.  This
contains some refactoring of the directory logic, to share more code
between uuid directories and "root" directories and generally make
things a bit more readable by splitting things up into smaller bits.
See: #7998 #8032
(cherry-picked from commit 858749f7312bd0adb5433075a92e1c35a2fb56ac)
Resolves: #1540538
---
 src/journal/journal-file.h     |   2 +
 src/journal/journal-internal.h |   2 +
 src/journal/sd-journal.c       | 237 ++++++++++++++++++++++++++++++++---------
 src/shared/path-util.c         |  14 +++
 src/shared/path-util.h         |   2 +
 5 files changed, 206 insertions(+), 51 deletions(-)
diff --git a/src/journal/journal-file.h b/src/journal/journal-file.h
index c74ad5fc5..dd8ef52d2 100644
--- a/src/journal/journal-file.h
+++ b/src/journal/journal-file.h
@@ -121,6 +121,8 @@ typedef struct JournalFile {
         void *fsprg_seed;
         size_t fsprg_seed_size;
+
+        unsigned last_seen_generation;
 #endif
 } JournalFile;
diff --git a/src/journal/journal-internal.h b/src/journal/journal-internal.h
index eb23ac28a..999e9d8cb 100644
--- a/src/journal/journal-internal.h
+++ b/src/journal/journal-internal.h
@@ -81,6 +81,7 @@ struct Directory {
         char *path;
         int wd;
         bool is_root;
+        unsigned last_seen_generation;
 };
 struct sd_journal {
@@ -102,6 +103,7 @@ struct sd_journal {
         int inotify_fd;
         unsigned current_invalidate_counter, last_invalidate_counter;
         usec_t last_process_usec;
+        unsigned generation;
         char *unique_field;
         JournalFile *unique_file;
diff --git a/src/journal/sd-journal.c b/src/journal/sd-journal.c
index 14b65cfed..9186f5188 100644
--- a/src/journal/sd-journal.c
+++ b/src/journal/sd-journal.c
@@ -1229,8 +1229,16 @@ static int add_any_file(sd_journal *j, const char *path) {
         assert(j);
         assert(path);
-        if (ordered_hashmap_get(j->files, path))
-                return 0;
+        if (path) {
+                f = ordered_hashmap_get(j->files, path);
+                if (f) {
+                        /* Mark this file as seen in this generation. This is used to GC old files in
+                         * process_q_overflow() to detect journal files that are still and discern them from those who
+                         * are gone. */
+                        f->last_seen_generation = j->generation;
+                        return 0;
+                }
+        }
         if (ordered_hashmap_size(j->files) >= JOURNAL_FILES_MAX) {
                 log_debug("Too many open journal files, not adding %s.", path);
@@ -1252,6 +1260,8 @@ static int add_any_file(sd_journal *j, const char *path) {
                 goto fail;
         }
+        f->last_seen_generation = j->generation;
+
         log_debug("File %s added.", f->path);
         check_network(j, f->fd);
@@ -1346,10 +1356,96 @@ static int dirname_is_machine_id(const char *fn) {
         return sd_id128_equal(id, machine);
 }
+static bool dirent_is_journal_file(const struct dirent *de) {
+        assert(de);
+
+        if (!IN_SET(de->d_type, DT_REG, DT_LNK, DT_UNKNOWN))
+                return false;
+
+        return endswith(de->d_name, ".journal") ||
+                endswith(de->d_name, ".journal~");
+}
+
+static bool dirent_is_id128_subdir(const struct dirent *de) {
+        assert(de);
+
+        if (!IN_SET(de->d_type, DT_DIR, DT_LNK, DT_UNKNOWN))
+                return false;
+
+        return id128_is_valid(de->d_name);
+}
+
+static int directory_open(sd_journal *j, const char *path, DIR **ret) {
+        DIR *d;
+
+        assert(j);
+        assert(path);
+        assert(ret);
+
+        d = opendir(path);
+        if (!d)
+                return -errno;
+
+        *ret = d;
+        return 0;
+}
+
+static int add_directory(sd_journal *j, const char *prefix, const char *dirname);
+
+static void directory_enumerate(sd_journal *j, Directory *m, DIR *d) {
+        struct dirent *de;
+
+        assert(j);
+        assert(m);
+        assert(d);
+
+        FOREACH_DIRENT_ALL(de, d, goto fail) {
+                if (dirent_is_journal_file(de))
+                        (void) add_file(j, m->path, de->d_name);
+
+                if (m->is_root && dirent_is_id128_subdir(de))
+                        (void) add_directory(j, m->path, de->d_name);
+        }
+
+        return;
+
+fail:
+        log_debug_errno(errno, "Failed to enumerate directory %s, ignoring: %m", m->path);
+}
+
+static void directory_watch(sd_journal *j, Directory *m, int fd, uint32_t mask) {
+        int r;
+
+        assert(j);
+        assert(m);
+        assert(fd >= 0);
+
+        /* Watch this directory if that's enabled and if it not being watched yet. */
+
+        if (m->wd > 0) /* Already have a watch? */
+                return;
+        if (j->inotify_fd < 0) /* Not watching at all? */
+                return;
+
+        m->wd = inotify_add_watch_fd(j->inotify_fd, fd, mask);
+        if (m->wd < 0) {
+                log_debug_errno(errno, "Failed to watch journal directory '%s', ignoring: %m", m->path);
+                return;
+        }
+
+        r = hashmap_put(j->directories_by_wd, INT_TO_PTR(m->wd), m);
+        if (r == -EEXIST)
+                log_debug_errno(r, "Directory '%s' already being watched under a different path, ignoring: %m", m->path);
+        if (r < 0) {
+                log_debug_errno(r, "Failed to add watch for journal directory '%s' to hashmap, ignoring: %m", m->path);
+                (void) inotify_rm_watch(j->inotify_fd, m->wd);
+                m->wd = -1;
+        }
+}
+
 static int add_directory(sd_journal *j, const char *prefix, const char *dirname) {
         _cleanup_free_ char *path = NULL;
         _cleanup_closedir_ DIR *d = NULL;
-        struct dirent *de = NULL;
         Directory *m;
         int r, k;
@@ -1357,7 +1453,7 @@ static int add_directory(sd_journal *j, const char *prefix, const char *dirname)
         assert(prefix);
         assert(dirname);
-        log_debug("Considering %s/%s.", prefix, dirname);
+        log_debug("Considering '%s/%s'.", prefix, dirname);
         if ((j->flags & SD_JOURNAL_LOCAL_ONLY) &&
             !(dirname_is_machine_id(dirname) > 0 || path_startswith(prefix, "/run")))
@@ -1369,9 +1465,9 @@ static int add_directory(sd_journal *j, const char *prefix, const char *dirname)
                 goto fail;
         }
-        d = opendir(path);
-        if (!d) {
-                r = log_debug_errno(errno, "Failed to open directory %s: %m", path);
+        r = directory_open(j, path, &d);
+        if (r < 0) {
+                r = log_debug_errno(errno, "Failed to open directory '%s': %m", path);
                 goto fail;
         }
@@ -1398,25 +1494,17 @@ static int add_directory(sd_journal *j, const char *prefix, const char *dirname)
                 log_debug("Directory %s added.", m->path);
         } else if (m->is_root)
-                return 0;
-
-        if (m->wd <= 0 && j->inotify_fd >= 0) {
-
-                m->wd = inotify_add_watch(j->inotify_fd, m->path,
-                                          IN_CREATE|IN_MOVED_TO|IN_MODIFY|IN_ATTRIB|IN_DELETE|
-                                          IN_DELETE_SELF|IN_MOVE_SELF|IN_UNMOUNT|IN_MOVED_FROM|
-                                          IN_ONLYDIR);
+                return 0; /* Don't 'downgrade' from root directory */
-                if (m->wd > 0 && hashmap_put(j->directories_by_wd, INT_TO_PTR(m->wd), m) < 0)
-                        inotify_rm_watch(j->inotify_fd, m->wd);
-        }
+        m->last_seen_generation = j->generation;
-        FOREACH_DIRENT_ALL(de, d, return log_debug_errno(errno, "Failed to read directory %s: %m", m->path)) {
+        directory_watch(j, m, dirfd(d),
+                        IN_CREATE|IN_MOVED_TO|IN_MODIFY|IN_ATTRIB|IN_DELETE|
+                        IN_DELETE_SELF|IN_MOVE_SELF|IN_UNMOUNT|IN_MOVED_FROM|
+                        IN_ONLYDIR);
-                if (dirent_is_file_with_suffix(de, ".journal") ||
-                    dirent_is_file_with_suffix(de, ".journal~"))
-                        (void) add_file(j, m->path, de->d_name);
-        }
+        if (!j->no_new_files)
+                directory_enumerate(j, m, d);
         check_network(j, dirfd(d));
@@ -1432,13 +1520,14 @@ fail:
 static int add_root_directory(sd_journal *j, const char *p, bool missing_ok) {
         _cleanup_closedir_ DIR *d = NULL;
-        struct dirent *de;
         Directory *m;
         int r, k;
         assert(j);
         assert(p);
+        log_debug("Considering root directory '%s'.", p);
+
         if ((j->flags & SD_JOURNAL_RUNTIME_ONLY) &&
             !path_startswith(p, "/run"))
                 return -EINVAL;
@@ -1446,12 +1535,11 @@ static int add_root_directory(sd_journal *j, const char *p, bool missing_ok) {
         if (j->prefix)
                 p = strjoina(j->prefix, p);
-        d = opendir(p);
-        if (!d) {
-                if (errno == ENOENT && missing_ok)
-                        return 0;
-
-                r = log_debug_errno(errno, "Failed to open root directory %s: %m", p);
+        r = directory_open(j, p, &d);
+        if (r == -ENOENT && missing_ok)
+                return 0;
+        if (r < 0) {
+                log_debug_errno(r, "Failed to open root directory %s: %m", p);
                 goto fail;
         }
@@ -1495,19 +1583,12 @@ static int add_root_directory(sd_journal *j, const char *p, bool missing_ok) {
                         inotify_rm_watch(j->inotify_fd, m->wd);
         }
-        if (j->no_new_files)
-                return 0;
-
-        FOREACH_DIRENT_ALL(de, d, return log_debug_errno(errno, "Failed to read directory %s: %m", m->path)) {
-                sd_id128_t id;
+        directory_watch(j, m, dirfd(d),
+                        IN_CREATE|IN_MOVED_TO|IN_MODIFY|IN_ATTRIB|IN_DELETE|
+                        IN_ONLYDIR);
-                if (dirent_is_file_with_suffix(de, ".journal") ||
-                    dirent_is_file_with_suffix(de, ".journal~"))
-                        (void) add_file(j, m->path, de->d_name);
-                else if (IN_SET(de->d_type, DT_DIR, DT_LNK, DT_UNKNOWN) &&
-                         sd_id128_from_string(de->d_name, &id) >= 0)
-                        (void) add_directory(j, m->path, de->d_name);
-        }
+        if (!j->no_new_files)
+                directory_enumerate(j, m, d);
         check_network(j, dirfd(d));
@@ -2068,6 +2149,18 @@ _public_ void sd_journal_restart_data(sd_journal *j) {
         j->current_field = 0;
 }
+static int reiterate_all_paths(sd_journal *j) {
+        assert(j);
+
+        if (j->no_new_files)
+                return add_current_paths(j);
+
+        if (j->path)
+                return add_root_directory(j, j->path, true);
+
+        return add_search_paths(j);
+}
+
 _public_ int sd_journal_get_fd(sd_journal *j) {
         int r;
@@ -2081,15 +2174,11 @@ _public_ int sd_journal_get_fd(sd_journal *j) {
         if (r < 0)
                 return r;
-        /* Iterate through all dirs again, to add them to the
-         * inotify */
-        if (j->no_new_files)
-                r = add_current_paths(j);
-        else if (j->path)
-                r = add_root_directory(j, j->path, true);
-        else
-                r = add_search_paths(j);
-        if (r < 0)
+         log_debug("Reiterating files to get inotify watches established.");
+
+        /* Iterate through all dirs again, to add them to the inotify */
+        r = reiterate_all_paths(j);
+         if (r < 0)
                 return r;
         return j->inotify_fd;
@@ -2131,12 +2220,58 @@ _public_ int sd_journal_get_timeout(sd_journal *j, uint64_t *timeout_usec) {
         return 1;
 }
+static void process_q_overflow(sd_journal *j) {
+        JournalFile *f;
+        Directory *m;
+        Iterator i;
+
+        assert(j);
+
+        /* When the inotify queue overruns we need to enumerate and re-validate all journal files to bring our list
+         * back in sync with what's on disk. For this we pick a new generation counter value. It'll be assigned to all
+         * journal files we encounter. All journal files and all directories that don't carry it after reenumeration
+         * are subject for unloading. */
+
+        log_debug("Inotify queue overrun, reiterating everything.");
+
+        j->generation++;
+        (void) reiterate_all_paths(j);
+
+        ORDERED_HASHMAP_FOREACH(f, j->files, i) {
+
+                if (f->last_seen_generation == j->generation)
+                        continue;
+
+                log_debug("File '%s' hasn't been seen in this enumeration, removing.", f->path);
+                remove_file_real(j, f);
+        }
+
+        HASHMAP_FOREACH(m, j->directories_by_path, i) {
+
+                if (m->last_seen_generation == j->generation)
+                        continue;
+
+                if (m->is_root) /* Never GC root directories */
+                        continue;
+
+                log_debug("Directory '%s' hasn't been seen in this enumeration, removing.", f->path);
+                remove_directory(j, m);
+        }
+
+        log_debug("Reiteration complete.");
+}
+
 static void process_inotify_event(sd_journal *j, struct inotify_event *e) {
         Directory *d;
         assert(j);
         assert(e);
+        if (e->mask & IN_Q_OVERFLOW) {
+                process_q_overflow(j);
+                return;
+        }
+
         /* Is this a subdirectory we watch? */
         d = hashmap_get(j->directories_by_wd, INT_TO_PTR(e->wd));
         if (d) {
diff --git a/src/shared/path-util.c b/src/shared/path-util.c
index 1181ffb9d..4d6e5e772 100644
--- a/src/shared/path-util.c
+++ b/src/shared/path-util.c
@@ -738,3 +738,17 @@ char *prefix_root(const char *root, const char *path) {
         strcpy(p, path);
         return n;
 }
+
+int inotify_add_watch_fd(int fd, int what, uint32_t mask) {
+        char path[strlen("/proc/self/fd/") + DECIMAL_STR_MAX(int) + 1];
+        int r;
+
+        /* This is like inotify_add_watch(), except that the file to watch is not referenced by a path, but by an fd */
+        xsprintf(path, "/proc/self/fd/%i", what);
+
+        r = inotify_add_watch(fd, path, mask);
+        if (r < 0)
+                return -errno;
+
+        return r;
+}
diff --git a/src/shared/path-util.h b/src/shared/path-util.h
index 71bb740e9..e14702da8 100644
--- a/src/shared/path-util.h
+++ b/src/shared/path-util.h
@@ -65,6 +65,8 @@ int fsck_exists(const char *fstype);
 char *prefix_root(const char *root, const char *path);
+int inotify_add_watch_fd(int fd, int what, uint32_t mask);
+
 /* Similar to prefix_root(), but returns an alloca() buffer, or
  * possibly a const pointer into the path parameter */
 #define prefix_roota(root, path)                                        \
SOURCES/0515-sd-journal-make-sure-it-s-safe-to-call-sd_journal_pr.patch
New file
@@ -0,0 +1,36 @@
From 3822b85386edacc9aaf07231b057fa847cdbde08 Mon Sep 17 00:00:00 2001
From: Lennart Poettering <lennart@poettering.net>
Date: Fri, 9 Feb 2018 22:38:46 +0100
Subject: [PATCH] sd-journal: make sure it's safe to call sd_journal_process()
 before the first sd_journal_wait()
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
In that case we have no inotify fd yet, and there's nothing to process
hence. Let's make the call a NOP.
(Previously, without this change we'd end up trying to read off inotify
fd -1, which is quite a problem... 😢)
(cherry picked from commit 10c4d6405f74258ea4fac5db4888c1bf49ad5399)
Related: #1540538
---
 src/journal/sd-journal.c | 3 +++
 1 file changed, 3 insertions(+)
diff --git a/src/journal/sd-journal.c b/src/journal/sd-journal.c
index 9186f5188..e1cde6e1c 100644
--- a/src/journal/sd-journal.c
+++ b/src/journal/sd-journal.c
@@ -2329,6 +2329,9 @@ _public_ int sd_journal_process(sd_journal *j) {
         assert_return(j, -EINVAL);
         assert_return(!journal_pid_changed(j), -ECHILD);
+        if (j->inotify_fd < 0) /* We have no inotify fd yet? Then there's noting to process. */
+                return 0;
+
         j->last_process_usec = now(CLOCK_MONOTONIC);
         j->last_invalidate_counter = j->current_invalidate_counter;
SOURCES/0516-sd-journal-when-picking-up-a-new-file-compare-inode-.patch
New file
@@ -0,0 +1,77 @@
From 6fc12b327982851378333d24fc902c3540dadc0e Mon Sep 17 00:00:00 2001
From: Michal Sekletar <msekleta@redhat.com>
Date: Tue, 20 Feb 2018 14:16:15 +0100
Subject: [PATCH] sd-journal: when picking up a new file, compare inode/device
 info with previous open file by same name
Let's make sure we aren't confused if a journal file is replaced by a
different one (for example due to rotation) if we are in a q overflow:
let's compare the inode/device information, and if it changed replace
any open file object as needed.
Fixes: #8198
(cherry-picked from commit 32cb1983ad6f7084ff86e259ff079742a8139719)
[msekleta: this is very slimmed down version of the above commit because
a lot of code from is not applicable to RHEL-7 version]
Related: #1540538
---
 src/journal/sd-journal.c | 35 +++++++++++++++++++++++++++++------
 1 file changed, 29 insertions(+), 6 deletions(-)
diff --git a/src/journal/sd-journal.c b/src/journal/sd-journal.c
index e1cde6e1c..004fe646d 100644
--- a/src/journal/sd-journal.c
+++ b/src/journal/sd-journal.c
@@ -1224,20 +1224,43 @@ static bool file_type_wanted(int flags, const char *filename) {
 static int add_any_file(sd_journal *j, const char *path) {
         JournalFile *f = NULL;
+        struct stat st;
         int r, k;
         assert(j);
         assert(path);
-        if (path) {
-                f = ordered_hashmap_get(j->files, path);
-                if (f) {
-                        /* Mark this file as seen in this generation. This is used to GC old files in
-                         * process_q_overflow() to detect journal files that are still and discern them from those who
-                         * are gone. */
+        if (stat(path, &st) < 0) {
+                r = log_debug_errno(errno, "Failed to stat file '%s': %m", path);
+                return -errno;
+        }
+        if (S_ISDIR(st.st_mode)) {
+                log_debug("Uh, file '%s' is a directory? Refusing.", path);
+                return -EISDIR;
+        }
+        if (!S_ISREG(st.st_mode)) {
+                log_debug("Uh, file '%s' is not a regular file? Refusing.", path);
+                return -EBADFD;
+        }
+
+        f = ordered_hashmap_get(j->files, path);
+        if (f) {
+
+                if (f->last_stat.st_dev == st.st_dev &&
+                    f->last_stat.st_ino == st.st_ino) {
+
+                        /* We already track this file, under the same path and with the same device/inode numbers, it's hence
+                         * really the same. Mark this file as seen in this generation. This is used to GC old files in
+                         * process_q_overflow() to detect journal files that are still and discern them from those who are
+                         * gone. */
                         f->last_seen_generation = j->generation;
                         return 0;
                 }
+
+                /* So we tracked a file under this name, but it has a different inode/device. In that case, it got
+                 * replaced (probably due to rotation?), let's drop it hence from our list. */
+                remove_file_real(j, f);
+                f = NULL;
         }
         if (ordered_hashmap_size(j->files) >= JOURNAL_FILES_MAX) {
SOURCES/0517-journalctl-Periodically-call-sd_journal_process-in-j.patch
New file
@@ -0,0 +1,59 @@
From 6241ea7b14a47a820e5d692a44056fa7181242cf Mon Sep 17 00:00:00 2001
From: Peter Portante <peter.a.portante@gmail.com>
Date: Sun, 28 Jan 2018 16:48:04 -0500
Subject: [PATCH] journalctl: Periodically call sd_journal_process in
 journalctl
If `journalctl` take a long time to process messages, and during that
time journal file rotation occurs, a `journalctl` client will keep
those rotated files open until it calls `sd_journal_process()`, which
typically happens as a result of calling `sd_journal_wait()` below in
the "following" case.  By periodically calling `sd_journal_process()`
during the processing loop we shrink the window of time a client
instance has open file descriptors for rotated (deleted) journal
files.
(Lennart: slightly reworked version, that dropped some of the commenting
which was solved otherwise)
(cherry picked from commit ec316d199a13d8db3f6550d60e369893de2fb417)
Related: #1540538
---
 src/journal/journalctl.c | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)
diff --git a/src/journal/journalctl.c b/src/journal/journalctl.c
index 0be70764e..1e6d0761c 100644
--- a/src/journal/journalctl.c
+++ b/src/journal/journalctl.c
@@ -67,6 +67,8 @@
 #define DEFAULT_FSS_INTERVAL_USEC (15*USEC_PER_MINUTE)
+#define PROCESS_INOTIFY_INTERVAL 1024   /* Every 1,024 messages processed */
+
 enum {
         /* Special values for arg_lines */
         ARG_LINES_DEFAULT = -2,
@@ -2294,6 +2296,20 @@ int main(int argc, char *argv[]) {
                                 goto finish;
                         n_shown++;
+
+                        /* If journalctl take a long time to process messages, and during that time journal file
+                         * rotation occurs, a journalctl client will keep those rotated files open until it calls
+                         * sd_journal_process(), which typically happens as a result of calling sd_journal_wait() below
+                         * in the "following" case.  By periodically calling sd_journal_process() during the processing
+                         * loop we shrink the window of time a client instance has open file descriptors for rotated
+                         * (deleted) journal files. */
+                        if ((n_shown % PROCESS_INOTIFY_INTERVAL) == 0) {
+                                r = sd_journal_process(j);
+                                if (r < 0) {
+                                        log_error_errno(r, "Failed to process inotify events: %m");
+                                        goto finish;
+                                }
+                        }
                 }
                 if (!arg_follow) {
SPECS/systemd.spec
@@ -7,7 +7,7 @@
Name:           systemd
Url:            http://www.freedesktop.org/wiki/Software/systemd
Version:        219
Release:        42%{?dist}.7
Release:        42%{?dist}.10
# For a breakdown of the licensing, see README
License:        LGPLv2+ and MIT and GPLv2+
Summary:        A System and Service Manager
@@ -546,6 +546,16 @@
Patch0505: 0505-shutdown-fix-incorrect-fscanf-result-check-6806.patch
Patch0506: 0506-manager-fix-connecting-to-bus-when-dbus-is-actually-.patch
Patch0507: 0507-automount-ack-automount-requests-even-when-already-m.patch
Patch0508: 0508-shared-cgroup-utils-_CGROUP_CONTROLLER_MASK_ALL-does.patch
Patch0509: 0509-core-execute-fix-fork-fail-handling-in-exec_spawn.patch
Patch0510: 0510-journal-remove-error-check-that-never-happens.patch
Patch0511: 0511-sd-journal-various-clean-ups-and-modernizations.patch
Patch0512: 0512-journalctl-continue-operation-even-if-we-run-into-an.patch
Patch0513: 0513-journalctl-when-we-fail-to-open-a-journal-file-print.patch
Patch0514: 0514-sd-journal-properly-handle-inotify-queue-overflow.patch
Patch0515: 0515-sd-journal-make-sure-it-s-safe-to-call-sd_journal_pr.patch
Patch0516: 0516-sd-journal-when-picking-up-a-new-file-compare-inode-.patch
Patch0517: 0517-journalctl-Periodically-call-sd_journal_process-in-j.patch
%global num_patches %{lua: c=0; for i,p in ipairs(patches) do c=c+1; end; print(c);}
@@ -1514,6 +1524,22 @@
%{_mandir}/man8/systemd-resolved.*
%changelog
* Wed Feb 21 2018 Lukas Nykryn <lnykryn@redhat.com> - 219-42.10
- journal: remove error check that never happens (#1465759)
- sd-journal: various clean-ups and modernizations (#1465759)
- journalctl: continue operation, even if we run into an invalid file (#1465759)
- journalctl: when we fail to open a journal file, print why (#1465759)
- sd-journal: properly handle inotify queue overflow (#1540538)
- sd-journal: make sure it's safe to call sd_journal_process() before the first sd_journal_wait() (#1540538)
- sd-journal: when picking up a new file, compare inode/device info with previous open file by same name (#1540538)
- journalctl: Periodically call sd_journal_process in journalctl (#1540538)
* Mon Feb 19 2018 Lukas Nykryn <lnykryn@redhat.com> - 219-42.9
- core:execute: fix fork() fail handling in exec_spawn() (#1437114)
* Tue Feb 13 2018 Lukas Nykryn <lnykryn@redhat.com> - 219-42.8
- shared/cgroup-utils: _CGROUP_CONTROLLER_MASK_ALL does not cover CGROUP_PIDS (#1532586)
* Wed Jan 17 2018 Lukas Nykryn <lnykryn@redhat.com> - 219-42.7
- automount: ack automount requests even when already mounted (#1535135)