From 212d592a2a56bbba7d22df755d41700439ad4843 Mon Sep 17 00:00:00 2001 From: CentOS Sources Date: Dec 08 2020 10:11:26 +0000 Subject: import rhel-system-roles-1.0-21.el8 --- diff --git a/SOURCES/logging-0001-test-playbooks-enhancement.diff b/SOURCES/logging-0001-test-playbooks-enhancement.diff new file mode 100644 index 0000000..69bf819 --- /dev/null +++ b/SOURCES/logging-0001-test-playbooks-enhancement.diff @@ -0,0 +1,136 @@ +From 90952a1bb7ddbba45ed8cbd62e6a8e0edb6f6148 Mon Sep 17 00:00:00 2001 +From: Noriko Hosoi +Date: Tue, 25 Aug 2020 09:05:03 -0700 +Subject: [PATCH 1/7] Test playbooks enhancement + +In the code to check the log message is successfully logged or not +in the /var/log/messages file, adding "until: __result is success" +and waiting up to 5 seconds. +--- + tests/tests_basics_files.yml | 4 ++++ + tests/tests_basics_files2.yml | 4 ++++ + tests/tests_basics_files_forwards.yml | 4 ++++ + tests/tests_basics_files_log_dir.yml | 4 ++++ + tests/tests_basics_forwards_implicit_files.yml | 4 ++++ + tests/tests_combination.yml | 4 ++++ + tests/tests_combination2.yml | 4 ++++ + tests/tests_imuxsock_files.yml | 4 ++++ + 8 files changed, 32 insertions(+) + +diff --git a/tests/tests_basics_files.yml b/tests/tests_basics_files.yml +index 080890f..87950d8 100644 +--- a/tests/tests_basics_files.yml ++++ b/tests/tests_basics_files.yml +@@ -74,4 +74,8 @@ + + - name: Check the test log message in {{ __default_system_log }} + command: /bin/grep testMessage0000 {{ __default_system_log }} ++ register: __result ++ until: __result is success ++ retries: 5 ++ delay: 1 + changed_when: false +diff --git a/tests/tests_basics_files2.yml b/tests/tests_basics_files2.yml +index ae61be2..094b125 100644 +--- a/tests/tests_basics_files2.yml ++++ b/tests/tests_basics_files2.yml +@@ -99,4 +99,8 @@ + + - name: Check the test log message in {{ __default_system_log }} + command: /bin/grep testMessage0000 "{{ __default_system_log }}" ++ register: __result ++ until: __result is success ++ retries: 5 ++ delay: 1 + changed_when: false +diff --git a/tests/tests_basics_files_forwards.yml b/tests/tests_basics_files_forwards.yml +index f43b8eb..d08a207 100644 +--- a/tests/tests_basics_files_forwards.yml ++++ b/tests/tests_basics_files_forwards.yml +@@ -105,6 +105,10 @@ + + - name: Check the test log message in {{ __default_system_log }} + command: /bin/grep testMessage0000 '{{ __default_system_log }}' ++ register: __result ++ until: __result is success ++ retries: 5 ++ delay: 1 + changed_when: false + + - name: Check if the forwarding config exists +diff --git a/tests/tests_basics_files_log_dir.yml b/tests/tests_basics_files_log_dir.yml +index ca900b8..f5ca266 100644 +--- a/tests/tests_basics_files_log_dir.yml ++++ b/tests/tests_basics_files_log_dir.yml +@@ -78,6 +78,10 @@ + + - name: Check the files output config that the path is {{ logging_system_log_dir }}/messages + command: /bin/grep '\*.info;mail.none;authpriv.none;cron.none.*{{ logging_system_log_dir }}/messages' {{ __test_files_conf }} ++ register: __result ++ until: __result is success ++ retries: 5 ++ delay: 1 + changed_when: false + + - name: Check the test log message in {{ logging_system_log_dir }}/messages +diff --git a/tests/tests_basics_forwards_implicit_files.yml b/tests/tests_basics_forwards_implicit_files.yml +index 6744d53..1d23911 100644 +--- a/tests/tests_basics_forwards_implicit_files.yml ++++ b/tests/tests_basics_forwards_implicit_files.yml +@@ -92,6 +92,10 @@ + + - name: Check if the test message is in {{ __default_system_log }} + command: /bin/grep testMessage0000 '{{ __default_system_log }}' ++ register: __result ++ until: __result is success ++ retries: 5 ++ delay: 1 + changed_when: false + + - name: Get the forwarding config stat +diff --git a/tests/tests_combination.yml b/tests/tests_combination.yml +index 99d57dc..8aae855 100644 +--- a/tests/tests_combination.yml ++++ b/tests/tests_combination.yml +@@ -129,6 +129,10 @@ + + - name: Check the test log message in {{ __default_system_log }} + command: /bin/grep testMessage0000 '{{ __default_system_log }}' ++ register: __result ++ until: __result is success ++ retries: 5 ++ delay: 1 + changed_when: false + + - name: Generated a file to check severity_and_facility +diff --git a/tests/tests_combination2.yml b/tests/tests_combination2.yml +index 5d49a57..5fe43cb 100644 +--- a/tests/tests_combination2.yml ++++ b/tests/tests_combination2.yml +@@ -138,6 +138,10 @@ + + - name: Check the test log message in {{ __default_system_log }} + command: /bin/grep testMessage0000 '{{ __default_system_log }}' ++ register: __result ++ until: __result is success ++ retries: 5 ++ delay: 1 + changed_when: false + + - name: Check the forwarding config stat +diff --git a/tests/tests_imuxsock_files.yml b/tests/tests_imuxsock_files.yml +index 2d6840d..35db253 100644 +--- a/tests/tests_imuxsock_files.yml ++++ b/tests/tests_imuxsock_files.yml +@@ -76,4 +76,8 @@ + + - name: Check the test log message in {{ __default_system_log }} + command: /bin/grep testMessage0000 "{{ __default_system_log }}" ++ register: __result ++ until: __result is success ++ retries: 5 ++ delay: 1 + changed_when: false +-- +2.26.2 + diff --git a/SOURCES/logging-0002-elasticsearch-output-template.diff b/SOURCES/logging-0002-elasticsearch-output-template.diff new file mode 100644 index 0000000..6bb8a3a --- /dev/null +++ b/SOURCES/logging-0002-elasticsearch-output-template.diff @@ -0,0 +1,81 @@ +From e7f255a64a1ffe83b06e93c944c73b8079f1db3a Mon Sep 17 00:00:00 2001 +From: Noriko Hosoi +Date: Thu, 10 Sep 2020 17:15:32 -0700 +Subject: [PATCH 2/7] Fixing a logic bug in elasticsearch output template. + +When evaluated, the retryfailures value was denied by "not", which +should not have been. Removing the "not" and adding a test case to +tests_files_elasticsearch_use_local_cert.yml. + +(cherry picked from commit 108f06926f7bec929fdfc24ce2fbcfe195078ae2) +--- + roles/rsyslog/templates/output_elasticsearch.j2 | 2 +- + .../tests_files_elasticsearch_use_local_cert.yml | 16 +++++++++++++--- + 2 files changed, 14 insertions(+), 4 deletions(-) + +diff --git a/roles/rsyslog/templates/output_elasticsearch.j2 b/roles/rsyslog/templates/output_elasticsearch.j2 +index c3cd1df..c4db10f 100644 +--- a/roles/rsyslog/templates/output_elasticsearch.j2 ++++ b/roles/rsyslog/templates/output_elasticsearch.j2 +@@ -44,7 +44,7 @@ ruleset(name="{{ item.name }}") { + bulkid="{{ item.bulkid | d("id_template") }}" + dynbulkid="{{ item.dynbulkid | d('on') }}" + allowUnsignedCerts="{{ item.allowUnsignedCerts | d("off") }}" +-{% if not item.retryfailures | d(true) %} ++{% if item.retryfailures | d(true) %} + {% if item.retryruleset | d() | length > 0 %} + retryfailures="on" + retryruleset="{{ item.retryruleset }}" +diff --git a/tests/tests_files_elasticsearch_use_local_cert.yml b/tests/tests_files_elasticsearch_use_local_cert.yml +index 2559ce7..8b1eaa4 100644 +--- a/tests/tests_files_elasticsearch_use_local_cert.yml ++++ b/tests/tests_files_elasticsearch_use_local_cert.yml +@@ -44,6 +44,7 @@ + __test_ca_cert: /tmp/es-ca.crt + __test_cert: /tmp/es-cert.pem + __test_key: /tmp/es-key.pem ++ __test_el: elasticsearch_output + + tasks: + - name: Generate fake key/certs files. +@@ -60,13 +61,13 @@ + - name: deploy config to send to elasticsearch + vars: + logging_outputs: +- - name: elasticsearch_output ++ - name: "{{ __test_el }}" + type: elasticsearch + server_host: logging-es + server_port: 9200 + index_prefix: project. + input_type: ovirt +- retryfailures: false ++ retryfailures: on + ca_cert_src: "{{ __test_ca_cert }}" + cert_src: "{{ __test_cert }}" + private_key_src: "{{ __test_key }}" +@@ -77,7 +78,7 @@ + logging_flows: + - name: flow_0 + inputs: [files_input] +- outputs: [elasticsearch_output, elasticsearch_output_ops] ++ outputs: "[{{ __test_el }}]" + include_role: + name: linux-system-roles.logging + +@@ -119,3 +120,12 @@ + - mycert: "{{ __test_cert }}" + - myprivkey: "{{ __test_key }}" + changed_when: false ++ ++ - name: Check retryfailures in {{ __test_outputfiles_conf }} ++ command: /bin/grep 'retryfailures="on"' {{ __test_outputfiles_conf }} ++ changed_when: false ++ ++ - name: Check retryruleset in {{ __test_outputfiles_conf }} ++ command: /bin/grep 'retryruleset="{{ __test_el }}"' {{ __test_outputfiles_conf }} ++ changed_when: false ++ +-- +2.26.2 + diff --git a/SOURCES/logging-0003-README.diff b/SOURCES/logging-0003-README.diff new file mode 100644 index 0000000..8f7fcdd --- /dev/null +++ b/SOURCES/logging-0003-README.diff @@ -0,0 +1,55 @@ +From 76b4418f937fd1dbaa1061fa5f83f11ea046dc40 Mon Sep 17 00:00:00 2001 +From: Noriko Hosoi +Date: Thu, 10 Sep 2020 16:35:43 -0700 +Subject: [PATCH 3/7] Adding "Port and SELinux" section to README. + +(cherry picked from commit 5f144bc74edbcd80a53a2fe84aa464f7ea9f44ef) +--- + README.md | 16 +++++++++++++--- + 1 file changed, 13 insertions(+), 3 deletions(-) + +diff --git a/README.md b/README.md +index 0eafde8..db29dc5 100644 +--- a/README.md ++++ b/README.md +@@ -19,6 +19,7 @@ + * [Standalone configuration](#standalone-configuration) + * [Client configuration](#client-configuration) + * [Server configuration](#server-configuration) ++ * [Port and SELinux](#port-and-selinux) + * [Providers](#providers) + * [Tests](#tests) + * [Implementation Details](#implementation-details) +@@ -111,10 +112,10 @@ This is a schematic logging configuration to show log messages from input_nameA + - `ovirt` type - `ovirt` input supports oVirt specific inputs.
+ For the details, visit [oVirt Support](../../design_docs/rsyslog_ovirt_support.md). + +-- `remote` type - `remote` input supports receiving logs from the remote logging system over the network. This input type makes rsyslog a server.
++- `remote` type - `remote` input supports receiving logs from the remote logging system over the network.
+ **available options** +- - `udp_ports`: List of UDP port numbers to listen. If set, the `remote` input listens on the UDP ports. No defaults. If both `udp_ports` and `tcp_ports` are set in a `remote` input item, `udp_ports` is used and `tcp_ports` is dropped. +- - `tcp_ports`: List of TCP port numbers to listen. If set, the `remote` input listens on the TCP ports. Default to `[514]`. If both `udp_ports` and `tcp_ports` are set in a `remote` input item, `udp_ports` is used and `tcp_ports` is dropped. If both `udp_ports` and `tcp_ports` are not set in a `remote` input item, `tcp_ports: [514]` is added to the item. ++ - `udp_ports`: List of UDP port numbers to listen. If set, the `remote` input listens on the UDP ports. No defaults. If both `udp_ports` and `tcp_ports` are set in a `remote` input item, `udp_ports` is used and `tcp_ports` is dropped. See also [Port and SELinux](#port-and-selinux). ++ - `tcp_ports`: List of TCP port numbers to listen. If set, the `remote` input listens on the TCP ports. Default to `[514]`. If both `udp_ports` and `tcp_ports` are set in a `remote` input item, `udp_ports` is used and `tcp_ports` is dropped. If both `udp_ports` and `tcp_ports` are not set in a `remote` input item, `tcp_ports: [514]` is added to the item. See also [Port and SELinux](#port-and-selinux). + - `tls`: Set to `true` to encrypt the connection using the default TLS implementation used by the provider. Default to `false`. + - `pki_authmode`: Specifying the default network driver authentication mode. `x509/name`, `x509/fingerprint`, `anon` is accepted. Default to `x509/name`. + - `permitted_clients`: List of hostnames, IP addresses, fingerprints(sha1), and wildcard DNS domains which will be allowed by the `logging` server to connect and send logs over TLS. Default to `['*.{{ logging_domain }}']` +@@ -591,6 +592,15 @@ The following playbook generates the same logging configuration files. + outputs: [remote_files_output0, remote_files_output1] + ``` + ++### Port and SELinux ++ ++SELinux is only configured to allow sending and receiving on the following ports by default: ++``` ++syslogd_port_t tcp 514, 20514 ++syslogd_port_t udp 514, 20514 ++``` ++If other ports need to be configured, you can use [linux-system-roles/selinux](https://github.com/linux-system-roles/selinux) to manage SELinux contexts. ++ + ## Providers + + [Rsyslog](roles/rsyslog) - This documentation contains rsyslog specific information. +-- +2.26.2 + diff --git a/SOURCES/logging-0004-yamllint-errors.diff b/SOURCES/logging-0004-yamllint-errors.diff new file mode 100644 index 0000000..8adf0e1 --- /dev/null +++ b/SOURCES/logging-0004-yamllint-errors.diff @@ -0,0 +1,31 @@ +From 6ef1f1020abb074525724e9060ddada526ad0102 Mon Sep 17 00:00:00 2001 +From: Noriko Hosoi +Date: Tue, 29 Sep 2020 15:50:03 -0700 +Subject: [PATCH 4/7] Fixing yamllint errors. + +(cherry picked from commit b131f9e26b3fd74d759b237d7b3b26b6732371d2) +--- + tests/tests_files_elasticsearch_use_local_cert.yml | 3 +-- + 1 file changed, 1 insertion(+), 2 deletions(-) + +diff --git a/tests/tests_files_elasticsearch_use_local_cert.yml b/tests/tests_files_elasticsearch_use_local_cert.yml +index 8b1eaa4..90b12af 100644 +--- a/tests/tests_files_elasticsearch_use_local_cert.yml ++++ b/tests/tests_files_elasticsearch_use_local_cert.yml +@@ -67,7 +67,7 @@ + server_port: 9200 + index_prefix: project. + input_type: ovirt +- retryfailures: on ++ retryfailures: true + ca_cert_src: "{{ __test_ca_cert }}" + cert_src: "{{ __test_cert }}" + private_key_src: "{{ __test_key }}" +@@ -128,4 +128,3 @@ + - name: Check retryruleset in {{ __test_outputfiles_conf }} + command: /bin/grep 'retryruleset="{{ __test_el }}"' {{ __test_outputfiles_conf }} + changed_when: false +- +-- +2.26.2 + diff --git a/SOURCES/logging-0005-property-based-filters.diff b/SOURCES/logging-0005-property-based-filters.diff new file mode 100644 index 0000000..1158774 --- /dev/null +++ b/SOURCES/logging-0005-property-based-filters.diff @@ -0,0 +1,324 @@ +From b72e8a48be07a1cebce8b2237d7344220678c2ec Mon Sep 17 00:00:00 2001 +From: Noriko Hosoi +Date: Fri, 16 Oct 2020 08:15:11 -0700 +Subject: [PATCH 5/7] Logging - support property-based filters in the files and + forwards outputs + +Adding property-based filter options to files, forwards and remote_files output. +A test case is added to tests_basics_files2.yml. + +In addition, fixing a bug caused by a left over file from the previous tests. + +Issue - https://github.com/linux-system-roles/logging/issues/179 + +(cherry picked from commit 6ac8f9ff680a4b0230446062f5927f5921829f80) +--- + README.md | 68 ++++++++++++------- + roles/rsyslog/templates/output_files.j2 | 4 +- + roles/rsyslog/templates/output_forwards.j2 | 4 +- + .../rsyslog/templates/output_remote_files.j2 | 4 +- + tests/tests_basics_files2.yml | 40 +++++++++-- + tests/tests_basics_forwards_cert.yml | 8 +++ + tests/tests_basics_forwards_cert_missing.yml | 4 ++ + tests/tests_server_conflict.yml | 8 +++ + 8 files changed, 108 insertions(+), 32 deletions(-) + +diff --git a/README.md b/README.md +index db29dc5..4352ee7 100644 +--- a/README.md ++++ b/README.md +@@ -180,11 +180,16 @@ This is a schematic logging configuration to show log messages from input_nameA + + - `files` type - `files` output supports storing logs in the local files usually in /var/log.
+ **available options** +- - `facility`: Facility; default to `*`. +- - `severity`: Severity; default to `*`. +- - `exclude`: Exclude list; default to none. ++ - `facility`: Facility in selector; default to `*`. ++ - `severity`: Severity in selector; default to `*`. ++ - `exclude`: Exclude list used in selector; default to none. ++ - `property`: Property in property-based filter; no default ++ - `prop_op`: Operation in property-based filter; In case of not `!`, put the `prop_op` value in quotes; default to `contains` ++ - `prop_value`: Value in property-based filter; default to `error` + - `path`: Path to the output file. + ++ Selector options and property-based filter options are exclusive. If Property-based filter options are defined, selector options will be ignored. ++ + Unless the above options are given, these local file outputs are configured. + ``` + kern.* /dev/console +@@ -199,8 +204,12 @@ This is a schematic logging configuration to show log messages from input_nameA + + - `forwards` type - `forwards` output sends logs to the remote logging system over the network. This is for the client rsyslog.
+ **available options** +- - `facility`: Facility; default to `*`. +- - `severity`: Severity; default to `*`. ++ - `facility`: Facility in selector; default to `*`. ++ - `severity`: Severity in selector; default to `*`. ++ - `exclude`: Exclude list used in selector; default to none. ++ - `property`: Property in property-based filter; no default ++ - `prop_op`: Operation in property-based filter; In case of not `!`, put the `prop_op` value in quotes; default to `contains` ++ - `prop_value`: Value in property-based filter; default to `error` + - `target`: Target host (fqdn). **Required**. + - `udp_port`: UDP port number. Default to `514`. + - `tcp_port`: TCP port number. Default to `514`. +@@ -208,11 +217,16 @@ This is a schematic logging configuration to show log messages from input_nameA + - `pki_authmode`: Specifying the default network driver authentication mode. `x509/name`, `x509/fingerprint`, `anon` is accepted. Default to `x509/name`. + - `permitted_server`: Hostname, IP address, fingerprint(sha1) or wildcard DNS domain of the server which this client will be allowed to connect and send logs over TLS. Default to `*.{{ logging_domain }}` + ++ Selector options and property-based filter options are exclusive. If Property-based filter options are defined, selector options will be ignored. ++ + - `remote_files` type - `remote_files` output stores logs to the local files per remote host and program name originated the logs.
+ **available options** +- - `facility`: Facility; default to `*`. +- - `severity`: Severity; default to `*`. +- - `exclude`: Exclude list; default to none. ++ - `facility`: Facility in selector; default to `*`. ++ - `severity`: Severity in selector; default to `*`. ++ - `exclude`: Exclude list used in selector; default to none. ++ - `property`: Property in property-based filter; no default ++ - `prop_op`: Operation in property-based filter; In case of not `!`, put the `prop_op` value in quotes; default to `contains` ++ - `prop_value`: Value in property-based filter; default to `error` + - `async_writing`: If set to `true`, the files are written asynchronously. Allowed value is `true` or `false`. Default to `false`. + - `client_count`: Count of client logging system supported this rsyslog server. Default to `10`. + - `io_buffer_size`: Buffer size used to write output data. Default to `65536` bytes. +@@ -221,6 +235,8 @@ This is a schematic logging configuration to show log messages from input_nameA + `/path/to/output/dir/%HOSTNAME%/%PROGRAMNAME:::secpath-replace%.log` + - `remote_sub_path`: Relative path to logging_system_log_dir to store the filtered logs. + ++ Selector options and property-based filter options are exclusive. If Property-based filter options are defined, selector options will be ignored. ++ + if both `remote_log_path` and `remote_sub_path` are _not_ specified, the remote_file output configured with the following settings. + ``` + template( +@@ -446,32 +462,38 @@ The following playbook generates the same logging configuration files. + outputs: [files_output0, files_output1] + ``` + +-5. Deploying `files input` reading logs from a local file and `elasticsearch output` to store the logs. Assuming the ca_cert, cert and key to connect to Elasticsearch are prepared. ++5. Deploying `files input` reading logs from local files and `files output` to write to the local files based on the property-based filters. + ```yaml + --- +-- name: Deploying basic input and elasticsearch output ++- name: Deploying files input and configured files output + hosts: all + roles: + - linux-system-roles.logging + vars: + logging_inputs: +- - name: files_input ++ - name: files_input0 + type: files +- input_log_path: /var/log/containers/*.log ++ input_log_path: /var/log/containerA/*.log ++ - name: files_input1 ++ type: files ++ input_log_path: /var/log/containerB/*.log + logging_outputs: +- - name: elasticsearch_output +- type: elasticsearch +- server_host: your_target_host +- server_port: 9200 +- index_prefix: project. +- input_type: ovirt +- ca_cert_src: /local/path/to/ca_cert +- cert_src: /local/path/to/cert +- private_key_src: /local/path/to/key ++ - name: files_output0 ++ type: files ++ property: msg ++ prop_op: contains ++ prop_value: error ++ path: /var/log/errors.log ++ - name: files_output1 ++ type: files ++ property: msg ++ prop_op: "!contains" ++ prop_value: error ++ path: /var/log/others.log + logging_flows: + - name: flow0 +- inputs: [files_input] +- outputs: [elasticsearch_output] ++ inputs: [files_input0, files_input1] ++ outputs: [files_output0, files_output1] + ``` + + ### Client configuration +diff --git a/roles/rsyslog/templates/output_files.j2 b/roles/rsyslog/templates/output_files.j2 +index d994414..e15e4cd 100644 +--- a/roles/rsyslog/templates/output_files.j2 ++++ b/roles/rsyslog/templates/output_files.j2 +@@ -1,6 +1,8 @@ + {% if item.path is defined %} + ruleset(name="{{ item.name }}") { +-{% if item.exclude | d([]) %} ++{% if item.property | d() %} ++ :{{ item.property }}, {{ item.prop_op | d('contains') }}, "{{ item.prop_value | d('error') }}" {{ item.path }} ++{% elif item.exclude | d([]) %} + {{ item.facility | d('*') }}.{{ item.severity | d('*') }};{{ item.exclude | join(';') }} {{ item.path }} + {% else %} + {{ item.facility | d('*') }}.{{ item.severity | d('*') }} {{ item.path }} +diff --git a/roles/rsyslog/templates/output_forwards.j2 b/roles/rsyslog/templates/output_forwards.j2 +index 61254ee..35030b4 100644 +--- a/roles/rsyslog/templates/output_forwards.j2 ++++ b/roles/rsyslog/templates/output_forwards.j2 +@@ -9,7 +9,9 @@ + {% set __forwards_protocol = '' %} + {% endif %} + ruleset(name="{{ item.name }}") { +-{% if item.exclude | d([]) %} ++{% if item.property | d() %} ++ :{{ item.property }}, {{ item.prop_op | d('contains') }}, "{{ item.prop_value | d('error') }}" action(name="{{ item.name }}" ++{% elif item.exclude | d([]) %} + {{ item.facility | d('*') }}.{{ item.severity | d('*') }};{{ item.exclude | join(';') }} action(name="{{ item.name }}" + {% else %} + {{ item.facility | d('*') }}.{{ item.severity | d('*') }} action(name="{{ item.name }}" +diff --git a/roles/rsyslog/templates/output_remote_files.j2 b/roles/rsyslog/templates/output_remote_files.j2 +index 3c9339f..aaf547e 100644 +--- a/roles/rsyslog/templates/output_remote_files.j2 ++++ b/roles/rsyslog/templates/output_remote_files.j2 +@@ -17,7 +17,9 @@ ruleset(name="{{ item.name }}" + queue.size="{{ logging_server_queue_size }}" + queue.workerThreads="{{ logging_server_threads }}") { + # Store remote logs in separate logfiles +-{% if item.exclude | d([]) %} ++{% if item.property | d() %} ++ :{{ item.property }}, {{ item.prop_op | d('contains') }}, "{{ item.prop_value | d('error') }}" action(name="{{ item.name }}" type="omfile" DynaFile="{{ item.name }}_template" DynaFileCacheSize="{{ item.client_count | d(10) }}" ioBufferSize="{{ item.io_buffer_size | d('65536') }}" asyncWriting="{{ 'on' if item.async_writing | d(false) | bool else 'off' }}") ++{% elif item.exclude | d([]) %} + {{ item.facility | d('*') }}.{{ item.severity | d('*') }};{{ item.exclude | join(';') }} action(name="{{ item.name }}" type="omfile" DynaFile="{{ item.name }}_template" DynaFileCacheSize="{{ item.client_count | d(10) }}" ioBufferSize="{{ item.io_buffer_size | d('65536') }}" asyncWriting="{{ 'on' if item.async_writing | d(false) | bool else 'off' }}") + {% else %} + {{ item.facility | d('*') }}.{{ item.severity | d('*') }} action(name="{{ item.name }}" type="omfile" DynaFile="{{ item.name }}_template" DynaFileCacheSize="{{ item.client_count | d(10) }}" ioBufferSize="{{ item.io_buffer_size | d('65536') }}" asyncWriting="{{ 'on' if item.async_writing | d(false) | bool else 'off' }}") +diff --git a/tests/tests_basics_files2.yml b/tests/tests_basics_files2.yml +index 094b125..b1a0f62 100644 +--- a/tests/tests_basics_files2.yml ++++ b/tests/tests_basics_files2.yml +@@ -10,9 +10,9 @@ + # If logging role is executed, the file size is about 100 bytes. + # Thus, assert the size is less than 1000. + # 2. Check file count in /etc/rsyslog.d. +-# If logging role is executed, 8 config files are generated. ++# If logging role is executed, 9 config files are generated. + # By setting logging_purge_confs, pre-existing config files are deleted. +-# Thus, assert the the count is equal to 8. ++# Thus, assert the the count is equal to 9. + # 3. Check systemctl status of rsyslog as well as error or specific message in the output. + # 4. To verify the generated filename is correct, check the config file of files output exists. + # 4.1 Check the config file contains the expected filter and the output file as configured. +@@ -24,6 +24,8 @@ + vars: + __test_files_conf: /etc/rsyslog.d/30-output-files-files_output1.conf + __default_system_log: /var/log/messages ++ __prop_based_log0: /var/log/property_based_filter_in.log ++ __prop_based_log1: /var/log/property_based_filter_out.log + + tasks: + - name: deploy config to output into local files +@@ -49,15 +51,23 @@ + path: :omusrmsg:* + - name: files_output3 + type: files +- facility: local7 +- path: /var/log/boot.log ++ property: msg ++ prop_op: contains ++ prop_value: property_based_filter_test ++ path: "{{ __prop_based_log0 }}" ++ - name: files_output4 ++ type: files ++ property: msg ++ prop_op: "!contains" ++ prop_value: property_based_filter_test ++ path: "{{ __prop_based_log1 }}" + logging_inputs: + - name: basic_input + type: basics + logging_flows: + - name: flow_0 + inputs: [basic_input] +- outputs: [files_output0, files_output1, files_output2, files_output3] ++ outputs: [files_output0, files_output1, files_output2, files_output3, files_output4] + include_role: + name: linux-system-roles.logging + +@@ -74,7 +84,7 @@ + + - name: Check file counts in rsyslog.d + assert: +- that: rsyslog_d_file_count.matched == 8 ++ that: rsyslog_d_file_count.matched == 9 + + # Checking 'error' in stdout from systemctl status is for detecting the case in which rsyslog is running, + # but some functionality is disabled due to some error, e.g., error: 'tls.cacert' file couldn't be accessed. +@@ -104,3 +114,21 @@ + retries: 5 + delay: 1 + changed_when: false ++ ++ - name: Run logger to generate a test log message containing property_based_filter_test ++ command: /bin/logger -i -p local6.info -t testTag1 property_based_filter_test ++ changed_when: false ++ ++ - name: Check the test log message in {{ __prop_based_log0 }} ++ command: /bin/grep property_based_filter_test "{{ __prop_based_log0 }}" ++ register: __result ++ until: __result is success ++ retries: 5 ++ delay: 1 ++ changed_when: false ++ ++ - name: Check the test log message not in {{ __prop_based_log1 }} ++ command: /bin/grep property_based_filter_test "{{ __prop_based_log1 }}" ++ register: __result ++ changed_when: false ++ failed_when: "__result is not failed" +diff --git a/tests/tests_basics_forwards_cert.yml b/tests/tests_basics_forwards_cert.yml +index e27e016..48263ae 100644 +--- a/tests/tests_basics_forwards_cert.yml ++++ b/tests/tests_basics_forwards_cert.yml +@@ -139,3 +139,11 @@ + - /etc/pki/tls/certs/{{ __test_ca_cert_name }} + - /etc/pki/tls/certs/{{ __test_cert_name }} + - /etc/pki/tls/private/{{ __test_key_name }} ++ ++ - name: clean up test files ++ file: path="{{ item }}" state=absent ++ loop: ++ - "{{ __test_ca_cert }}" ++ - "{{ __test_cert }}" ++ - "{{ __test_key }}" ++ delegate_to: localhost +diff --git a/tests/tests_basics_forwards_cert_missing.yml b/tests/tests_basics_forwards_cert_missing.yml +index 3e82856..0ad0569 100644 +--- a/tests/tests_basics_forwards_cert_missing.yml ++++ b/tests/tests_basics_forwards_cert_missing.yml +@@ -63,6 +63,10 @@ + assert: + that: "'{{ ansible_failed_result.results.0.msg }}' is match('{{ __expected_error }}')" + ++ - name: clean up test files ++ file: path="{{ __test_key }}" state=absent ++ delegate_to: localhost ++ + - name: default run for cleanup + vars: + logging_inputs: +diff --git a/tests/tests_server_conflict.yml b/tests/tests_server_conflict.yml +index 36eeeb7..8c182f6 100644 +--- a/tests/tests_server_conflict.yml ++++ b/tests/tests_server_conflict.yml +@@ -76,3 +76,11 @@ + - assert: + that: item.msg is not defined or item.msg is defined and item.msg == __expected_error + loop: "{{ ansible_failed_result.results }}" ++ ++ - name: clean up test files ++ file: path="{{ item }}" state=absent ++ loop: ++ - "{{ __test_ca_cert }}" ++ - "{{ __test_cert }}" ++ - "{{ __test_key }}" ++ delegate_to: localhost +-- +2.26.2 + diff --git a/SOURCES/logging-0006-property_op.diff b/SOURCES/logging-0006-property_op.diff new file mode 100644 index 0000000..1f1ed57 --- /dev/null +++ b/SOURCES/logging-0006-property_op.diff @@ -0,0 +1,136 @@ +From ca2baffbfc14fba077c7c70d849c02b9c69c9e1f Mon Sep 17 00:00:00 2001 +From: Noriko Hosoi +Date: Fri, 16 Oct 2020 11:08:00 -0700 +Subject: [PATCH 6/7] Replacing prop_op with property_op and prop_value with + property_value. + +(cherry picked from commit 1c951e6acef886548029151dbca9d002f20ef425) +--- + README.md | 20 +++++++++---------- + roles/rsyslog/templates/output_files.j2 | 2 +- + roles/rsyslog/templates/output_forwards.j2 | 2 +- + .../rsyslog/templates/output_remote_files.j2 | 2 +- + tests/tests_basics_files2.yml | 8 ++++---- + 5 files changed, 17 insertions(+), 17 deletions(-) + +diff --git a/README.md b/README.md +index 4352ee7..d94ec04 100644 +--- a/README.md ++++ b/README.md +@@ -184,8 +184,8 @@ This is a schematic logging configuration to show log messages from input_nameA + - `severity`: Severity in selector; default to `*`. + - `exclude`: Exclude list used in selector; default to none. + - `property`: Property in property-based filter; no default +- - `prop_op`: Operation in property-based filter; In case of not `!`, put the `prop_op` value in quotes; default to `contains` +- - `prop_value`: Value in property-based filter; default to `error` ++ - `property_op`: Operation in property-based filter; In case of not `!`, put the `property_op` value in quotes; default to `contains` ++ - `property_value`: Value in property-based filter; default to `error` + - `path`: Path to the output file. + + Selector options and property-based filter options are exclusive. If Property-based filter options are defined, selector options will be ignored. +@@ -208,8 +208,8 @@ This is a schematic logging configuration to show log messages from input_nameA + - `severity`: Severity in selector; default to `*`. + - `exclude`: Exclude list used in selector; default to none. + - `property`: Property in property-based filter; no default +- - `prop_op`: Operation in property-based filter; In case of not `!`, put the `prop_op` value in quotes; default to `contains` +- - `prop_value`: Value in property-based filter; default to `error` ++ - `property_op`: Operation in property-based filter; In case of not `!`, put the `property_op` value in quotes; default to `contains` ++ - `property_value`: Value in property-based filter; default to `error` + - `target`: Target host (fqdn). **Required**. + - `udp_port`: UDP port number. Default to `514`. + - `tcp_port`: TCP port number. Default to `514`. +@@ -225,8 +225,8 @@ This is a schematic logging configuration to show log messages from input_nameA + - `severity`: Severity in selector; default to `*`. + - `exclude`: Exclude list used in selector; default to none. + - `property`: Property in property-based filter; no default +- - `prop_op`: Operation in property-based filter; In case of not `!`, put the `prop_op` value in quotes; default to `contains` +- - `prop_value`: Value in property-based filter; default to `error` ++ - `property_op`: Operation in property-based filter; In case of not `!`, put the `property_op` value in quotes; default to `contains` ++ - `property_value`: Value in property-based filter; default to `error` + - `async_writing`: If set to `true`, the files are written asynchronously. Allowed value is `true` or `false`. Default to `false`. + - `client_count`: Count of client logging system supported this rsyslog server. Default to `10`. + - `io_buffer_size`: Buffer size used to write output data. Default to `65536` bytes. +@@ -481,14 +481,14 @@ The following playbook generates the same logging configuration files. + - name: files_output0 + type: files + property: msg +- prop_op: contains +- prop_value: error ++ property_op: contains ++ property_value: error + path: /var/log/errors.log + - name: files_output1 + type: files + property: msg +- prop_op: "!contains" +- prop_value: error ++ property_op: "!contains" ++ property_value: error + path: /var/log/others.log + logging_flows: + - name: flow0 +diff --git a/roles/rsyslog/templates/output_files.j2 b/roles/rsyslog/templates/output_files.j2 +index e15e4cd..40f5b90 100644 +--- a/roles/rsyslog/templates/output_files.j2 ++++ b/roles/rsyslog/templates/output_files.j2 +@@ -1,7 +1,7 @@ + {% if item.path is defined %} + ruleset(name="{{ item.name }}") { + {% if item.property | d() %} +- :{{ item.property }}, {{ item.prop_op | d('contains') }}, "{{ item.prop_value | d('error') }}" {{ item.path }} ++ :{{ item.property }}, {{ item.property_op | d('contains') }}, "{{ item.property_value | d('error') }}" {{ item.path }} + {% elif item.exclude | d([]) %} + {{ item.facility | d('*') }}.{{ item.severity | d('*') }};{{ item.exclude | join(';') }} {{ item.path }} + {% else %} +diff --git a/roles/rsyslog/templates/output_forwards.j2 b/roles/rsyslog/templates/output_forwards.j2 +index 35030b4..87d7a09 100644 +--- a/roles/rsyslog/templates/output_forwards.j2 ++++ b/roles/rsyslog/templates/output_forwards.j2 +@@ -10,7 +10,7 @@ + {% endif %} + ruleset(name="{{ item.name }}") { + {% if item.property | d() %} +- :{{ item.property }}, {{ item.prop_op | d('contains') }}, "{{ item.prop_value | d('error') }}" action(name="{{ item.name }}" ++ :{{ item.property }}, {{ item.property_op | d('contains') }}, "{{ item.property_value | d('error') }}" action(name="{{ item.name }}" + {% elif item.exclude | d([]) %} + {{ item.facility | d('*') }}.{{ item.severity | d('*') }};{{ item.exclude | join(';') }} action(name="{{ item.name }}" + {% else %} +diff --git a/roles/rsyslog/templates/output_remote_files.j2 b/roles/rsyslog/templates/output_remote_files.j2 +index aaf547e..84317f2 100644 +--- a/roles/rsyslog/templates/output_remote_files.j2 ++++ b/roles/rsyslog/templates/output_remote_files.j2 +@@ -18,7 +18,7 @@ ruleset(name="{{ item.name }}" + queue.workerThreads="{{ logging_server_threads }}") { + # Store remote logs in separate logfiles + {% if item.property | d() %} +- :{{ item.property }}, {{ item.prop_op | d('contains') }}, "{{ item.prop_value | d('error') }}" action(name="{{ item.name }}" type="omfile" DynaFile="{{ item.name }}_template" DynaFileCacheSize="{{ item.client_count | d(10) }}" ioBufferSize="{{ item.io_buffer_size | d('65536') }}" asyncWriting="{{ 'on' if item.async_writing | d(false) | bool else 'off' }}") ++ :{{ item.property }}, {{ item.property_op | d('contains') }}, "{{ item.property_value | d('error') }}" action(name="{{ item.name }}" type="omfile" DynaFile="{{ item.name }}_template" DynaFileCacheSize="{{ item.client_count | d(10) }}" ioBufferSize="{{ item.io_buffer_size | d('65536') }}" asyncWriting="{{ 'on' if item.async_writing | d(false) | bool else 'off' }}") + {% elif item.exclude | d([]) %} + {{ item.facility | d('*') }}.{{ item.severity | d('*') }};{{ item.exclude | join(';') }} action(name="{{ item.name }}" type="omfile" DynaFile="{{ item.name }}_template" DynaFileCacheSize="{{ item.client_count | d(10) }}" ioBufferSize="{{ item.io_buffer_size | d('65536') }}" asyncWriting="{{ 'on' if item.async_writing | d(false) | bool else 'off' }}") + {% else %} +diff --git a/tests/tests_basics_files2.yml b/tests/tests_basics_files2.yml +index b1a0f62..9f69ed5 100644 +--- a/tests/tests_basics_files2.yml ++++ b/tests/tests_basics_files2.yml +@@ -52,14 +52,14 @@ + - name: files_output3 + type: files + property: msg +- prop_op: contains +- prop_value: property_based_filter_test ++ property_op: contains ++ property_value: property_based_filter_test + path: "{{ __prop_based_log0 }}" + - name: files_output4 + type: files + property: msg +- prop_op: "!contains" +- prop_value: property_based_filter_test ++ property_op: "!contains" ++ property_value: property_based_filter_test + path: "{{ __prop_based_log1 }}" + logging_inputs: + - name: basic_input +-- +2.26.2 + diff --git a/SOURCES/logging-0007-RHELPLAN-56807.diff b/SOURCES/logging-0007-RHELPLAN-56807.diff new file mode 100644 index 0000000..e3db3f5 --- /dev/null +++ b/SOURCES/logging-0007-RHELPLAN-56807.diff @@ -0,0 +1,114 @@ +From 3967a2b0e7e61dfb6317296a4cf15d0fe91a1638 Mon Sep 17 00:00:00 2001 +From: Noriko Hosoi +Date: Thu, 15 Oct 2020 10:52:29 -0700 +Subject: [PATCH 7/7] RHELPLAN-56807 - Logging - elasticsearch - need to adjust + jinja2 boolean values to the rsyslog config values + +Resetting the values of the following params as rsyslog expects. + dynSearchIndex, bulkmode, dynbulkid, allowUnsignedCerts, usehttps + +Adding test cases to tests_ovirt_elasticsearch_params.yml + +(cherry picked from commit c98aabd864f6d07c11d6db991bf0af0aaee7f123) +--- + .../rsyslog/templates/output_elasticsearch.j2 | 13 ++++----- + tests/tests_ovirt_elasticsearch_params.yml | 29 +++++++++++++++++-- + 2 files changed, 33 insertions(+), 9 deletions(-) + +diff --git a/roles/rsyslog/templates/output_elasticsearch.j2 b/roles/rsyslog/templates/output_elasticsearch.j2 +index c4db10f..6c6255b 100644 +--- a/roles/rsyslog/templates/output_elasticsearch.j2 ++++ b/roles/rsyslog/templates/output_elasticsearch.j2 +@@ -37,25 +37,24 @@ ruleset(name="{{ item.name }}") { + serverport="{{ item.server_port | d(9200) | int }}" + template="{{ item.template | d("es_template") }}" + searchIndex="{{ item.searchIndex | d("index_template") }}" +- dynSearchIndex="{{ item.dynSearchIndex | d("on") }}" ++ dynSearchIndex="{{ item.dynSearchIndex | d(true) | ternary('on', 'off') }}" + searchType="{{ item.searchType | d("com.redhat.viaq.common") }}" +- bulkmode="{{ item.bulkmode | d("on") }}" ++ bulkmode="{{ item.bulkmode | d(true) | ternary('on', 'off') }}" + writeoperation="{{ item.writeoperation | d("create") }}" + bulkid="{{ item.bulkid | d("id_template") }}" +- dynbulkid="{{ item.dynbulkid | d('on') }}" +- allowUnsignedCerts="{{ item.allowUnsignedCerts | d("off") }}" ++ dynbulkid="{{ item.dynbulkid | d(true) | ternary('on', 'off') }}" ++ allowUnsignedCerts="{{ item.allowUnsignedCerts | d(false) | ternary('on', 'off') }}" + {% if item.retryfailures | d(true) %} +-{% if item.retryruleset | d() | length > 0 %} + retryfailures="on" ++{% if item.retryruleset | d() | length > 0 %} + retryruleset="{{ item.retryruleset }}" + {% else %} +- retryfailures="on" + retryruleset="{{ item.name }}" + {% endif %} + {% else %} + retryfailures="off" + {% endif %} +- usehttps="{{ item.usehttps | default("on") }}" ++ usehttps="{{ item.usehttps | d(true) | ternary('on', 'off') }}" + {% if item.use_cert | default(true) %} + tls.cacert="{{ item.ca_cert | default('/etc/rsyslog.d/es-ca.crt') }}" + tls.mycert="{{ item.cert | default('/etc/rsyslog.d/es-cert.pem') }}" +diff --git a/tests/tests_ovirt_elasticsearch_params.yml b/tests/tests_ovirt_elasticsearch_params.yml +index 34d9e1d..4fefe59 100644 +--- a/tests/tests_ovirt_elasticsearch_params.yml ++++ b/tests/tests_ovirt_elasticsearch_params.yml +@@ -34,6 +34,8 @@ + __test_ovirt_engine_conf: /etc/rsyslog.d/90-input-ovirt-ovirt_engine_input.conf + __test_ovirt_vdsm_conf: /etc/rsyslog.d/90-input-ovirt-ovirt_vdsm_input.conf + __test_ovirt_bogus_conf: /etc/rsyslog.d/90-input-ovirt-ovirt_bogus_input.conf ++ __test_es_conf: /etc/rsyslog.d/31-output-elasticsearch-elasticsearch_output.conf ++ __test_es_ops_conf: /etc/rsyslog.d/31-output-elasticsearch-elasticsearch_output_ops.conf + __test_collectd_name: ovirt_collectd_input + __test_engine_name: ovirt_engine_input + __test_vdsm_name: ovirt_vdsm_input +@@ -56,7 +58,6 @@ + server_port: 9200 + index_prefix: project. + input_type: ovirt +- retryfailures: false + ca_cert: "/etc/rsyslog.d/es-ca.crt" + cert: "/etc/rsyslog.d/es-cert.pem" + private_key: "/etc/rsyslog.d/es-key.pem" +@@ -70,6 +71,11 @@ + ca_cert: "/etc/rsyslog.d/es-ca.crt" + cert: "/etc/rsyslog.d/es-cert.pem" + private_key: "/etc/rsyslog.d/es-key.pem" ++ dynSearchIndex: false ++ bulkmode: false ++ dynbulkid: false ++ allowUnsignedCerts: true ++ usehttps: false + logging_inputs: + - name: basic_input + type: basics +@@ -164,4 +170,23 @@ + + - name: Check index_prefix is "{{ __test_logs_index }}" in "{{ __test_ovirt_vdsm_conf }}" + command: /bin/grep 'set $.index_prefix = "{{ __test_logs_index }}"' {{ __test_ovirt_vdsm_conf }} +- changed_when: false ++ ++ - name: Check default config params in "{{ __test_es_conf }}" ++ command: /bin/grep {{ item }} {{ __test_es_conf }} ++ loop: ++ - "dynSearchIndex=.on." ++ - "bulkmode=.on." ++ - "dynbulkid=.on." ++ - "allowUnsignedCerts=.off." ++ - "usehttps=.on." ++ - "retryfailures=.on." ++ ++ - name: Check modified config params in "{{ __test_es_ops_conf }}" ++ command: /bin/grep {{ item }} {{ __test_es_ops_conf }} ++ loop: ++ - "dynSearchIndex=.off." ++ - "bulkmode=.off." ++ - "dynbulkid=.off." ++ - "allowUnsignedCerts=.on." ++ - "usehttps=.off." ++ - "retryfailures=.off." +-- +2.26.2 + diff --git a/SPECS/rhel-system-roles.spec b/SPECS/rhel-system-roles.spec index 6ff2510..8eca7b3 100644 --- a/SPECS/rhel-system-roles.spec +++ b/SPECS/rhel-system-roles.spec @@ -5,7 +5,7 @@ Name: linux-system-roles %endif Summary: Set of interfaces for unified system management Version: 1.0 -Release: 20%{?dist} +Release: 21%{?dist} #Group: Development/Libraries License: GPLv3+ and MIT and BSD @@ -119,6 +119,15 @@ Patch53: network-tier1-tags.diff Patch61: storage-safemode-luks.diff +Patch1001: logging-0001-test-playbooks-enhancement.diff +Patch1002: logging-0002-elasticsearch-output-template.diff +Patch1003: logging-0003-README.diff +Patch1004: logging-0004-yamllint-errors.diff +Patch1005: logging-0005-property-based-filters.diff +Patch1006: logging-0006-property_op.diff +Patch1007: logging-0007-RHELPLAN-56807.diff + + Url: https://github.com/linux-system-roles/ BuildArch: noarch @@ -183,6 +192,15 @@ cd %{rolename6}-%{id6} %endif %patch61 -p1 cd .. +cd %{rolename10}-%{id10} +%patch1001 -p1 +%patch1002 -p1 +%patch1003 -p1 +%patch1004 -p1 +%patch1005 -p1 +%patch1006 -p1 +%patch1007 -p1 +cd .. # for some roles, the prefix change can be scripted - see below %if "%{roleprefix}" != "linux-system-roles." @@ -458,6 +476,14 @@ cp -p $RPM_BUILD_ROOT%{_datadir}/ansible/roles/%{roleprefix}certificate/README.m %license %{_datadir}/ansible/roles/%{roleprefix}certificate/LICENSE %changelog +* Tue Nov 24 2020 Noriko Hosoi - 1.0-21 +- logging: Support oVirt input + elasticsearch output. + Resolves: rhbz#1889893 +- logging: Fixing a logic bug in elasticsearch output template. + Resolves: rhbz#1878857 +- logging: Support property-based filters in the files and forwards outputs. + Resolves: rhbz#1889492 + * Tue Sep 22 2020 Pavel Cahyna - 1.0-20 - storage: backport upstream PR #168 to prevent toggling encryption in safe mode, as it is a destructive operation. Resolves rhbz#1881524