From 1036c6b55b95a27be57b065a4b9acfecc83639b3 Mon Sep 17 00:00:00 2001
From: Alexander Scheel <alex.scheel@canonical.com>
Date: Tue, 6 Jul 2021 16:02:03 -0400
Subject: [PATCH 01/19] Add tests for package_installed template
Signed-off-by: Alexander Scheel <alex.scheel@canonical.com>
---
.../package_installed/tests/package-installed-removed.fail.sh | 4 ++++
.../package_installed/tests/package-installed.pass.sh | 3 +++
.../templates/package_installed/tests/package-removed.fail.sh | 3 +++
3 files changed, 10 insertions(+)
create mode 100644 shared/templates/package_installed/tests/package-installed-removed.fail.sh
create mode 100644 shared/templates/package_installed/tests/package-installed.pass.sh
create mode 100644 shared/templates/package_installed/tests/package-removed.fail.sh
diff --git a/shared/templates/package_installed/tests/package-installed-removed.fail.sh b/shared/templates/package_installed/tests/package-installed-removed.fail.sh
new file mode 100644
index 00000000000..1ce59225303
--- /dev/null
+++ b/shared/templates/package_installed/tests/package-installed-removed.fail.sh
@@ -0,0 +1,4 @@
+#!/bin/bash
+
+{{{ bash_package_install(PKGNAME) }}}
+{{{ bash_package_remove(PKGNAME) }}}
diff --git a/shared/templates/package_installed/tests/package-installed.pass.sh b/shared/templates/package_installed/tests/package-installed.pass.sh
new file mode 100644
index 00000000000..2a0506c57fd
--- /dev/null
+++ b/shared/templates/package_installed/tests/package-installed.pass.sh
@@ -0,0 +1,3 @@
+#!/bin/bash
+
+{{{ bash_package_install(PKGNAME) }}}
diff --git a/shared/templates/package_installed/tests/package-removed.fail.sh b/shared/templates/package_installed/tests/package-removed.fail.sh
new file mode 100644
index 00000000000..c2838396f56
--- /dev/null
+++ b/shared/templates/package_installed/tests/package-removed.fail.sh
@@ -0,0 +1,3 @@
+#!/bin/bash
+
+{{{ bash_package_remove(PKGNAME) }}}
From 3008339e7779b144a2b7fa72854d1185e33438f5 Mon Sep 17 00:00:00 2001
From: Alexander Scheel <alex.scheel@canonical.com>
Date: Tue, 6 Jul 2021 16:16:56 -0400
Subject: [PATCH 02/19] Refactor ssg/templates.py generation (SCE)
This refactors SSG's template generation code to better align between
where we are now upstream and future changes related to templated SCE
support.
At this point in time, we wish to make some changes to allow building
templated (both in the Jinja and shared/templates senses) test cases,
and many of the SCE-enablement changes are relevant, I chose to pull
them in almost entirely. The original commit message is preserved below:
Refactor template generation for SCEs
ssg/templates.py holds the core of template generation logic. We
will need a template_builder for SCE (so we can load SCE metadata
from templated content) but we do not wish to write this SCE content
out immediately; instead, we wish to wait until
build-scripts/build_templated_content.py is run.
We refactor to make the resolved languages for a rule (the
intersection between theoretical languages allowed by the rule and
the actual languages allowed by the template) accessible to callers,
and, also allow reading just a single templated language artifact
into memory (instead of from disk).
Notably, this last change is most useful to us; we don't want to put the
file in a template/build-system specified location; we wish to control
its location exactly here.
Note that no changes to ssg/templates.py for the test suite change were
done here; this was purely a change to sync future changes for SCE
content.
Signed-off-by: Alexander Scheel <alex.scheel@canonical.com>
---
ssg/templates.py | 112 +++++++++++++++++++++++++++++++++++++----------
1 file changed, 89 insertions(+), 23 deletions(-)
diff --git a/ssg/templates.py b/ssg/templates.py
index 6783d0f2d5e..47a8a5eb852 100644
--- a/ssg/templates.py
+++ b/ssg/templates.py
@@ -107,14 +107,17 @@ def __init__(
self.checks_dir = checks_dir
self.output_dirs = dict()
for lang in languages:
+ lang_dir = lang
if lang == "oval":
# OVAL checks need to be put to a different directory because
- # they are processed differently than remediations later in the
- # build process
+ # they are processed differently than remediations later in
+ # the build process
output_dir = self.checks_dir
+ if lang.startswith("sce-"):
+ lang_dir = "sce"
else:
output_dir = self.remediations_dir
- dir_ = os.path.join(output_dir, lang)
+ dir_ = os.path.join(output_dir, lang_dir)
self.output_dirs[lang] = dir_
# scan directory structure and dynamically create list of templates
for item in sorted(os.listdir(self.templates_dir)):
@@ -124,25 +127,41 @@ def __init__(
maybe_template.load()
templates[item] = maybe_template
+ def build_lang_file(
+ self, rule_id, template_name, template_vars, lang, local_env_yaml):
+ """
+ Builds and returns templated content for a given rule for a given
+ language; does not write the output to disk.
+ """
+ if lang not in templates[template_name].langs:
+ return None
+
+ template_file_name = lang + ".template"
+ template_file_path = os.path.join(self.templates_dir, template_name, template_file_name)
+ template_parameters = templates[template_name].preprocess(template_vars, lang)
+ jinja_dict = ssg.utils.merge_dicts(local_env_yaml, template_parameters)
+ filled_template = ssg.jinja.process_file_with_macros(
+ template_file_path, jinja_dict)
+
+ return filled_template
def build_lang(
- self, rule_id, template_name, template_vars, lang, local_env_yaml):
+ self, rule_id, template_name, template_vars, lang, local_env_yaml, platforms=None):
"""
Builds templated content for a given rule for a given language.
Writes the output to the correct build directories.
"""
if lang not in templates[template_name].langs:
return
- template_file_name = lang + ".template"
- template_file_path = os.path.join(self.templates_dir, template_name, template_file_name)
+
+ filled_template = self.build_lang_file(rule_id, template_name,
+ template_vars, lang, local_env_yaml)
+
ext = lang_to_ext_map[lang]
output_file_name = rule_id + ext
output_filepath = os.path.join(
self.output_dirs[lang], output_file_name)
- template_parameters = templates[template_name].preprocess(template_vars, lang)
- jinja_dict = ssg.utils.merge_dicts(local_env_yaml, template_parameters)
- filled_template = ssg.jinja.process_file_with_macros(
- template_file_path, jinja_dict)
+
with open(output_filepath, "w") as f:
f.write(filled_template)
@@ -168,6 +187,37 @@ def get_langs_to_generate(self, rule):
else:
return languages
+ def get_template_name(self, template):
+ """
+ Given a template dictionary from a Rule instance, determine the name
+ of the template (from templates) this rule uses.
+ """
+ try:
+ template_name = template["name"]
+ except KeyError:
+ raise ValueError(
+ "Rule {0} is missing template name under template key".format(
+ rule_id))
+ if template_name not in templates.keys():
+ raise ValueError(
+ "Rule {0} uses template {1} which does not exist.".format(
+ rule_id, template_name))
+ return template_name
+
+ def get_resolved_langs_to_generate(self, rule):
+ """
+ Given a specific Rule instance, determine which languages are
+ generated by the combination of the rule's template_backends AND
+ the rule's template keys.
+ """
+ if rule.template is None:
+ return None
+
+ rule_langs = set(self.get_langs_to_generate(rule))
+ template_name = self.get_template_name(rule.template)
+ template_langs = set(templates[template_name].langs)
+ return rule_langs.intersection(template_langs)
+
def process_product_vars(self, all_variables):
"""
Given a dictionary with the format key[@<product>]=value, filter out
@@ -183,21 +233,12 @@ def process_product_vars(self, all_variables):
return processed
- def build_rule(self, rule_id, rule_title, template, langs_to_generate):
+ def build_rule(self, rule_id, rule_title, template, langs_to_generate, platforms=None):
"""
Builds templated content for a given rule for selected languages,
writing the output to the correct build directories.
"""
- try:
- template_name = template["name"]
- except KeyError:
- raise ValueError(
- "Rule {0} is missing template name under template key".format(
- rule_id))
- if template_name not in templates.keys():
- raise ValueError(
- "Rule {0} uses template {1} which does not exist.".format(
- rule_id, template_name))
+ template_name = self.get_template_name(template)
try:
template_vars = self.process_product_vars(template["vars"])
except KeyError:
@@ -216,11 +257,36 @@ def build_rule(self, rule_id, rule_title, template, langs_to_generate):
for lang in langs_to_generate:
try:
self.build_lang(
- rule_id, template_name, template_vars, lang, local_env_yaml)
+ rule_id, template_name, template_vars, lang, local_env_yaml, platforms)
except Exception as e:
print("Error building templated {0} content for rule {1}".format(lang, rule_id), file=sys.stderr)
raise e
+ def get_lang_for_rule(self, rule_id, rule_title, template, language):
+ """
+ For the specified rule, build and return only the specified language
+ content.
+ """
+ template_name = self.get_template_name(template)
+ try:
+ template_vars = self.process_product_vars(template["vars"])
+ except KeyError:
+ raise ValueError(
+ "Rule {0} does not contain mandatory 'vars:' key under "
+ "'template:' key.".format(rule_id))
+ # Add the rule ID which will be reused in OVAL templates as OVAL
+ # definition ID so that the build system matches the generated
+ # check with the rule.
+ template_vars["_rule_id"] = rule_id
+ # checks and remediations are processed with a custom YAML dict
+ local_env_yaml = self.env_yaml.copy()
+ local_env_yaml["rule_id"] = rule_id
+ local_env_yaml["rule_title"] = rule_title
+ local_env_yaml["products"] = self.env_yaml["product"]
+
+ return self.build_lang_file(rule_id, template_name, template_vars,
+ language, local_env_yaml)
+
def build_extra_ovals(self):
declaration_path = os.path.join(self.templates_dir, "extra_ovals.yml")
declaration = ssg.yaml.open_raw(declaration_path)
@@ -245,7 +311,7 @@ def build_all_rules(self):
continue
langs_to_generate = self.get_langs_to_generate(rule)
self.build_rule(
- rule.id_, rule.title, rule.template, langs_to_generate)
+ rule.id_, rule.title, rule.template, langs_to_generate, platforms=rule.platforms)
def build(self):
"""
From 8da04c6a85d40432a19904b16251885e12b75999 Mon Sep 17 00:00:00 2001
From: Alexander Scheel <alex.scheel@canonical.com>
Date: Tue, 6 Jul 2021 16:50:06 -0400
Subject: [PATCH 03/19] Remove incorrect tests for package installation
These tests assume that the package manager is yum (and forcibly
installs it) in some cases. This doesn't work on Debian-like systems;
prefer the templated version instead, which uses the
bash_package_install Jinja macro instead (and uses the
bash_package_remove macro for the negative case)
Signed-off-by: Alexander Scheel <alex.scheel@canonical.com>
---
.../ntp/package_chrony_installed/tests/installed.pass.sh | 4 ----
.../ntp/package_chrony_installed/tests/removed.fail.sh | 3 ---
.../package_audit_installed/tests/installed.pass.sh | 4 ----
.../package_audit_installed/tests/removed.fail.sh | 3 ---
.../package_rsyslog_installed/tests/installed.pass.sh | 3 ---
.../package_rsyslog_installed/tests/notinstalled.fail.sh | 3 ---
.../package_firewalld_installed/tests/installed.pass.sh | 4 ----
.../package_firewalld_installed/tests/removed.fail.sh | 3 ---
.../package_libselinux_installed/tests/installed.pass.sh | 4 ----
.../aide/package_aide_installed/tests/installed.pass.sh | 3 ---
.../aide/package_aide_installed/tests/notinstalled.fail.sh | 3 ---
.../sudo/package_sudo_installed/tests/installed.pass.sh | 4 ----
.../sudo/package_sudo_installed/tests/removed.fail.sh | 3 ---
13 files changed, 53 deletions(-)
delete mode 100644 linux_os/guide/services/ntp/package_chrony_installed/tests/installed.pass.sh
delete mode 100644 linux_os/guide/services/ntp/package_chrony_installed/tests/removed.fail.sh
delete mode 100644 linux_os/guide/system/auditing/package_audit_installed/tests/installed.pass.sh
delete mode 100644 linux_os/guide/system/auditing/package_audit_installed/tests/removed.fail.sh
delete mode 100644 linux_os/guide/system/logging/package_rsyslog_installed/tests/installed.pass.sh
delete mode 100644 linux_os/guide/system/logging/package_rsyslog_installed/tests/notinstalled.fail.sh
delete mode 100644 linux_os/guide/system/network/network-firewalld/firewalld_activation/package_firewalld_installed/tests/installed.pass.sh
delete mode 100644 linux_os/guide/system/network/network-firewalld/firewalld_activation/package_firewalld_installed/tests/removed.fail.sh
delete mode 100644 linux_os/guide/system/selinux/package_libselinux_installed/tests/installed.pass.sh
delete mode 100644 linux_os/guide/system/software/integrity/software-integrity/aide/package_aide_installed/tests/installed.pass.sh
delete mode 100644 linux_os/guide/system/software/integrity/software-integrity/aide/package_aide_installed/tests/notinstalled.fail.sh
delete mode 100644 linux_os/guide/system/software/sudo/package_sudo_installed/tests/installed.pass.sh
delete mode 100644 linux_os/guide/system/software/sudo/package_sudo_installed/tests/removed.fail.sh
diff --git a/linux_os/guide/services/ntp/package_chrony_installed/tests/installed.pass.sh b/linux_os/guide/services/ntp/package_chrony_installed/tests/installed.pass.sh
deleted file mode 100644
index c692df2b282..00000000000
--- a/linux_os/guide/services/ntp/package_chrony_installed/tests/installed.pass.sh
+++ /dev/null
@@ -1,4 +0,0 @@
-#!/bin/bash
-# package = yum
-
-yum install -y chrony
diff --git a/linux_os/guide/services/ntp/package_chrony_installed/tests/removed.fail.sh b/linux_os/guide/services/ntp/package_chrony_installed/tests/removed.fail.sh
deleted file mode 100644
index eba551f99d4..00000000000
--- a/linux_os/guide/services/ntp/package_chrony_installed/tests/removed.fail.sh
+++ /dev/null
@@ -1,3 +0,0 @@
-#!/bin/bash
-
-yum remove -y chrony
diff --git a/linux_os/guide/system/auditing/package_audit_installed/tests/installed.pass.sh b/linux_os/guide/system/auditing/package_audit_installed/tests/installed.pass.sh
deleted file mode 100644
index 1c19ce31a87..00000000000
--- a/linux_os/guide/system/auditing/package_audit_installed/tests/installed.pass.sh
+++ /dev/null
@@ -1,4 +0,0 @@
-#!/bin/bash
-# package = yum
-
-yum install -y audit
diff --git a/linux_os/guide/system/auditing/package_audit_installed/tests/removed.fail.sh b/linux_os/guide/system/auditing/package_audit_installed/tests/removed.fail.sh
deleted file mode 100644
index 45dd05a9638..00000000000
--- a/linux_os/guide/system/auditing/package_audit_installed/tests/removed.fail.sh
+++ /dev/null
@@ -1,3 +0,0 @@
-#!/bin/bash
-
-yum remove -y audit
diff --git a/linux_os/guide/system/logging/package_rsyslog_installed/tests/installed.pass.sh b/linux_os/guide/system/logging/package_rsyslog_installed/tests/installed.pass.sh
deleted file mode 100644
index 6e4107fa3bd..00000000000
--- a/linux_os/guide/system/logging/package_rsyslog_installed/tests/installed.pass.sh
+++ /dev/null
@@ -1,3 +0,0 @@
-#!/bin/bash
-# packages = rsyslog
-
diff --git a/linux_os/guide/system/logging/package_rsyslog_installed/tests/notinstalled.fail.sh b/linux_os/guide/system/logging/package_rsyslog_installed/tests/notinstalled.fail.sh
deleted file mode 100644
index f64bf1b340a..00000000000
--- a/linux_os/guide/system/logging/package_rsyslog_installed/tests/notinstalled.fail.sh
+++ /dev/null
@@ -1,3 +0,0 @@
-#!/bin/bash
-
-yum remove -y rsyslog
diff --git a/linux_os/guide/system/network/network-firewalld/firewalld_activation/package_firewalld_installed/tests/installed.pass.sh b/linux_os/guide/system/network/network-firewalld/firewalld_activation/package_firewalld_installed/tests/installed.pass.sh
deleted file mode 100644
index 7a4748863bf..00000000000
--- a/linux_os/guide/system/network/network-firewalld/firewalld_activation/package_firewalld_installed/tests/installed.pass.sh
+++ /dev/null
@@ -1,4 +0,0 @@
-#!/bin/bash
-# package = yum
-
-yum install -y firewalld
diff --git a/linux_os/guide/system/network/network-firewalld/firewalld_activation/package_firewalld_installed/tests/removed.fail.sh b/linux_os/guide/system/network/network-firewalld/firewalld_activation/package_firewalld_installed/tests/removed.fail.sh
deleted file mode 100644
index a9107df7de8..00000000000
--- a/linux_os/guide/system/network/network-firewalld/firewalld_activation/package_firewalld_installed/tests/removed.fail.sh
+++ /dev/null
@@ -1,3 +0,0 @@
-#!/bin/bash
-
-yum remove -y firewalld
diff --git a/linux_os/guide/system/selinux/package_libselinux_installed/tests/installed.pass.sh b/linux_os/guide/system/selinux/package_libselinux_installed/tests/installed.pass.sh
deleted file mode 100644
index 5d30cf77841..00000000000
--- a/linux_os/guide/system/selinux/package_libselinux_installed/tests/installed.pass.sh
+++ /dev/null
@@ -1,4 +0,0 @@
-#!/bin/bash
-# package = yum
-
-yum install -y libselinux
diff --git a/linux_os/guide/system/software/integrity/software-integrity/aide/package_aide_installed/tests/installed.pass.sh b/linux_os/guide/system/software/integrity/software-integrity/aide/package_aide_installed/tests/installed.pass.sh
deleted file mode 100644
index fa8b85b..0000000
--- a/linux_os/guide/system/software/integrity/software-integrity/aide/package_aide_installed/tests/installed.pass.sh
+++ /dev/null
@@ -1,3 +0,0 @@
-#!/bin/bash
-# packages = aide
-
diff --git a/linux_os/guide/system/software/integrity/software-integrity/aide/package_aide_installed/tests/notinstalled.fail.sh b/linux_os/guide/system/software/integrity/software-integrity/aide/package_aide_installed/tests/notinstalled.fail.sh
deleted file mode 100644
index 75978b6..0000000
--- a/linux_os/guide/system/software/integrity/software-integrity/aide/package_aide_installed/tests/notinstalled.fail.sh
+++ /dev/null
@@ -1,3 +0,0 @@
-#!/bin/bash
-
-yum remove -y aide
diff --git a/linux_os/guide/system/software/sudo/package_sudo_installed/tests/installed.pass.sh b/linux_os/guide/system/software/sudo/package_sudo_installed/tests/installed.pass.sh
deleted file mode 100644
index dafc6998a9a..00000000000
--- a/linux_os/guide/system/software/sudo/package_sudo_installed/tests/installed.pass.sh
+++ /dev/null
@@ -1,4 +0,0 @@
-#!/bin/bash
-# package = yum
-
-yum install -y sudo
diff --git a/linux_os/guide/system/software/sudo/package_sudo_installed/tests/removed.fail.sh b/linux_os/guide/system/software/sudo/package_sudo_installed/tests/removed.fail.sh
deleted file mode 100644
index d22b562c15e..00000000000
--- a/linux_os/guide/system/software/sudo/package_sudo_installed/tests/removed.fail.sh
+++ /dev/null
@@ -1,3 +0,0 @@
-#!/bin/bash
-
-rpm -e --nodeps sudo
From 6c49dcb79aaf5b239f09ecfdac07b443736b5b5e Mon Sep 17 00:00:00 2001
From: Alexander Scheel <alex.scheel@canonical.com>
Date: Tue, 6 Jul 2021 16:51:05 -0400
Subject: [PATCH 04/19] Support templating test content
We introduced a new method to the template builder in ssg/templates.py
for finding (and building) test content under shared/templates/...
This uses Jinja macros and has the full context of the template present,
so for instance, package_installed tests can use the correct package
name on the target platform.
Signed-off-by: Alexander Scheel <alex.scheel@canonical.com>
---
ssg/templates.py | 46 ++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 46 insertions(+)
diff --git a/ssg/templates.py b/ssg/templates.py
index 47a8a5eb852..0a39c3ba094 100644
--- a/ssg/templates.py
+++ b/ssg/templates.py
@@ -145,6 +145,52 @@ def build_lang_file(
return filled_template
+ def get_all_tests(
+ self, rule_id, rule_template, local_env_yaml, platforms=None):
+ """
+ Builds a dictionary of a test case path -> test case value mapping.
+
+ Here, we want to know what the relative path on disk (under the tests/
+ subdirectory) is (such as "installed.pass.sh"), along with the actual
+ contents of the test case.
+
+ Presumably, we'll find the test case we want (all of them when
+ building a test case tarball) and write them to disk in the
+ appropriate location.
+ """
+ template_name = rule_template['name']
+ template_vars = rule_template['vars']
+
+ base_dir = os.path.abspath(os.path.join(self.templates_dir, template_name, "tests"))
+ results = dict()
+
+ # If no test cases exist, return an empty dictionary.
+ if not os.path.exists(base_dir):
+ return results
+
+ # Walk files; note that we don't need to do anything about directories
+ # as only files are recorded in the mapping; directories can be
+ # inferred from the path.
+ for dirpath, _, filenames in os.walk(base_dir):
+ if not filenames:
+ continue
+
+ for filename in filenames:
+ # Relative path to the file becomes our results key.
+ absolute_path = os.path.abspath(os.path.join(dirpath, filename))
+ relative_path = os.path.relpath(absolute_path, base_dir)
+
+ # Load template parameters and apply it to the test case.
+ template_parameters = templates[template_name].preprocess(template_vars, "tests")
+ jinja_dict = ssg.utils.merge_dicts(local_env_yaml, template_parameters)
+ filled_template = ssg.jinja.process_file_with_macros(
+ absolute_path, jinja_dict)
+
+ # Save the results under the relative path.
+ results[relative_path] = filled_template
+
+ return results
+
def build_lang(
self, rule_id, template_name, template_vars, lang, local_env_yaml, platforms=None):
"""
From 4bd5061ffbb524b09975982fbd6dae26f35770c3 Mon Sep 17 00:00:00 2001
From: Alexander Scheel <alex.scheel@canonical.com>
Date: Tue, 6 Jul 2021 16:53:44 -0400
Subject: [PATCH 05/19] Add templated tests into templated directory
This extends the existing templated directory generation to also include
tests under shared/templates/.../tests. The complicated portion is in
subdirectory handling: we need to ensure we create all necessary nested
subdirectories that are implicitly implied by the relative paths. This
is a shortcoming in the get_all_tests's return format.
Signed-off-by: Alexander Scheel <alex.scheel@canonical.com>
---
tests/ssg_test_suite/common.py | 43 ++++++++++++++++++++++++++++++++++
1 file changed, 43 insertions(+)
diff --git a/tests/ssg_test_suite/common.py b/tests/ssg_test_suite/common.py
index 3dbeaf304a4..2bd4b2bd1c7 100644
--- a/tests/ssg_test_suite/common.py
+++ b/tests/ssg_test_suite/common.py
@@ -21,6 +21,9 @@
from ssg.rule_yaml import parse_prodtype
from ssg_test_suite.log import LogHelper
+import ssg.templates
+
+
Scenario_run = namedtuple(
"Scenario_run",
("rule_id", "script"))
@@ -39,6 +42,8 @@
_SHARED_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), '../shared'))
+_SHARED_TEMPLATES = os.path.abspath(os.path.join(SSG_ROOT, 'shared/templates'))
+
REMOTE_USER = "root"
REMOTE_USER_HOME_DIRECTORY = "/root"
REMOTE_TEST_SCENARIOS_DIRECTORY = os.path.join(REMOTE_USER_HOME_DIRECTORY, "ssgts")
@@ -278,6 +283,11 @@ def template_tests(product=None):
yaml_path = product_yaml_path(SSG_ROOT, product)
product_yaml = load_product_yaml(yaml_path)
+ # Initialize a mock template_builder.
+ empty = "/ssgts/empty/placeholder"
+ template_builder = ssg.templates.Builder(product_yaml, empty,
+ _SHARED_TEMPLATES, empty, empty)
+
# Below we could run into a DocumentationNotComplete error. However,
# because the test suite isn't executed in the context of a particular
# build (though, ideally it would be linked), we may not know exactly
@@ -299,8 +309,11 @@ def template_tests(product=None):
# Load rule content in our environment. We use this to satisfy
# some implied properties that might be used in the test suite.
+ # Make sure we normalize to a specific product as well so that
+ # when we load templated content it is correct.
rule_path = get_rule_dir_yaml(dirpath)
rule = RuleYAML.from_yaml(rule_path, product_yaml)
+ rule.normalize(product)
# Note that most places would check prodtype, but we don't care
# about that here: if the rule is available to the product, we
@@ -320,6 +333,36 @@ def template_tests(product=None):
dest_path = os.path.join(tmpdir, rule.id_)
os.mkdir(dest_path)
+ # The priority order is rule-specific tests over templated tests.
+ # That is, for any test under rule_id/tests with a name matching a
+ # test under shared/templates/<template_name>/tests/, the former
+ # will preferred. This means we need to process templates first,
+ # so they'll be overwritten later if necessary.
+ if rule.template:
+ templated_tests = template_builder.get_all_tests(
+ rule.id_, rule.template, local_env_yaml)
+
+ for relative_path in templated_tests:
+ output_path = os.path.join(dest_path, relative_path)
+
+ # If there's a separator in the file name, it means we
+ # have nested directories to deal with.
+ if os.path.sep in relative_path:
+ parts = os.path.split(relative_path)[:-1]
+ for subdir_index in range(len(parts)):
+ # We need to expand all directories in correct
+ # order, preserving any previous directories (as
+ # they're nested). Use the star operator to splat
+ # array parts into arguments to os.path.join(...).
+ new_directory = os.path.join(dest_path, *parts[:subdir_index])
+ os.mkdir(new_directory)
+
+ # Write out the test content to the desired location on
+ # disk.
+ with open(output_path, 'w') as output_fp:
+ test_content = templated_tests[relative_path]
+ print(test_content, file=output_fp)
+
# Walk the test directory, writing all tests into the output
# directory, recursively.
tests_dir_path = os.path.join(dirpath, "tests")
From 013b247c1be897bd12ae4cf2bc2443279b1a52c9 Mon Sep 17 00:00:00 2001
From: Alexander Scheel <alex.scheel@canonical.com>
Date: Wed, 7 Jul 2021 07:00:37 -0400
Subject: [PATCH 06/19] Teach iterate_over_rules about templated tests
There are two parts to rule-based test execution:
1. Building the tarball of tests and shipping it over to the remote
machine.
2. Iterating over these tests to see which scenarios are actually
applicable and any other metadata actions we need to take.
In particular, item two is controlled by iterate_over_rules. Here we
need to:
1. Teach it about the templating system.
2. Switch it from returning a list of test cases to a dictionary
mapping test case -> contents. This allows us to contain the
templating system to common.py only and not leak it into other parts
of the test suite.
3. Do a little bit of refactoring to contain shared code between
iterate_over_rules and template_tests.
Signed-off-by: Alexander Scheel <alex.scheel@canonical.com>
---
tests/ssg_test_suite/common.py | 146 ++++++++++++++++++++++++---------
1 file changed, 108 insertions(+), 38 deletions(-)
diff --git a/tests/ssg_test_suite/common.py b/tests/ssg_test_suite/common.py
index 2bd4b2bd1c7..b357f816027 100644
--- a/tests/ssg_test_suite/common.py
+++ b/tests/ssg_test_suite/common.py
@@ -264,6 +264,62 @@ def _rel_abs_path(current_path, base_path):
return os.path.relpath(current_path, base_path)
+def get_product_context(product=None):
+ """
+ Returns a product YAML context if any product is specified. Hard-coded to
+ assume a debug build.
+ """
+ # Load product's YAML file if present. This will allow us to parse
+ # tests in the context of the product we're executing under.
+ product_yaml = dict()
+ if product:
+ yaml_path = product_yaml_path(SSG_ROOT, product)
+ product_yaml = load_product_yaml(yaml_path)
+
+ # We could run into a DocumentationNotComplete error when loading a
+ # rule's YAML contents. However, because the test suite isn't executed
+ # in the context of a particular build (though, ideally it would be
+ # linked), we may not know exactly whether the top-level rule/profile
+ # we're testing is actually completed. Thus, forcibly set the required
+ # property to bypass this error.
+ product_yaml['cmake_build_type'] = 'Debug'
+
+ return product_yaml
+
+
+def load_rule_and_env(rule_dir_path, env_yaml, product=None):
+ """
+ Loads a rule and returns the combination of the RuleYAML class and
+ the corresponding local environment for that rule.
+ """
+
+ # First build the path to the rule.yml file
+ rule_path = get_rule_dir_yaml(rule_dir_path)
+
+ # Load rule content in our environment. We use this to satisfy
+ # some implied properties that might be used in the test suite.
+ # Make sure we normalize to a specific product as well so that
+ # when we load templated content it is correct.
+ rule = RuleYAML.from_yaml(rule_path, env_yaml)
+ rule.normalize(product)
+
+ # Note that most places would check prodtype, but we don't care
+ # about that here: if the rule is available to the product, we
+ # load and parse it anyways as we have no knowledge of the
+ # top-level profile or rule passed into the test suite.
+ prodtypes = parse_prodtype(rule.prodtype)
+
+ # Our local copy of env_yaml needs some properties from rule.yml
+ # for completeness.
+ local_env_yaml = dict()
+ local_env_yaml.update(env_yaml)
+ local_env_yaml['rule_id'] = rule.id_
+ local_env_yaml['rule_title'] = rule.title
+ local_env_yaml['products'] = prodtypes
+
+ return rule, local_env_yaml
+
+
def template_tests(product=None):
"""
Create a temporary directory with test cases parsed via jinja using
@@ -276,26 +332,14 @@ def template_tests(product=None):
# it on success. Wrap in a try/except block and reraise the original
# exception after removing the temporary directory.
try:
- # Load product's YAML file if present. This will allow us to parse
- # tests in the context of the product we're executing under.
- product_yaml = dict()
- if product:
- yaml_path = product_yaml_path(SSG_ROOT, product)
- product_yaml = load_product_yaml(yaml_path)
+ # Load the product context we're executing under, if any.
+ product_yaml = get_product_context(product)
# Initialize a mock template_builder.
empty = "/ssgts/empty/placeholder"
template_builder = ssg.templates.Builder(product_yaml, empty,
_SHARED_TEMPLATES, empty, empty)
- # Below we could run into a DocumentationNotComplete error. However,
- # because the test suite isn't executed in the context of a particular
- # build (though, ideally it would be linked), we may not know exactly
- # whether the top-level rule/profile we're testing is actually
- # completed. Thus, forcibly set the required property to bypass this
- # error.
- product_yaml['cmake_build_type'] = 'Debug'
-
# Note that we're not exactly copying 1-for-1 the contents of the
# directory structure into the temporary one. Instead we want a
# flattened mapping with all rules in a single top-level directory
@@ -307,27 +351,8 @@ def template_tests(product=None):
if "tests" not in dirnames or not is_rule_dir(dirpath):
continue
- # Load rule content in our environment. We use this to satisfy
- # some implied properties that might be used in the test suite.
- # Make sure we normalize to a specific product as well so that
- # when we load templated content it is correct.
- rule_path = get_rule_dir_yaml(dirpath)
- rule = RuleYAML.from_yaml(rule_path, product_yaml)
- rule.normalize(product)
-
- # Note that most places would check prodtype, but we don't care
- # about that here: if the rule is available to the product, we
- # load and parse it anyways as we have no knowledge of the
- # top-level profile or rule passed into the test suite.
- prodtypes = parse_prodtype(rule.prodtype)
-
- # Our local copy of env_yaml needs some properties from rule.yml
- # for completeness.
- local_env_yaml = dict()
- local_env_yaml.update(product_yaml)
- local_env_yaml['rule_id'] = rule.id_
- local_env_yaml['rule_title'] = rule.title
- local_env_yaml['products'] = prodtypes
+ # Load the rule and its environment
+ rule, local_env_yaml = load_rule_and_env(dirpath, product_yaml, product)
# Create the destination directory.
dest_path = os.path.join(tmpdir, rule.id_)
@@ -473,21 +498,66 @@ def iterate_over_rules(product=None):
id -- full rule id as it is present in datastream
short_id -- short rule ID, the same as basename of the directory
containing the test scenarios in Bash
- files -- list of executable .sh files in the "tests" directory
+ files -- list of executable .sh files in the uploaded tarball
"""
+
+ # Here we need to perform some magic to handle parsing the rule (from a
+ # product perspective) and loading any templated tests. In particular,
+ # identifying which tests to potentially run involves invoking the
+ # templating engine.
+ #
+ # Begin by loading context about our execution environment, if any.
+ product_yaml = get_product_context(product)
+
+ # Initialize a mock template_builder.
+ empty = "/ssgts/empty/placeholder"
+ template_builder = ssg.templates.Builder(product_yaml, empty,
+ _SHARED_TEMPLATES, empty, empty)
+
for dirpath, dirnames, filenames in walk_through_benchmark_dirs(product):
if "rule.yml" in filenames and "tests" in dirnames:
short_rule_id = os.path.basename(dirpath)
+
+ # Load the rule itself to check for a template.
+ rule, local_env_yaml = load_rule_and_env(dirpath, product_yaml, product)
+
+ # All tests is a mapping from path (in the tarball) to contents
+ # of the test case. This is necessary because later code (which
+ # attempts to parse headers from the test case) don't have easy
+ # access to templated content. By reading it and returning it
+ # here, we can save later code from having to understand the
+ # templating system.
+ all_tests = dict()
+
+ # Start
+ if rule.template:
+ templated_tests = template_builder.get_all_tests(
+ rule.id_, rule.template, local_env_yaml)
+ all_tests.update(templated_tests)
+
+ # Add additional tests from the local rule directory. Note that,
+ # like the behavior in template_tests, this will overwrite any
+ # templated tests with the same file name.
tests_dir = os.path.join(dirpath, "tests")
tests_dir_files = os.listdir(tests_dir)
+ for test_case in tests_dir_files:
+ test_path = os.path.join(tests_dir, test_case)
+ if os.path.isdir(test_path):
+ continue
+
+ with open(test_path) as fp:
+ all_tests[test_case] = fp.read()
+
# Filter out everything except the shell test scenarios.
# Other files in rule directories are editor swap files
# or other content than a test case.
- scripts = filter(lambda x: x.endswith(".sh"), tests_dir_files)
+ allowed_scripts = filter(lambda x: x.endswith(".sh"), all_tests)
+ content_mapping = {x: all_tests[x] for x in allowed_scripts}
+
full_rule_id = OSCAP_RULE + short_rule_id
result = Rule(
directory=tests_dir, id=full_rule_id, short_id=short_rule_id,
- files=scripts)
+ files=content_mapping)
yield result
From 84014297a6881610e682ec7d54eca658b6ff127a Mon Sep 17 00:00:00 2001
From: Alexander Scheel <alex.scheel@canonical.com>
Date: Wed, 7 Jul 2021 07:05:13 -0400
Subject: [PATCH 07/19] Update rule execution to support templated tests
We needed to make a few small changes to rule-based test execution in
order for it to correctly understand templated tests:
1. Product selection (from the CLI) needs to be passed down into the
iterate_over_rules helper function.
2. Scenario loading needs to have knowledge of the returned test data.
3. Parsing of parameters needs to occur from the returned test data
rather than attempting to re-read it ourselves.
My assumption is that this will suffice for combined, since that method
of operation uses this method internally.
Signed-off-by: Alexander Scheel <alex.scheel@canonical.com>
---
tests/ssg_test_suite/rule.py | 30 +++++++++++++++---------------
1 file changed, 15 insertions(+), 15 deletions(-)
diff --git a/tests/ssg_test_suite/rule.py b/tests/ssg_test_suite/rule.py
index 9f390749077..5ad7eb8ed27 100644
--- a/tests/ssg_test_suite/rule.py
+++ b/tests/ssg_test_suite/rule.py
@@ -24,7 +24,7 @@
Scenario = collections.namedtuple(
- "Scenario", ["script", "context", "script_params"])
+ "Scenario", ["script", "context", "script_params", "contents"])
def get_viable_profiles(selected_profiles, datastream, benchmark, script=None):
@@ -250,7 +250,7 @@ def _prepare_environment(self, scenarios_by_rule):
def _get_rules_to_test(self, target):
rules_to_test = []
- for rule in common.iterate_over_rules():
+ for rule in common.iterate_over_rules(self.test_env.product):
if not self._rule_should_be_tested(rule, target):
continue
if not xml_operations.find_rule_in_benchmark(
@@ -300,7 +300,7 @@ def _modify_parameters(self, script, params):
.format(OSCAP_PROFILE_ALL_ID, script))
return params
- def _parse_parameters(self, script):
+ def _parse_parameters(self, script_content):
"""Parse parameters from script header"""
params = {'profiles': [],
'templates': [],
@@ -309,16 +309,15 @@ def _parse_parameters(self, script):
'remediation': ['all'],
'variables': [],
}
- with open(script, 'r') as script_file:
- script_content = script_file.read()
- for parameter in params:
- found = re.search(r'^# {0} = (.*)$'.format(parameter),
- script_content,
- re.MULTILINE)
- if found is None:
- continue
- splitted = found.group(1).split(',')
- params[parameter] = [value.strip() for value in splitted]
+
+ for parameter in params:
+ found = re.search(r'^# {0} = (.*)$'.format(parameter),
+ script_content, re.MULTILINE)
+ if found is None:
+ continue
+ splitted = found.group(1).split(',')
+ params[parameter] = [value.strip() for value in splitted]
+
return params
def _get_scenarios(self, rule_dir, scripts, scenarios_regex, benchmark_cpes):
@@ -331,6 +330,7 @@ def _get_scenarios(self, rule_dir, scripts, scenarios_regex, benchmark_cpes):
scenarios = []
for script in scripts:
+ script_contents = scripts[script]
if scenarios_regex is not None:
if scenarios_pattern.match(script) is None:
logging.debug("Skipping script %s - it did not match "
@@ -338,10 +338,10 @@ def _get_scenarios(self, rule_dir, scripts, scenarios_regex, benchmark_cpes):
continue
script_context = _get_script_context(script)
if script_context is not None:
- script_params = self._parse_parameters(os.path.join(rule_dir, script))
+ script_params = self._parse_parameters(script_contents)
script_params = self._modify_parameters(script, script_params)
if common.matches_platform(script_params["platform"], benchmark_cpes):
- scenarios += [Scenario(script, script_context, script_params)]
+ scenarios += [Scenario(script, script_context, script_params, script_contents)]
else:
logging.warning("Script %s is not applicable on given platform" % script)
From c763af671037cb9bfd5bfd5bd24f5efe9e4fc4c1 Mon Sep 17 00:00:00 2001
From: Alexander Scheel <alex.scheel@canonical.com>
Date: Wed, 7 Jul 2021 07:22:47 -0400
Subject: [PATCH 08/19] Make sure we read all test contents with Jinja
Signed-off-by: Alexander Scheel <alex.scheel@canonical.com>
---
tests/ssg_test_suite/common.py | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
diff --git a/tests/ssg_test_suite/common.py b/tests/ssg_test_suite/common.py
index b357f816027..865c8a8b988 100644
--- a/tests/ssg_test_suite/common.py
+++ b/tests/ssg_test_suite/common.py
@@ -545,8 +545,7 @@ def iterate_over_rules(product=None):
if os.path.isdir(test_path):
continue
- with open(test_path) as fp:
- all_tests[test_case] = fp.read()
+ all_tests[test_case] = process_file(test_path, local_env_yaml)
# Filter out everything except the shell test scenarios.
# Other files in rule directories are editor swap files
From 1234addaa4f776e3c814192c0aaf8f2137a74481 Mon Sep 17 00:00:00 2001
From: Alexander Scheel <alex.scheel@canonical.com>
Date: Thu, 8 Jul 2021 07:44:16 -0400
Subject: [PATCH 09/19] Fix pep8 issues
Signed-off-by: Alexander Scheel <alex.scheel@canonical.com>
---
ssg/templates.py | 5 +++--
tests/ssg_test_suite/common.py | 5 +++--
2 files changed, 6 insertions(+), 4 deletions(-)
diff --git a/ssg/templates.py b/ssg/templates.py
index 0a39c3ba094..818b1f79f68 100644
--- a/ssg/templates.py
+++ b/ssg/templates.py
@@ -201,7 +201,8 @@ def build_lang(
return
filled_template = self.build_lang_file(rule_id, template_name,
- template_vars, lang, local_env_yaml)
+ template_vars, lang,
+ local_env_yaml)
ext = lang_to_ext_map[lang]
output_file_name = rule_id + ext
@@ -331,7 +332,7 @@ def get_lang_for_rule(self, rule_id, rule_title, template, language):
local_env_yaml["products"] = self.env_yaml["product"]
return self.build_lang_file(rule_id, template_name, template_vars,
- language, local_env_yaml)
+ language, local_env_yaml)
def build_extra_ovals(self):
declaration_path = os.path.join(self.templates_dir, "extra_ovals.yml")
diff --git a/tests/ssg_test_suite/common.py b/tests/ssg_test_suite/common.py
index 865c8a8b988..977e9f52c24 100644
--- a/tests/ssg_test_suite/common.py
+++ b/tests/ssg_test_suite/common.py
@@ -338,7 +338,8 @@ def template_tests(product=None):
# Initialize a mock template_builder.
empty = "/ssgts/empty/placeholder"
template_builder = ssg.templates.Builder(product_yaml, empty,
- _SHARED_TEMPLATES, empty, empty)
+ _SHARED_TEMPLATES, empty,
+ empty)
# Note that we're not exactly copying 1-for-1 the contents of the
# directory structure into the temporary one. Instead we want a
@@ -512,7 +513,7 @@ def iterate_over_rules(product=None):
# Initialize a mock template_builder.
empty = "/ssgts/empty/placeholder"
template_builder = ssg.templates.Builder(product_yaml, empty,
- _SHARED_TEMPLATES, empty, empty)
+ _SHARED_TEMPLATES, empty, empty)
for dirpath, dirnames, filenames in walk_through_benchmark_dirs(product):
if "rule.yml" in filenames and "tests" in dirnames:
From 81fcdf3e18f83da44b697ef7f47759a8eadd9854 Mon Sep 17 00:00:00 2001
From: Alexander Scheel <alex.scheel@canonical.com>
Date: Wed, 14 Jul 2021 08:06:57 -0400
Subject: [PATCH 10/19] Split template_tests into template_rule_tests
Signed-off-by: Alexander Scheel <alex.scheel@canonical.com>
---
tests/ssg_test_suite/common.py | 146 +++++++++++++++++----------------
1 file changed, 77 insertions(+), 69 deletions(-)
diff --git a/tests/ssg_test_suite/common.py b/tests/ssg_test_suite/common.py
index 977e9f52c24..4e77bd888fe 100644
--- a/tests/ssg_test_suite/common.py
+++ b/tests/ssg_test_suite/common.py
@@ -320,6 +320,82 @@ def load_rule_and_env(rule_dir_path, env_yaml, product=None):
return rule, local_env_yaml
+def template_rule_tests(product, product_yaml, template_builder, tmpdir, dirpath):
+ """
+ For a given rule directory, templates all contained tests into the output
+ (tmpdir) directory.
+ """
+
+ # Load the rule and its environment
+ rule, local_env_yaml = load_rule_and_env(dirpath, product_yaml, product)
+
+ # Create the destination directory.
+ dest_path = os.path.join(tmpdir, rule.id_)
+ os.mkdir(dest_path)
+
+ # The priority order is rule-specific tests over templated tests.
+ # That is, for any test under rule_id/tests with a name matching a
+ # test under shared/templates/<template_name>/tests/, the former
+ # will preferred. This means we need to process templates first,
+ # so they'll be overwritten later if necessary.
+ if rule.template:
+ templated_tests = template_builder.get_all_tests(rule.id_, rule.template,
+ local_env_yaml)
+
+ for relative_path in templated_tests:
+ output_path = os.path.join(dest_path, relative_path)
+
+ # If there's a separator in the file name, it means we
+ # have nested directories to deal with.
+ if os.path.sep in relative_path:
+ parts = os.path.split(relative_path)[:-1]
+ for subdir_index in range(len(parts)):
+ # We need to expand all directories in correct
+ # order, preserving any previous directories (as
+ # they're nested). Use the star operator to splat
+ # array parts into arguments to os.path.join(...).
+ new_directory = os.path.join(dest_path, *parts[:subdir_index])
+ os.mkdir(new_directory)
+
+ # Write out the test content to the desired location on
+ # disk.
+ with open(output_path, 'w') as output_fp:
+ test_content = templated_tests[relative_path]
+ print(test_content, file=output_fp)
+
+ # Walk the test directory, writing all tests into the output
+ # directory, recursively.
+ tests_dir_path = os.path.join(dirpath, "tests")
+ tests_dir_path = os.path.abspath(tests_dir_path)
+ for dirpath, dirnames, filenames in os.walk(tests_dir_path):
+ for dirname in dirnames:
+ # We want to recreate the correct path under the temporary
+ # directory. Resolve it to a relative path from the tests/
+ # directory.
+ dir_path = _rel_abs_path(os.path.join(dirpath, dirname), tests_dir_path)
+ assert '../' not in dir_path
+ tmp_dir_path = os.path.join(dest_path, dir_path)
+ os.mkdir(tmp_dir_path)
+
+ for filename in filenames:
+ # We want to recreate the correct path under the temporary
+ # directory. Resolve it to a relative path from the tests/
+ # directory. Assumption: directories should be created
+ # prior to recursing into them, so we don't need to handle
+ # if a file's parent directory doesn't yet exist under the
+ # destination.
+ src_test_path = os.path.join(dirpath, filename)
+ rel_test_path = _rel_abs_path(src_test_path, tests_dir_path)
+ dest_test_path = os.path.join(dest_path, rel_test_path)
+
+ # Rather than performing an OS-level copy, we need to
+ # first parse the test with jinja and then write it back
+ # out to the destination.
+ parsed_test = process_file(src_test_path, local_env_yaml)
+ with open(dest_test_path, 'w') as output_fp:
+ print(parsed_test, file=output_fp)
+
+
def template_tests(product=None):
"""
Create a temporary directory with test cases parsed via jinja using
@@ -352,75 +428,7 @@ def template_tests(product=None):
if "tests" not in dirnames or not is_rule_dir(dirpath):
continue
- # Load the rule and its environment
- rule, local_env_yaml = load_rule_and_env(dirpath, product_yaml, product)
-
- # Create the destination directory.
- dest_path = os.path.join(tmpdir, rule.id_)
- os.mkdir(dest_path)
-
- # The priority order is rule-specific tests over templated tests.
- # That is, for any test under rule_id/tests with a name matching a
- # test under shared/templates/<template_name>/tests/, the former
- # will preferred. This means we need to process templates first,
- # so they'll be overwritten later if necessary.
- if rule.template:
- templated_tests = template_builder.get_all_tests(
- rule.id_, rule.template, local_env_yaml)
-
- for relative_path in templated_tests:
- output_path = os.path.join(dest_path, relative_path)
-
- # If there's a separator in the file name, it means we
- # have nested directories to deal with.
- if os.path.sep in relative_path:
- parts = os.path.split(relative_path)[:-1]
- for subdir_index in range(len(parts)):
- # We need to expand all directories in correct
- # order, preserving any previous directories (as
- # they're nested). Use the star operator to splat
- # array parts into arguments to os.path.join(...).
- new_directory = os.path.join(dest_path, *parts[:subdir_index])
- os.mkdir(new_directory)
-
- # Write out the test content to the desired location on
- # disk.
- with open(output_path, 'w') as output_fp:
- test_content = templated_tests[relative_path]
- print(test_content, file=output_fp)
-
- # Walk the test directory, writing all tests into the output
- # directory, recursively.
- tests_dir_path = os.path.join(dirpath, "tests")
- tests_dir_path = os.path.abspath(tests_dir_path)
- for dirpath, dirnames, filenames in os.walk(tests_dir_path):
- for dirname in dirnames:
- # We want to recreate the correct path under the temporary
- # directory. Resolve it to a relative path from the tests/
- # directory.
- dir_path = _rel_abs_path(os.path.join(dirpath, dirname), tests_dir_path)
- assert '../' not in dir_path
- tmp_dir_path = os.path.join(dest_path, dir_path)
- os.mkdir(tmp_dir_path)
-
- for filename in filenames:
- # We want to recreate the correct path under the temporary
- # directory. Resolve it to a relative path from the tests/
- # directory. Assumption: directories should be created
- # prior to recursing into them, so we don't need to handle
- # if a file's parent directory doesn't yet exist under the
- # destination.
- src_test_path = os.path.join(dirpath, filename)
- rel_test_path = _rel_abs_path(src_test_path, tests_dir_path)
- dest_test_path = os.path.join(dest_path, rel_test_path)
-
- # Rather than performing an OS-level copy, we need to
- # first parse the test with jinja and then write it back
- # out to the destination.
- parsed_test = process_file(src_test_path, local_env_yaml)
- with open(dest_test_path, 'w') as output_fp:
- print(parsed_test, file=output_fp)
-
+ template_rule_tests(product, product_yaml, template_builder, tmpdir, dirpath)
except Exception as exp:
shutil.rmtree(tmpdir, ignore_errors=True)
raise exp
From 9bd6bf1bb5a51a69dc30d4d6eaa2e159c2ac0446 Mon Sep 17 00:00:00 2001
From: Alexander Scheel <alex.scheel@canonical.com>
Date: Wed, 14 Jul 2021 08:23:27 -0400
Subject: [PATCH 11/19] Refactor template_rule_tests into write_* helpers
Signed-off-by: Alexander Scheel <alex.scheel@canonical.com>
---
tests/ssg_test_suite/common.py | 93 ++++++++++++++++++----------------
1 file changed, 50 insertions(+), 43 deletions(-)
diff --git a/tests/ssg_test_suite/common.py b/tests/ssg_test_suite/common.py
index 4e77bd888fe..1f3cad807e6 100644
--- a/tests/ssg_test_suite/common.py
+++ b/tests/ssg_test_suite/common.py
@@ -320,49 +320,27 @@ def load_rule_and_env(rule_dir_path, env_yaml, product=None):
return rule, local_env_yaml
-def template_rule_tests(product, product_yaml, template_builder, tmpdir, dirpath):
- """
- For a given rule directory, templates all contained tests into the output
- (tmpdir) directory.
- """
-
- # Load the rule and its environment
- rule, local_env_yaml = load_rule_and_env(dirpath, product_yaml, product)
-
- # Create the destination directory.
- dest_path = os.path.join(tmpdir, rule.id_)
- os.mkdir(dest_path)
-
- # The priority order is rule-specific tests over templated tests.
- # That is, for any test under rule_id/tests with a name matching a
- # test under shared/templates/<template_name>/tests/, the former
- # will preferred. This means we need to process templates first,
- # so they'll be overwritten later if necessary.
- if rule.template:
- templated_tests = template_builder.get_all_tests(rule.id_, rule.template,
- local_env_yaml)
-
- for relative_path in templated_tests:
- output_path = os.path.join(dest_path, relative_path)
-
- # If there's a separator in the file name, it means we
- # have nested directories to deal with.
- if os.path.sep in relative_path:
- parts = os.path.split(relative_path)[:-1]
- for subdir_index in range(len(parts)):
- # We need to expand all directories in correct
- # order, preserving any previous directories (as
- # they're nested). Use the star operator to splat
- # array parts into arguments to os.path.join(...).
- new_directory = os.path.join(dest_path, *parts[:subdir_index])
- os.mkdir(new_directory)
-
- # Write out the test content to the desired location on
- # disk.
- with open(output_path, 'w') as output_fp:
- test_content = templated_tests[relative_path]
- print(test_content, file=output_fp)
-
+def write_rule_templated_tests(dest_path, relative_path, test_content):
+ output_path = os.path.join(dest_path, relative_path)
+
+ # If there's a separator in the file name, it means we have nested
+ # directories to deal with.
+ if os.path.sep in relative_path:
+ parts = os.path.split(relative_path)[:-1]
+ for subdir_index in range(len(parts)):
+ # We need to expand all directories in the correct order,
+ # preserving any previous directories (as they're nested).
+ # Use the star operator to splat array parts into arguments
+ # to os.path.join(...).
+ new_directory = os.path.join(dest_path, *parts[:subdir_index])
+ os.mkdir(new_directory)
+
+ # Write out the test content to the desired location on disk.
+ with open(output_path, 'w') as output_fp:
+ print(test_content, file=output_fp)
+
+
+def write_rule_dir_tests(local_env_yaml, dest_path, dirpath):
# Walk the test directory, writing all tests into the output
# directory, recursively.
tests_dir_path = os.path.join(dirpath, "tests")
@@ -396,6 +374,35 @@ def template_rule_tests(product, product_yaml, template_builder, tmpdir, dirpath
print(parsed_test, file=output_fp)
+def template_rule_tests(product, product_yaml, template_builder, tmpdir, dirpath):
+ """
+ For a given rule directory, templates all contained tests into the output
+ (tmpdir) directory.
+ """
+
+ # Load the rule and its environment
+ rule, local_env_yaml = load_rule_and_env(dirpath, product_yaml, product)
+
+ # Create the destination directory.
+ dest_path = os.path.join(tmpdir, rule.id_)
+ os.mkdir(dest_path)
+
+ # The priority order is rule-specific tests over templated tests.
+ # That is, for any test under rule_id/tests with a name matching a
+ # test under shared/templates/<template_name>/tests/, the former
+ # will preferred. This means we need to process templates first,
+ # so they'll be overwritten later if necessary.
+ if rule.template:
+ templated_tests = template_builder.get_all_tests(rule.id_, rule.template,
+ local_env_yaml)
+
+ for relative_path in templated_tests:
+ test_content = templated_tests[relative_path]
+ write_rule_templated_tests(dest_path, relative_path, test_content)
+
+ write_rule_dir_tests(local_env_yaml, dest_path, dirpath)
+
+
def template_tests(product=None):
"""
Create a temporary directory with test cases parsed via jinja using
From 5b74ff0aa8ef1b085ef9d6c0148283835a2917f8 Mon Sep 17 00:00:00 2001
From: Alexander Scheel <alex.scheel@canonical.com>
Date: Wed, 14 Jul 2021 08:24:10 -0400
Subject: [PATCH 12/19] Remove unnecessary assertion
Signed-off-by: Alexander Scheel <alex.scheel@canonical.com>
---
tests/ssg_test_suite/common.py | 1 -
1 file changed, 1 deletion(-)
diff --git a/tests/ssg_test_suite/common.py b/tests/ssg_test_suite/common.py
index 1f3cad807e6..9c232dcad02 100644
--- a/tests/ssg_test_suite/common.py
+++ b/tests/ssg_test_suite/common.py
@@ -351,7 +351,6 @@ def write_rule_dir_tests(local_env_yaml, dest_path, dirpath):
# directory. Resolve it to a relative path from the tests/
# directory.
dir_path = _rel_abs_path(os.path.join(dirpath, dirname), tests_dir_path)
- assert '../' not in dir_path
tmp_dir_path = os.path.join(dest_path, dir_path)
os.mkdir(tmp_dir_path)
From 0ab1b95de66d60cf135c985697d6269464777de9 Mon Sep 17 00:00:00 2001
From: Alexander Scheel <alex.scheel@canonical.com>
Date: Wed, 14 Jul 2021 08:25:26 -0400
Subject: [PATCH 13/19] Remove unnecessary _rel_abs_path
As pointed out by Matej, the extra abspath is unnecessary prior to
calling relpath. Remove our helper and switch to calling relpath
directly.
Signed-off-by: Alexander Scheel <alex.scheel@canonical.com>
---
tests/ssg_test_suite/common.py | 15 ++-------------
1 file changed, 2 insertions(+), 13 deletions(-)
diff --git a/tests/ssg_test_suite/common.py b/tests/ssg_test_suite/common.py
index 9c232dcad02..04117359203 100644
--- a/tests/ssg_test_suite/common.py
+++ b/tests/ssg_test_suite/common.py
@@ -253,17 +253,6 @@ def _make_file_root_owned(tarinfo):
return tarinfo
-def _rel_abs_path(current_path, base_path):
- """
- Return the value of the current path, relative to the base path, but
- resolving paths absolutely first. This helps when walking a nested
- directory structure and want to get the subtree relative to the original
- path
- """
- tmp_path = os.path.abspath(current_path)
- return os.path.relpath(current_path, base_path)
-
-
def get_product_context(product=None):
"""
Returns a product YAML context if any product is specified. Hard-coded to
@@ -350,7 +339,7 @@ def write_rule_dir_tests(local_env_yaml, dest_path, dirpath):
# We want to recreate the correct path under the temporary
# directory. Resolve it to a relative path from the tests/
# directory.
- dir_path = _rel_abs_path(os.path.join(dirpath, dirname), tests_dir_path)
+ dir_path = os.path.relpath(os.path.join(dirpath, dirname), tests_dir_path)
tmp_dir_path = os.path.join(dest_path, dir_path)
os.mkdir(tmp_dir_path)
@@ -362,7 +351,7 @@ def write_rule_dir_tests(local_env_yaml, dest_path, dirpath):
# if a file's parent directory doesn't yet exist under the
# destination.
src_test_path = os.path.join(dirpath, filename)
- rel_test_path = _rel_abs_path(src_test_path, tests_dir_path)
+ rel_test_path = os.path.relpath(src_test_path, tests_dir_path)
dest_test_path = os.path.join(dest_path, rel_test_path)
# Rather than performing an OS-level copy, we need to
From 0d2dadf00682efba465b5378803a68893dc62038 Mon Sep 17 00:00:00 2001
From: Alexander Scheel <alex.scheel@canonical.com>
Date: Wed, 14 Jul 2021 14:23:59 -0400
Subject: [PATCH 14/19] Only template rules with variables
The audit_rules_privileged_commands_unix2_chkpwd rule lacks variables on
most products, in addition to its limited prodtype. Because the existing
test harness lacks understanding of prodtype (for including rules in the
tarball), we don't check it yet either. This causes issues when the
template is present but has empty variables on the particular product
(such as when this rule is executed under Ubuntu)
Ultimately we should probably check prodtype in the future, but it
outside the scope of this PR.
Signed-off-by: Alexander Scheel <alex.scheel@canonical.com>
---
tests/ssg_test_suite/common.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/tests/ssg_test_suite/common.py b/tests/ssg_test_suite/common.py
index 04117359203..44105e3b7ae 100644
--- a/tests/ssg_test_suite/common.py
+++ b/tests/ssg_test_suite/common.py
@@ -380,7 +380,7 @@ def template_rule_tests(product, product_yaml, template_builder, tmpdir, dirpath
# test under shared/templates/<template_name>/tests/, the former
# will preferred. This means we need to process templates first,
# so they'll be overwritten later if necessary.
- if rule.template:
+ if rule.template and rule.template['vars']:
templated_tests = template_builder.get_all_tests(rule.id_, rule.template,
local_env_yaml)
@@ -534,7 +534,7 @@ def iterate_over_rules(product=None):
all_tests = dict()
# Start
- if rule.template:
+ if rule.template and rule.template['vars']:
templated_tests = template_builder.get_all_tests(
rule.id_, rule.template, local_env_yaml)
all_tests.update(templated_tests)
From e00262230d86b39d77dc1d75ffce5c048d5a4652 Mon Sep 17 00:00:00 2001
From: Alexander Scheel <alex.scheel@canonical.com>
Date: Mon, 19 Jul 2021 09:06:02 -0400
Subject: [PATCH 15/19] Document new JINJA/Template testing scenarios
Signed-off-by: Alexander Scheel <alex.scheel@canonical.com>
---
tests/README.md | 32 ++++++++++++++++++++++++++++++++
1 file changed, 32 insertions(+)
diff --git a/tests/README.md b/tests/README.md
index 6b5b4497baf..0b3dfabc6f5 100644
--- a/tests/README.md
+++ b/tests/README.md
@@ -147,6 +147,7 @@ the rule that `oscap` should return when the rule is evaluated.
very important to keep this naming form.
For example:
+
* `something.pass.sh`: Success scenario - script is expected to prepare machine
in such way that the rule is expected to pass.
* `something.fail.sh`: Fail scenario - script is expected to break machine so
@@ -200,6 +201,37 @@ Using `platform` and `variables` metadata:
echo "KerberosAuthentication $auth_enabled" >> /etc/ssh/sshd_config
```
+### Augmenting using Jinja macros
+
+Each scenario script is processed under the same jinja context as the
+corresponding OVAL and remediation content. This means that product-specific
+information is known to the scenario scripts at upload time (for example,
+`{{{ grub2_boot_path }}}`), allowing them to work across products. This
+also means Jinja macros such as `{{{ bash_package_install(...) }}}` work to
+install/remove specific packages during the course of testing (such as, if
+it is desired to both install and remove a package in the same scenario for
+the `package_installed` rules).
+
+Note that this does have some limitations: knowledge of the profile (and the
+variables it has and the values they take) is still not provided to the test
+scenario. The above `# profiles` or `# variables` directives will still have
+to be used to add any profile-specific information.
+
+### Augmenting using `shared/templates`
+
+Additionally, we have enabled test scenarios located under the templated
+directory, `shared/templates/.../tests`. Unlike with build-time content,
+`tests` does not need to be located in the template's manifest (at
+`template.yml`). Instead, SSGTS will automatically parse each rule and
+prefer rule-directory-specific test scenarios over any templated scenarios
+that the rule uses. (E.g., if `installed.pass.sh` is present in the
+template `package_installed` and in the `tests/` subdirectory of the rule
+directory, the latter takes precedence over the former).
+
+In addition to the Jinja context described above, the contents of the template
+variables (after processing in `template.py`) are also available to the
+test scenario. This enables template-specific checking.
+
## Example of adding new test scenarios
Let's add test scenarios for rule `accounts_password_minlen_login_defs`.
From c696ebf0001bd6bcced40aafea58cfbc6b870cc1 Mon Sep 17 00:00:00 2001
From: Alexander Scheel <alex.scheel@canonical.com>
Date: Tue, 27 Jul 2021 08:17:47 -0400
Subject: [PATCH 16/19] Correctly handle generation of templated tests
When rules lacked tests inside the rule directory but had them via a
template, SSGTS would ignore them and claim the rule wasn't found. Fix
this bug to allow template-only tests.
Signed-off-by: Alexander Scheel <alex.scheel@canonical.com>
---
tests/ssg_test_suite/common.py | 30 ++++++++++++++++++++++--------
1 file changed, 22 insertions(+), 8 deletions(-)
diff --git a/tests/ssg_test_suite/common.py b/tests/ssg_test_suite/common.py
index 44105e3b7ae..0cb5451e31b 100644
--- a/tests/ssg_test_suite/common.py
+++ b/tests/ssg_test_suite/common.py
@@ -334,6 +334,13 @@ def write_rule_dir_tests(local_env_yaml, dest_path, dirpath):
# directory, recursively.
tests_dir_path = os.path.join(dirpath, "tests")
tests_dir_path = os.path.abspath(tests_dir_path)
+
+ # Note that the tests/ directory may not always exist any more. In
+ # particular, when a rule uses a template, tests may be present there
+ # but not present in the actual rule directory.
+ if not os.path.exists(tests_dir_path):
+ return
+
for dirpath, dirnames, filenames in os.walk(tests_dir_path):
for dirname in dirnames:
# We want to recreate the correct path under the temporary
@@ -420,7 +427,7 @@ def template_tests(product=None):
# /group_a/rule_a/tests/something.pass.sh -> /rule_a/something.pass.sh
for dirpath, dirnames, _ in walk_through_benchmark_dirs(product):
# Skip anything that isn't obviously a rule.
- if "tests" not in dirnames or not is_rule_dir(dirpath):
+ if not is_rule_dir(dirpath):
continue
template_rule_tests(product, product_yaml, template_builder, tmpdir, dirpath)
@@ -519,7 +526,7 @@ def iterate_over_rules(product=None):
_SHARED_TEMPLATES, empty, empty)
for dirpath, dirnames, filenames in walk_through_benchmark_dirs(product):
- if "rule.yml" in filenames and "tests" in dirnames:
+ if is_rule_dir(dirpath):
short_rule_id = os.path.basename(dirpath)
# Load the rule itself to check for a template.
@@ -543,13 +550,14 @@ def iterate_over_rules(product=None):
# like the behavior in template_tests, this will overwrite any
# templated tests with the same file name.
tests_dir = os.path.join(dirpath, "tests")
- tests_dir_files = os.listdir(tests_dir)
- for test_case in tests_dir_files:
- test_path = os.path.join(tests_dir, test_case)
- if os.path.isdir(test_path):
- continue
+ if os.path.exists(tests_dir):
+ tests_dir_files = os.listdir(tests_dir)
+ for test_case in tests_dir_files:
+ test_path = os.path.join(tests_dir, test_case)
+ if os.path.isdir(test_path):
+ continue
- all_tests[test_case] = process_file(test_path, local_env_yaml)
+ all_tests[test_case] = process_file(test_path, local_env_yaml)
# Filter out everything except the shell test scenarios.
# Other files in rule directories are editor swap files
@@ -557,6 +565,12 @@ def iterate_over_rules(product=None):
allowed_scripts = filter(lambda x: x.endswith(".sh"), all_tests)
content_mapping = {x: all_tests[x] for x in allowed_scripts}
+ # Skip any rules that lack any content. This ensures that if we
+ # end up with rules with a template lacking tests and without any
+ # rule directory tests, we don't include the empty rule here.
+ if not content_mapping:
+ continue
+
full_rule_id = OSCAP_RULE + short_rule_id
result = Rule(
directory=tests_dir, id=full_rule_id, short_id=short_rule_id,
From 0661e505c1294a7d5bf6a59813afabff48f31b4c Mon Sep 17 00:00:00 2001
From: Alexander Scheel <alex.scheel@canonical.com>
Date: Tue, 27 Jul 2021 12:23:25 -0400
Subject: [PATCH 17/19] Template tests with matching prodtype
When prodtype is known to the test system, we can skip templating any
rule that has a prodtype that doesn't match. This saves time during
building the bundle and restricts us to only writing tests we care
about and can potentially use.
Note that there might be a mismatch between (datastream, rule): if a
rule is updated on disk but the datastream isn't regenerated, you might
run into weird edge cases where this either over- or under-provisions,
but the same issue would likely occur previously.
Signed-off-by: Alexander Scheel <alex.scheel@canonical.com>
---
tests/ssg_test_suite/common.py | 22 ++++++++++++++++++++++
1 file changed, 22 insertions(+)
diff --git a/tests/ssg_test_suite/common.py b/tests/ssg_test_suite/common.py
index 0cb5451e31b..946d7152af1 100644
--- a/tests/ssg_test_suite/common.py
+++ b/tests/ssg_test_suite/common.py
@@ -378,6 +378,17 @@ def template_rule_tests(product, product_yaml, template_builder, tmpdir, dirpath
# Load the rule and its environment
rule, local_env_yaml = load_rule_and_env(dirpath, product_yaml, product)
+ # Before we get too far, we wish to search the rule YAML to see if
+ # it is applicable to the current product. If we have a product
+ # and the rule isn't applicable for the product, there's no point
+ # in continuing with the rest of the loading. This should speed up
+ # the loading of the templated tests. Note that we've already
+ # parsed the prodtype into local_env_yaml
+ if product and local_env_yaml['products']:
+ prodtypes = local_env_yaml['products']
+ if "all" not in prodtypes and product not in prodtypes:
+ return
+
# Create the destination directory.
dest_path = os.path.join(tmpdir, rule.id_)
os.mkdir(dest_path)
@@ -532,6 +543,17 @@ def iterate_over_rules(product=None):
# Load the rule itself to check for a template.
rule, local_env_yaml = load_rule_and_env(dirpath, product_yaml, product)
+ # Before we get too far, we wish to search the rule YAML to see if
+ # it is applicable to the current product. If we have a product
+ # and the rule isn't applicable for the product, there's no point
+ # in continuing with the rest of the loading. This should speed up
+ # the loading of the templated tests. Note that we've already
+ # parsed the prodtype into local_env_yaml
+ if product and local_env_yaml['products']:
+ prodtypes = local_env_yaml['products']
+ if "all" not in prodtypes and product not in prodtypes:
+ continue
+
# All tests is a mapping from path (in the tarball) to contents
# of the test case. This is necessary because later code (which
# attempts to parse headers from the test case) don't have easy
From f1c0bbd7fb8af5f0af96bf3d442166724fcc7f94 Mon Sep 17 00:00:00 2001
From: Alexander Scheel <alex.scheel@canonical.com>
Date: Tue, 27 Jul 2021 14:51:53 -0400
Subject: [PATCH 18/19] Use process_file_with_macros rather than process_file
Signed-off-by: Alexander Scheel <alex.scheel@canonical.com>
---
tests/ssg_test_suite/common.py | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/tests/ssg_test_suite/common.py b/tests/ssg_test_suite/common.py
index 946d7152af1..291e0f5c9ad 100644
--- a/tests/ssg_test_suite/common.py
+++ b/tests/ssg_test_suite/common.py
@@ -15,7 +15,7 @@
from ssg.constants import MULTI_PLATFORM_MAPPING
from ssg.constants import FULL_NAME_TO_PRODUCT_MAPPING
from ssg.constants import OSCAP_RULE
-from ssg.jinja import process_file
+from ssg.jinja import process_file_with_macros
from ssg.products import product_yaml_path, load_product_yaml
from ssg.rules import get_rule_dir_yaml, is_rule_dir
from ssg.rule_yaml import parse_prodtype
@@ -364,7 +364,7 @@ def write_rule_dir_tests(local_env_yaml, dest_path, dirpath):
# Rather than performing an OS-level copy, we need to
# first parse the test with jinja and then write it back
# out to the destination.
- parsed_test = process_file(src_test_path, local_env_yaml)
+ parsed_test = process_file_with_macros(src_test_path, local_env_yaml)
with open(dest_test_path, 'w') as output_fp:
print(parsed_test, file=output_fp)
@@ -579,7 +579,7 @@ def iterate_over_rules(product=None):
if os.path.isdir(test_path):
continue
- all_tests[test_case] = process_file(test_path, local_env_yaml)
+ all_tests[test_case] = process_file_with_macros(test_path, local_env_yaml)
# Filter out everything except the shell test scenarios.
# Other files in rule directories are editor swap files
From 0f22db3a49bf8ccb14e0ea52e7884a525fa37f6f Mon Sep 17 00:00:00 2001
From: Alexander Scheel <alex.scheel@canonical.com>
Date: Thu, 29 Jul 2021 07:45:41 -0400
Subject: [PATCH 19/19] Add an option to not run duplicate templated tests
Signed-off-by: Alexander Scheel <alex.scheel@canonical.com>
---
tests/ssg_test_suite/combined.py | 4 ++--
tests/ssg_test_suite/common.py | 9 ++++++---
tests/ssg_test_suite/rule.py | 19 +++++++++++++++----
tests/test_suite.py | 10 ++++++++++
4 files changed, 33 insertions(+), 9 deletions(-)
diff --git a/tests/ssg_test_suite/combined.py b/tests/ssg_test_suite/combined.py
index 05270353235..4ef8898f602 100644
--- a/tests/ssg_test_suite/combined.py
+++ b/tests/ssg_test_suite/combined.py
@@ -39,10 +39,10 @@ def __init__(self, test_env):
self.results = list()
self._current_result = None
- def _rule_should_be_tested(self, rule, rules_to_be_tested):
+ def _rule_should_be_tested(self, rule, rules_to_be_tested, tested_templates):
if rule.short_id not in rules_to_be_tested:
return False
- return True
+ return not self._rule_template_been_tested(rule, tested_templates)
def _modify_parameters(self, script, params):
# If there is no profiles metadata in a script we will use
diff --git a/tests/ssg_test_suite/common.py b/tests/ssg_test_suite/common.py
index 291e0f5c9ad..132a004323f 100644
--- a/tests/ssg_test_suite/common.py
+++ b/tests/ssg_test_suite/common.py
@@ -31,7 +31,7 @@
"Scenario_conditions",
("backend", "scanning_mode", "remediated_by", "datastream"))
Rule = namedtuple(
- "Rule", ["directory", "id", "short_id", "files"])
+ "Rule", ["directory", "id", "short_id", "files", "template"])
SSG_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..'))
@@ -542,6 +542,7 @@ def iterate_over_rules(product=None):
# Load the rule itself to check for a template.
rule, local_env_yaml = load_rule_and_env(dirpath, product_yaml, product)
+ template_name = None
# Before we get too far, we wish to search the rule YAML to see if
# it is applicable to the current product. If we have a product
@@ -562,11 +563,13 @@ def iterate_over_rules(product=None):
# templating system.
all_tests = dict()
- # Start
+ # Start by checking for templating tests and provision them if
+ # present.
if rule.template and rule.template['vars']:
templated_tests = template_builder.get_all_tests(
rule.id_, rule.template, local_env_yaml)
all_tests.update(templated_tests)
+ template_name = rule.template['name']
# Add additional tests from the local rule directory. Note that,
# like the behavior in template_tests, this will overwrite any
@@ -596,7 +599,7 @@ def iterate_over_rules(product=None):
full_rule_id = OSCAP_RULE + short_rule_id
result = Rule(
directory=tests_dir, id=full_rule_id, short_id=short_rule_id,
- files=content_mapping)
+ files=content_mapping, template=template_name)
yield result
diff --git a/tests/ssg_test_suite/rule.py b/tests/ssg_test_suite/rule.py
index 5ad7eb8ed27..b707326179f 100644
--- a/tests/ssg_test_suite/rule.py
+++ b/tests/ssg_test_suite/rule.py
@@ -211,13 +211,23 @@ def _final_scan_went_ok(self, runner, rule_id):
logging.error(msg)
return success
- def _rule_should_be_tested(self, rule, rules_to_be_tested):
+ def _rule_template_been_tested(self, rule, tested_templates):
+ if rule.template is None:
+ return False
+ if self.test_env.duplicate_templates:
+ return False
+ if rule.template in tested_templates:
+ return True
+ tested_templates.add(rule.template)
+ return False
+
+ def _rule_should_be_tested(self, rule, rules_to_be_tested, tested_templates):
if 'ALL' in rules_to_be_tested:
# don't select rules that are not present in benchmark
if not xml_operations.find_rule_in_benchmark(
self.datastream, self.benchmark_id, rule.id):
return False
- return True
+ return not self._rule_template_been_tested(rule, tested_templates)
else:
for rule_to_be_tested in rules_to_be_tested:
# we check for a substring
@@ -226,7 +236,7 @@ def _rule_should_be_tested(self, rule, rules_to_be_tested):
else:
pattern = OSCAP_RULE + rule_to_be_tested
if fnmatch.fnmatch(rule.id, pattern):
- return True
+ return not self._rule_template_been_tested(rule, tested_templates)
return False
def _ensure_package_present_for_all_scenarios(self, scenarios_by_rule):
@@ -250,8 +260,9 @@ def _prepare_environment(self, scenarios_by_rule):
def _get_rules_to_test(self, target):
rules_to_test = []
+ tested_templates = set()
for rule in common.iterate_over_rules(self.test_env.product):
- if not self._rule_should_be_tested(rule, target):
+ if not self._rule_should_be_tested(rule, target, tested_templates):
continue
if not xml_operations.find_rule_in_benchmark(
self.datastream, self.benchmark_id, rule.id):
diff --git a/tests/test_suite.py b/tests/test_suite.py
index 00da15329a5..445a53f41d8 100755
--- a/tests/test_suite.py
+++ b/tests/test_suite.py
@@ -116,6 +116,15 @@ def parse_args():
"or remediation done by using remediation roles "
"that are saved to disk beforehand.")
+ common_parser.add_argument(
+ "--duplicate-templates",
+ dest="duplicate_templates",
+ default=False,
+ action="store_true",
+ help="Execute all tests even for tests using shared templates; "
+ "otherwise, executes one test per template type"
+ )
+
subparsers = parser.add_subparsers(dest="subparser_name",
help="Subcommands: profile, rule, combined")
subparsers.required = True
@@ -345,6 +354,7 @@ def normalize_passed_arguments(options):
# Add in product to the test environment. This is independent of actual
# test environment type so we do it after creation.
options.test_env.product = options.product
+ options.test_env.duplicate_templates = options.duplicate_templates
try:
benchmark_cpes = xml_operations.benchmark_get_applicable_platforms(