From 1036c6b55b95a27be57b065a4b9acfecc83639b3 Mon Sep 17 00:00:00 2001

From: Alexander Scheel <alex.scheel@canonical.com>

Date: Tue, 6 Jul 2021 16:02:03 -0400

Subject: [PATCH 01/19] Add tests for package_installed template

Signed-off-by: Alexander Scheel <alex.scheel@canonical.com>

---

 .../package_installed/tests/package-installed-removed.fail.sh | 4 ++++

 .../package_installed/tests/package-installed.pass.sh         | 3 +++

 .../templates/package_installed/tests/package-removed.fail.sh | 3 +++

 3 files changed, 10 insertions(+)

 create mode 100644 shared/templates/package_installed/tests/package-installed-removed.fail.sh

 create mode 100644 shared/templates/package_installed/tests/package-installed.pass.sh

 create mode 100644 shared/templates/package_installed/tests/package-removed.fail.sh

diff --git a/shared/templates/package_installed/tests/package-installed-removed.fail.sh b/shared/templates/package_installed/tests/package-installed-removed.fail.sh

new file mode 100644

index 00000000000..1ce59225303

--- /dev/null

+++ b/shared/templates/package_installed/tests/package-installed-removed.fail.sh

@@ -0,0 +1,4 @@

+#!/bin/bash

+

+{{{ bash_package_install(PKGNAME) }}}

+{{{ bash_package_remove(PKGNAME) }}}

diff --git a/shared/templates/package_installed/tests/package-installed.pass.sh b/shared/templates/package_installed/tests/package-installed.pass.sh

new file mode 100644

index 00000000000..2a0506c57fd

--- /dev/null

+++ b/shared/templates/package_installed/tests/package-installed.pass.sh

@@ -0,0 +1,3 @@

+#!/bin/bash

+

+{{{ bash_package_install(PKGNAME) }}}

diff --git a/shared/templates/package_installed/tests/package-removed.fail.sh b/shared/templates/package_installed/tests/package-removed.fail.sh

new file mode 100644

index 00000000000..c2838396f56

--- /dev/null

+++ b/shared/templates/package_installed/tests/package-removed.fail.sh

@@ -0,0 +1,3 @@

+#!/bin/bash

+

+{{{ bash_package_remove(PKGNAME) }}}

From 3008339e7779b144a2b7fa72854d1185e33438f5 Mon Sep 17 00:00:00 2001

From: Alexander Scheel <alex.scheel@canonical.com>

Date: Tue, 6 Jul 2021 16:16:56 -0400

Subject: [PATCH 02/19] Refactor ssg/templates.py generation (SCE)

This refactors SSG's template generation code to better align between

where we are now upstream and future changes related to templated SCE

support.

At this point in time, we wish to make some changes to allow building

templated (both in the Jinja and shared/templates senses) test cases,

and many of the SCE-enablement changes are relevant, I chose to pull

them in almost entirely. The original commit message is preserved below:

    Refactor template generation for SCEs

    ssg/templates.py holds the core of template generation logic. We

    will need a template_builder for SCE (so we can load SCE metadata

    from templated content) but we do not wish to write this SCE content

    out immediately; instead, we wish to wait until

    build-scripts/build_templated_content.py is run.

    We refactor to make the resolved languages for a rule (the

    intersection between theoretical languages allowed by the rule and

    the actual languages allowed by the template) accessible to callers,

    and, also allow reading just a single templated language artifact

    into memory (instead of from disk).

Notably, this last change is most useful to us; we don't want to put the

file in a template/build-system specified location; we wish to control

its location exactly here.

Note that no changes to ssg/templates.py for the test suite change were

done here; this was purely a change to sync future changes for SCE

content.

Signed-off-by: Alexander Scheel <alex.scheel@canonical.com>

---

 ssg/templates.py | 112 +++++++++++++++++++++++++++++++++++++----------

 1 file changed, 89 insertions(+), 23 deletions(-)

diff --git a/ssg/templates.py b/ssg/templates.py

index 6783d0f2d5e..47a8a5eb852 100644

--- a/ssg/templates.py

+++ b/ssg/templates.py

@@ -107,14 +107,17 @@ def __init__(

         self.checks_dir = checks_dir

         self.output_dirs = dict()

         for lang in languages:

+            lang_dir = lang

             if lang == "oval":

                 # OVAL checks need to be put to a different directory because

-                # they are processed differently than remediations later in the

-                # build process

+                # they are processed differently than remediations later in

+                # the build process

                 output_dir = self.checks_dir

+                if lang.startswith("sce-"):

+                    lang_dir = "sce"

             else:

                 output_dir = self.remediations_dir

-            dir_ = os.path.join(output_dir, lang)

+            dir_ = os.path.join(output_dir, lang_dir)

             self.output_dirs[lang] = dir_

         # scan directory structure and dynamically create list of templates

         for item in sorted(os.listdir(self.templates_dir)):

@@ -124,25 +127,41 @@ def __init__(

                 maybe_template.load()

                 templates[item] = maybe_template

+    def build_lang_file(

+            self, rule_id, template_name, template_vars, lang, local_env_yaml):

+        """

+        Builds and returns templated content for a given rule for a given

+        language; does not write the output to disk.

+        """

+        if lang not in templates[template_name].langs:

+            return None

+

+        template_file_name = lang + ".template"

+        template_file_path = os.path.join(self.templates_dir, template_name, template_file_name)

+        template_parameters = templates[template_name].preprocess(template_vars, lang)

+        jinja_dict = ssg.utils.merge_dicts(local_env_yaml, template_parameters)

+        filled_template = ssg.jinja.process_file_with_macros(

+            template_file_path, jinja_dict)

+

+        return filled_template

     def build_lang(

-            self, rule_id, template_name, template_vars, lang, local_env_yaml):

+            self, rule_id, template_name, template_vars, lang, local_env_yaml, platforms=None):

         """

         Builds templated content for a given rule for a given language.

         Writes the output to the correct build directories.

         """

         if lang not in templates[template_name].langs:

             return

-        template_file_name = lang + ".template"

-        template_file_path = os.path.join(self.templates_dir, template_name, template_file_name)

+

+        filled_template = self.build_lang_file(rule_id, template_name,

+            template_vars, lang, local_env_yaml)

+

         ext = lang_to_ext_map[lang]

         output_file_name = rule_id + ext

         output_filepath = os.path.join(

             self.output_dirs[lang], output_file_name)

-        template_parameters = templates[template_name].preprocess(template_vars, lang)

-        jinja_dict = ssg.utils.merge_dicts(local_env_yaml, template_parameters)

-        filled_template = ssg.jinja.process_file_with_macros(

-            template_file_path, jinja_dict)

+

         with open(output_filepath, "w") as f:

             f.write(filled_template)

@@ -168,6 +187,37 @@ def get_langs_to_generate(self, rule):

         else:

             return languages

+    def get_template_name(self, template):

+        """

+        Given a template dictionary from a Rule instance, determine the name

+        of the template (from templates) this rule uses.

+        """

+        try:

+            template_name = template["name"]

+        except KeyError:

+            raise ValueError(

+                "Rule {0} is missing template name under template key".format(

+                    rule_id))

+        if template_name not in templates.keys():

+            raise ValueError(

+                "Rule {0} uses template {1} which does not exist.".format(

+                    rule_id, template_name))

+        return template_name

+

+    def get_resolved_langs_to_generate(self, rule):

+        """

+        Given a specific Rule instance, determine which languages are

+        generated by the combination of the rule's template_backends AND

+        the rule's template keys.

+        """

+        if rule.template is None:

+            return None

+

+        rule_langs = set(self.get_langs_to_generate(rule))

+        template_name = self.get_template_name(rule.template)

+        template_langs = set(templates[template_name].langs)

+        return rule_langs.intersection(template_langs)

+

     def process_product_vars(self, all_variables):

         """

         Given a dictionary with the format key[@<product>]=value, filter out

@@ -183,21 +233,12 @@ def process_product_vars(self, all_variables):

         return processed

-    def build_rule(self, rule_id, rule_title, template, langs_to_generate):

+    def build_rule(self, rule_id, rule_title, template, langs_to_generate, platforms=None):

         """

         Builds templated content for a given rule for selected languages,

         writing the output to the correct build directories.

         """

-        try:

-            template_name = template["name"]

-        except KeyError:

-            raise ValueError(

-                "Rule {0} is missing template name under template key".format(

-                    rule_id))

-        if template_name not in templates.keys():

-            raise ValueError(

-                "Rule {0} uses template {1} which does not exist.".format(

-                    rule_id, template_name))

+        template_name = self.get_template_name(template)

         try:

             template_vars = self.process_product_vars(template["vars"])

         except KeyError:

@@ -216,11 +257,36 @@ def build_rule(self, rule_id, rule_title, template, langs_to_generate):

         for lang in langs_to_generate:

             try:

                 self.build_lang(

-                    rule_id, template_name, template_vars, lang, local_env_yaml)

+                    rule_id, template_name, template_vars, lang, local_env_yaml, platforms)

             except Exception as e:

                 print("Error building templated {0} content for rule {1}".format(lang, rule_id), file=sys.stderr)

                 raise e

+    def get_lang_for_rule(self, rule_id, rule_title, template, language):

+        """

+        For the specified rule, build and return only the specified language

+        content.

+        """

+        template_name = self.get_template_name(template)

+        try:

+            template_vars = self.process_product_vars(template["vars"])

+        except KeyError:

+            raise ValueError(

+                "Rule {0} does not contain mandatory 'vars:' key under "

+                "'template:' key.".format(rule_id))

+        # Add the rule ID which will be reused in OVAL templates as OVAL

+        # definition ID so that the build system matches the generated

+        # check with the rule.

+        template_vars["_rule_id"] = rule_id

+        # checks and remediations are processed with a custom YAML dict

+        local_env_yaml = self.env_yaml.copy()

+        local_env_yaml["rule_id"] = rule_id

+        local_env_yaml["rule_title"] = rule_title

+        local_env_yaml["products"] = self.env_yaml["product"]

+

+        return self.build_lang_file(rule_id, template_name, template_vars,

+            language, local_env_yaml)

+

     def build_extra_ovals(self):

         declaration_path = os.path.join(self.templates_dir, "extra_ovals.yml")

         declaration = ssg.yaml.open_raw(declaration_path)

@@ -245,7 +311,7 @@ def build_all_rules(self):

                 continue

             langs_to_generate = self.get_langs_to_generate(rule)

             self.build_rule(

-                rule.id_, rule.title, rule.template, langs_to_generate)

+                rule.id_, rule.title, rule.template, langs_to_generate, platforms=rule.platforms)

     def build(self):

         """

From 8da04c6a85d40432a19904b16251885e12b75999 Mon Sep 17 00:00:00 2001

From: Alexander Scheel <alex.scheel@canonical.com>

Date: Tue, 6 Jul 2021 16:50:06 -0400

Subject: [PATCH 03/19] Remove incorrect tests for package installation

These tests assume that the package manager is yum (and forcibly

installs it) in some cases. This doesn't work on Debian-like systems;

prefer the templated version instead, which uses the

bash_package_install Jinja macro instead (and uses the

bash_package_remove macro for the negative case)

Signed-off-by: Alexander Scheel <alex.scheel@canonical.com>

---

 .../ntp/package_chrony_installed/tests/installed.pass.sh  | 4 ----

 .../ntp/package_chrony_installed/tests/removed.fail.sh    | 3 ---

 .../package_audit_installed/tests/installed.pass.sh       | 4 ----

 .../package_audit_installed/tests/removed.fail.sh         | 3 ---

 .../package_rsyslog_installed/tests/installed.pass.sh     | 3 ---

 .../package_rsyslog_installed/tests/notinstalled.fail.sh  | 3 ---

 .../package_firewalld_installed/tests/installed.pass.sh   | 4 ----

 .../package_firewalld_installed/tests/removed.fail.sh     | 3 ---

 .../package_libselinux_installed/tests/installed.pass.sh  | 4 ----

 .../aide/package_aide_installed/tests/installed.pass.sh        | 3 ---

 .../aide/package_aide_installed/tests/notinstalled.fail.sh     | 3 ---

 .../sudo/package_sudo_installed/tests/installed.pass.sh   | 4 ----

 .../sudo/package_sudo_installed/tests/removed.fail.sh     | 3 ---

 13 files changed, 53 deletions(-)

 delete mode 100644 linux_os/guide/services/ntp/package_chrony_installed/tests/installed.pass.sh

 delete mode 100644 linux_os/guide/services/ntp/package_chrony_installed/tests/removed.fail.sh

 delete mode 100644 linux_os/guide/system/auditing/package_audit_installed/tests/installed.pass.sh

 delete mode 100644 linux_os/guide/system/auditing/package_audit_installed/tests/removed.fail.sh

 delete mode 100644 linux_os/guide/system/logging/package_rsyslog_installed/tests/installed.pass.sh

 delete mode 100644 linux_os/guide/system/logging/package_rsyslog_installed/tests/notinstalled.fail.sh

 delete mode 100644 linux_os/guide/system/network/network-firewalld/firewalld_activation/package_firewalld_installed/tests/installed.pass.sh

 delete mode 100644 linux_os/guide/system/network/network-firewalld/firewalld_activation/package_firewalld_installed/tests/removed.fail.sh

 delete mode 100644 linux_os/guide/system/selinux/package_libselinux_installed/tests/installed.pass.sh

 delete mode 100644 linux_os/guide/system/software/integrity/software-integrity/aide/package_aide_installed/tests/installed.pass.sh

 delete mode 100644 linux_os/guide/system/software/integrity/software-integrity/aide/package_aide_installed/tests/notinstalled.fail.sh

 delete mode 100644 linux_os/guide/system/software/sudo/package_sudo_installed/tests/installed.pass.sh

 delete mode 100644 linux_os/guide/system/software/sudo/package_sudo_installed/tests/removed.fail.sh

diff --git a/linux_os/guide/services/ntp/package_chrony_installed/tests/installed.pass.sh b/linux_os/guide/services/ntp/package_chrony_installed/tests/installed.pass.sh

deleted file mode 100644

index c692df2b282..00000000000

--- a/linux_os/guide/services/ntp/package_chrony_installed/tests/installed.pass.sh

+++ /dev/null

@@ -1,4 +0,0 @@

-#!/bin/bash

-# package = yum

-

-yum install -y chrony

diff --git a/linux_os/guide/services/ntp/package_chrony_installed/tests/removed.fail.sh b/linux_os/guide/services/ntp/package_chrony_installed/tests/removed.fail.sh

deleted file mode 100644

index eba551f99d4..00000000000

--- a/linux_os/guide/services/ntp/package_chrony_installed/tests/removed.fail.sh

+++ /dev/null

@@ -1,3 +0,0 @@

-#!/bin/bash

-

-yum remove -y chrony

diff --git a/linux_os/guide/system/auditing/package_audit_installed/tests/installed.pass.sh b/linux_os/guide/system/auditing/package_audit_installed/tests/installed.pass.sh

deleted file mode 100644

index 1c19ce31a87..00000000000

--- a/linux_os/guide/system/auditing/package_audit_installed/tests/installed.pass.sh

+++ /dev/null

@@ -1,4 +0,0 @@

-#!/bin/bash

-# package = yum

-

-yum install -y audit

diff --git a/linux_os/guide/system/auditing/package_audit_installed/tests/removed.fail.sh b/linux_os/guide/system/auditing/package_audit_installed/tests/removed.fail.sh

deleted file mode 100644

index 45dd05a9638..00000000000

--- a/linux_os/guide/system/auditing/package_audit_installed/tests/removed.fail.sh

+++ /dev/null

@@ -1,3 +0,0 @@

-#!/bin/bash

-

-yum remove -y audit

diff --git a/linux_os/guide/system/logging/package_rsyslog_installed/tests/installed.pass.sh b/linux_os/guide/system/logging/package_rsyslog_installed/tests/installed.pass.sh

deleted file mode 100644

index 6e4107fa3bd..00000000000

--- a/linux_os/guide/system/logging/package_rsyslog_installed/tests/installed.pass.sh

+++ /dev/null

@@ -1,3 +0,0 @@

-#!/bin/bash

-# packages = rsyslog

-

diff --git a/linux_os/guide/system/logging/package_rsyslog_installed/tests/notinstalled.fail.sh b/linux_os/guide/system/logging/package_rsyslog_installed/tests/notinstalled.fail.sh

deleted file mode 100644

index f64bf1b340a..00000000000

--- a/linux_os/guide/system/logging/package_rsyslog_installed/tests/notinstalled.fail.sh

+++ /dev/null

@@ -1,3 +0,0 @@

-#!/bin/bash

-

-yum remove -y rsyslog

diff --git a/linux_os/guide/system/network/network-firewalld/firewalld_activation/package_firewalld_installed/tests/installed.pass.sh b/linux_os/guide/system/network/network-firewalld/firewalld_activation/package_firewalld_installed/tests/installed.pass.sh

deleted file mode 100644

index 7a4748863bf..00000000000

--- a/linux_os/guide/system/network/network-firewalld/firewalld_activation/package_firewalld_installed/tests/installed.pass.sh

+++ /dev/null

@@ -1,4 +0,0 @@

-#!/bin/bash

-# package = yum

-

-yum install -y firewalld

diff --git a/linux_os/guide/system/network/network-firewalld/firewalld_activation/package_firewalld_installed/tests/removed.fail.sh b/linux_os/guide/system/network/network-firewalld/firewalld_activation/package_firewalld_installed/tests/removed.fail.sh

deleted file mode 100644

index a9107df7de8..00000000000

--- a/linux_os/guide/system/network/network-firewalld/firewalld_activation/package_firewalld_installed/tests/removed.fail.sh

+++ /dev/null

@@ -1,3 +0,0 @@

-#!/bin/bash

-

-yum remove -y firewalld

diff --git a/linux_os/guide/system/selinux/package_libselinux_installed/tests/installed.pass.sh b/linux_os/guide/system/selinux/package_libselinux_installed/tests/installed.pass.sh

deleted file mode 100644

index 5d30cf77841..00000000000

--- a/linux_os/guide/system/selinux/package_libselinux_installed/tests/installed.pass.sh

+++ /dev/null

@@ -1,4 +0,0 @@

-#!/bin/bash

-# package = yum

-

-yum install -y libselinux

diff --git a/linux_os/guide/system/software/integrity/software-integrity/aide/package_aide_installed/tests/installed.pass.sh b/linux_os/guide/system/software/integrity/software-integrity/aide/package_aide_installed/tests/installed.pass.sh

deleted file mode 100644

index fa8b85b..0000000

--- a/linux_os/guide/system/software/integrity/software-integrity/aide/package_aide_installed/tests/installed.pass.sh

+++ /dev/null

@@ -1,3 +0,0 @@

-#!/bin/bash

-# packages = aide

-

diff --git a/linux_os/guide/system/software/integrity/software-integrity/aide/package_aide_installed/tests/notinstalled.fail.sh b/linux_os/guide/system/software/integrity/software-integrity/aide/package_aide_installed/tests/notinstalled.fail.sh

deleted file mode 100644

index 75978b6..0000000

--- a/linux_os/guide/system/software/integrity/software-integrity/aide/package_aide_installed/tests/notinstalled.fail.sh

+++ /dev/null

@@ -1,3 +0,0 @@

-#!/bin/bash

-

-yum remove -y aide

diff --git a/linux_os/guide/system/software/sudo/package_sudo_installed/tests/installed.pass.sh b/linux_os/guide/system/software/sudo/package_sudo_installed/tests/installed.pass.sh

deleted file mode 100644

index dafc6998a9a..00000000000

--- a/linux_os/guide/system/software/sudo/package_sudo_installed/tests/installed.pass.sh

+++ /dev/null

@@ -1,4 +0,0 @@

-#!/bin/bash

-# package = yum

-

-yum install -y sudo

diff --git a/linux_os/guide/system/software/sudo/package_sudo_installed/tests/removed.fail.sh b/linux_os/guide/system/software/sudo/package_sudo_installed/tests/removed.fail.sh

deleted file mode 100644

index d22b562c15e..00000000000

--- a/linux_os/guide/system/software/sudo/package_sudo_installed/tests/removed.fail.sh

+++ /dev/null

@@ -1,3 +0,0 @@

-#!/bin/bash

-

-rpm -e --nodeps sudo

From 6c49dcb79aaf5b239f09ecfdac07b443736b5b5e Mon Sep 17 00:00:00 2001

From: Alexander Scheel <alex.scheel@canonical.com>

Date: Tue, 6 Jul 2021 16:51:05 -0400

Subject: [PATCH 04/19] Support templating test content

We introduced a new method to the template builder in ssg/templates.py

for finding (and building) test content under shared/templates/...

This uses Jinja macros and has the full context of the template present,

so for instance, package_installed tests can use the correct package

name on the target platform.

Signed-off-by: Alexander Scheel <alex.scheel@canonical.com>

---

 ssg/templates.py | 46 ++++++++++++++++++++++++++++++++++++++++++++++

 1 file changed, 46 insertions(+)

diff --git a/ssg/templates.py b/ssg/templates.py

index 47a8a5eb852..0a39c3ba094 100644

--- a/ssg/templates.py

+++ b/ssg/templates.py

@@ -145,6 +145,52 @@ def build_lang_file(

         return filled_template

+    def get_all_tests(

+            self, rule_id, rule_template, local_env_yaml, platforms=None):

+        """

+        Builds a dictionary of a test case path -> test case value mapping.

+

+        Here, we want to know what the relative path on disk (under the tests/

+        subdirectory) is (such as "installed.pass.sh"), along with the actual

+        contents of the test case.

+

+        Presumably, we'll find the test case we want (all of them when

+        building a test case tarball) and write them to disk in the

+        appropriate location.

+        """

+        template_name = rule_template['name']

+        template_vars = rule_template['vars']

+

+        base_dir = os.path.abspath(os.path.join(self.templates_dir, template_name, "tests"))

+        results = dict()

+

+        # If no test cases exist, return an empty dictionary.

+        if not os.path.exists(base_dir):

+            return results

+

+        # Walk files; note that we don't need to do anything about directories

+        # as only files are recorded in the mapping; directories can be

+        # inferred from the path.

+        for dirpath, _, filenames in os.walk(base_dir):

+            if not filenames:

+                continue

+

+            for filename in filenames:

+                # Relative path to the file becomes our results key.

+                absolute_path = os.path.abspath(os.path.join(dirpath, filename))

+                relative_path = os.path.relpath(absolute_path, base_dir)

+

+                # Load template parameters and apply it to the test case.

+                template_parameters = templates[template_name].preprocess(template_vars, "tests")

+                jinja_dict = ssg.utils.merge_dicts(local_env_yaml, template_parameters)

+                filled_template = ssg.jinja.process_file_with_macros(

+                    absolute_path, jinja_dict)

+

+                # Save the results under the relative path.

+                results[relative_path] = filled_template

+

+        return results

+

     def build_lang(

             self, rule_id, template_name, template_vars, lang, local_env_yaml, platforms=None):

         """

From 4bd5061ffbb524b09975982fbd6dae26f35770c3 Mon Sep 17 00:00:00 2001

From: Alexander Scheel <alex.scheel@canonical.com>

Date: Tue, 6 Jul 2021 16:53:44 -0400

Subject: [PATCH 05/19] Add templated tests into templated directory

This extends the existing templated directory generation to also include

tests under shared/templates/.../tests. The complicated portion is in

subdirectory handling: we need to ensure we create all necessary nested

subdirectories that are implicitly implied by the relative paths. This

is a shortcoming in the get_all_tests's return format.

Signed-off-by: Alexander Scheel <alex.scheel@canonical.com>

---

 tests/ssg_test_suite/common.py | 43 ++++++++++++++++++++++++++++++++++

 1 file changed, 43 insertions(+)

diff --git a/tests/ssg_test_suite/common.py b/tests/ssg_test_suite/common.py

index 3dbeaf304a4..2bd4b2bd1c7 100644

--- a/tests/ssg_test_suite/common.py

+++ b/tests/ssg_test_suite/common.py

@@ -21,6 +21,9 @@

 from ssg.rule_yaml import parse_prodtype

 from ssg_test_suite.log import LogHelper

+import ssg.templates

+

+

 Scenario_run = namedtuple(

     "Scenario_run",

     ("rule_id", "script"))

@@ -39,6 +42,8 @@

 _SHARED_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), '../shared'))

+_SHARED_TEMPLATES = os.path.abspath(os.path.join(SSG_ROOT, 'shared/templates'))

+

 REMOTE_USER = "root"

 REMOTE_USER_HOME_DIRECTORY = "/root"

 REMOTE_TEST_SCENARIOS_DIRECTORY = os.path.join(REMOTE_USER_HOME_DIRECTORY, "ssgts")

@@ -278,6 +283,11 @@ def template_tests(product=None):

             yaml_path = product_yaml_path(SSG_ROOT, product)

             product_yaml = load_product_yaml(yaml_path)

+        # Initialize a mock template_builder.

+        empty = "/ssgts/empty/placeholder"

+        template_builder = ssg.templates.Builder(product_yaml, empty,

+            _SHARED_TEMPLATES, empty, empty)

+

         # Below we could run into a DocumentationNotComplete error. However,

         # because the test suite isn't executed in the context of a particular

         # build (though, ideally it would be linked), we may not know exactly

@@ -299,8 +309,11 @@ def template_tests(product=None):

             # Load rule content in our environment. We use this to satisfy

             # some implied properties that might be used in the test suite.

+            # Make sure we normalize to a specific product as well so that

+            # when we load templated content it is correct.

             rule_path = get_rule_dir_yaml(dirpath)

             rule = RuleYAML.from_yaml(rule_path, product_yaml)

+            rule.normalize(product)

             # Note that most places would check prodtype, but we don't care

             # about that here: if the rule is available to the product, we

@@ -320,6 +333,36 @@ def template_tests(product=None):

             dest_path = os.path.join(tmpdir, rule.id_)

             os.mkdir(dest_path)

+            # The priority order is rule-specific tests over templated tests.

+            # That is, for any test under rule_id/tests with a name matching a

+            # test under shared/templates/<template_name>/tests/, the former

+            # will preferred. This means we need to process templates first,

+            # so they'll be overwritten later if necessary.

+            if rule.template:

+                templated_tests = template_builder.get_all_tests(

+                    rule.id_, rule.template, local_env_yaml)

+

+                for relative_path in templated_tests:

+                    output_path = os.path.join(dest_path, relative_path)

+

+                    # If there's a separator in the file name, it means we

+                    # have nested directories to deal with.

+                    if os.path.sep in relative_path:

+                        parts = os.path.split(relative_path)[:-1]

+                        for subdir_index in range(len(parts)):

+                            # We need to expand all directories in correct

+                            # order, preserving any previous directories (as

+                            # they're nested). Use the star operator to splat

+                            # array parts into arguments to os.path.join(...).

+                            new_directory = os.path.join(dest_path, *parts[:subdir_index])

+                            os.mkdir(new_directory)

+

+                    # Write out the test content to the desired location on

+                    # disk.

+                    with open(output_path, 'w') as output_fp:

+                        test_content = templated_tests[relative_path]

+                        print(test_content, file=output_fp)

+

             # Walk the test directory, writing all tests into the output

             # directory, recursively.

             tests_dir_path = os.path.join(dirpath, "tests")

From 013b247c1be897bd12ae4cf2bc2443279b1a52c9 Mon Sep 17 00:00:00 2001

From: Alexander Scheel <alex.scheel@canonical.com>

Date: Wed, 7 Jul 2021 07:00:37 -0400

Subject: [PATCH 06/19] Teach iterate_over_rules about templated tests

There are two parts to rule-based test execution:

 1. Building the tarball of tests and shipping it over to the remote

    machine.

 2. Iterating over these tests to see which scenarios are actually

    applicable and any other metadata actions we need to take.

In particular, item two is controlled by iterate_over_rules. Here we

need to:

 1. Teach it about the templating system.

 2. Switch it from returning a list of test cases to a dictionary

    mapping test case -> contents. This allows us to contain the

    templating system to common.py only and not leak it into other parts

    of the test suite.

 3. Do a little bit of refactoring to contain shared code between

    iterate_over_rules and template_tests.

Signed-off-by: Alexander Scheel <alex.scheel@canonical.com>

---

 tests/ssg_test_suite/common.py | 146 ++++++++++++++++++++++++---------

 1 file changed, 108 insertions(+), 38 deletions(-)

diff --git a/tests/ssg_test_suite/common.py b/tests/ssg_test_suite/common.py

index 2bd4b2bd1c7..b357f816027 100644

--- a/tests/ssg_test_suite/common.py

+++ b/tests/ssg_test_suite/common.py

@@ -264,6 +264,62 @@ def _rel_abs_path(current_path, base_path):

     return os.path.relpath(current_path, base_path)

+def get_product_context(product=None):

+    """

+    Returns a product YAML context if any product is specified. Hard-coded to

+    assume a debug build.

+    """

+    # Load product's YAML file if present. This will allow us to parse

+    # tests in the context of the product we're executing under.

+    product_yaml = dict()

+    if product:

+        yaml_path = product_yaml_path(SSG_ROOT, product)

+        product_yaml = load_product_yaml(yaml_path)

+

+    # We could run into a DocumentationNotComplete error when loading a

+    # rule's YAML contents. However, because the test suite isn't executed

+    # in the context of a particular build (though, ideally it would be

+    # linked), we may not know exactly whether the top-level rule/profile

+    # we're testing is actually completed. Thus, forcibly set the required

+    # property to bypass this error.

+    product_yaml['cmake_build_type'] = 'Debug'

+

+    return product_yaml

+

+

+def load_rule_and_env(rule_dir_path, env_yaml, product=None):

+    """

+    Loads a rule and returns the combination of the RuleYAML class and

+    the corresponding local environment for that rule.

+    """

+

+    # First build the path to the rule.yml file

+    rule_path = get_rule_dir_yaml(rule_dir_path)

+

+    # Load rule content in our environment. We use this to satisfy

+    # some implied properties that might be used in the test suite.

+    # Make sure we normalize to a specific product as well so that

+    # when we load templated content it is correct.

+    rule = RuleYAML.from_yaml(rule_path, env_yaml)

+    rule.normalize(product)

+

+    # Note that most places would check prodtype, but we don't care

+    # about that here: if the rule is available to the product, we

+    # load and parse it anyways as we have no knowledge of the

+    # top-level profile or rule passed into the test suite.

+    prodtypes = parse_prodtype(rule.prodtype)

+

+    # Our local copy of env_yaml needs some properties from rule.yml

+    # for completeness.

+    local_env_yaml = dict()

+    local_env_yaml.update(env_yaml)

+    local_env_yaml['rule_id'] = rule.id_

+    local_env_yaml['rule_title'] = rule.title

+    local_env_yaml['products'] = prodtypes

+

+    return rule, local_env_yaml

+

+

 def template_tests(product=None):

     """

     Create a temporary directory with test cases parsed via jinja using

@@ -276,26 +332,14 @@ def template_tests(product=None):

     # it on success. Wrap in a try/except block and reraise the original

     # exception after removing the temporary directory.

     try:

-        # Load product's YAML file if present. This will allow us to parse

-        # tests in the context of the product we're executing under.

-        product_yaml = dict()

-        if product:

-            yaml_path = product_yaml_path(SSG_ROOT, product)

-            product_yaml = load_product_yaml(yaml_path)

+        # Load the product context we're executing under, if any.

+        product_yaml = get_product_context(product)

         # Initialize a mock template_builder.

         empty = "/ssgts/empty/placeholder"

         template_builder = ssg.templates.Builder(product_yaml, empty,