From d65df550e2949cd4c100dafba07f534c762e983d Mon Sep 17 00:00:00 2001 From: Adam Saleh Date: Nov 07 2022 13:46:55 +0000 Subject: Merge pull request #82 from AdamSaleh/fixes_nd_runner Fixes for Stream 9 tests and a duffy runner --- diff --git a/.github/workflows/centos-stream8.yml b/.github/workflows/centos-stream8.yml index 1a75031..3ea65e3 100644 --- a/.github/workflows/centos-stream8.yml +++ b/.github/workflows/centos-stream8.yml @@ -3,9 +3,18 @@ jobs: name: test_0 runs-on: stream-8 steps: + - name: Install SSH key + uses: shimataro/ssh-key-action@v2 + with: + key: ${{ secrets.SSH_KEY }} + name: id_rsa # optional + known_hosts: NO + config: ${{ secrets.CONFIG }} # ssh_config; optional + if_key_exists: fail # replace / ignore / fail; optional (defaults to fail) - uses: "actions/checkout@v2" - run: sudo dnf -y install python3-pip - - run: sudo bash runtests.sh + - run: sudo python3 -m pip install duffy fabric + - run: cd t_functional_duffy_runner/src && python3 __main__.py --arch=x86 --release=centos-8s name: Greeting on: diff --git a/t_functional_duffy_runner/.gitignore b/t_functional_duffy_runner/.gitignore new file mode 100644 index 0000000..1dbbe6f --- /dev/null +++ b/t_functional_duffy_runner/.gitignore @@ -0,0 +1,4 @@ +# Distribution / packaging +build/ +dist/ +*.egg-info/ diff --git a/t_functional_duffy_runner/README.md b/t_functional_duffy_runner/README.md new file mode 100644 index 0000000..85e570c --- /dev/null +++ b/t_functional_duffy_runner/README.md @@ -0,0 +1,3 @@ +# t_functional duffy runner + +Simple script to run these tests on new duffy. diff --git a/t_functional_duffy_runner/pyproject.toml b/t_functional_duffy_runner/pyproject.toml new file mode 100644 index 0000000..9787c3b --- /dev/null +++ b/t_functional_duffy_runner/pyproject.toml @@ -0,0 +1,3 @@ +[build-system] +requires = ["setuptools", "wheel"] +build-backend = "setuptools.build_meta" diff --git a/t_functional_duffy_runner/requirements.txt b/t_functional_duffy_runner/requirements.txt new file mode 100644 index 0000000..6d61d41 --- /dev/null +++ b/t_functional_duffy_runner/requirements.txt @@ -0,0 +1,2 @@ +duffy +fabric diff --git a/t_functional_duffy_runner/setup.cfg b/t_functional_duffy_runner/setup.cfg new file mode 100644 index 0000000..96c7c99 --- /dev/null +++ b/t_functional_duffy_runner/setup.cfg @@ -0,0 +1,10 @@ +[metadata] +name = t_functional_duffy_runner +version = 0.0.1 + +[options] +packages = src + +[options.entry_points] +console_scripts = + app = src.cli:main diff --git a/t_functional_duffy_runner/setup.py b/t_functional_duffy_runner/setup.py new file mode 100644 index 0000000..6068493 --- /dev/null +++ b/t_functional_duffy_runner/setup.py @@ -0,0 +1,3 @@ +from setuptools import setup + +setup() diff --git a/t_functional_duffy_runner/src/__init__.py b/t_functional_duffy_runner/src/__init__.py new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/t_functional_duffy_runner/src/__init__.py diff --git a/t_functional_duffy_runner/src/__main__.py b/t_functional_duffy_runner/src/__main__.py new file mode 100644 index 0000000..6403ad8 --- /dev/null +++ b/t_functional_duffy_runner/src/__main__.py @@ -0,0 +1,140 @@ +from argparse import ArgumentParser +import io +import os +import tempfile + +from duffy.client import DuffyClient +from fabric import Connection +from invoke import run as local + +class DuffyWrapper: + def __init__(self, auth_name, auth_key): + self.c = DuffyClient(url="https://duffy.ci.centos.org/api/v1", auth_name=auth_name, auth_key=auth_key) + self.last_session = None + + def get_hostnames(self, *query): + if not self.last_session: + self.request_session(*query) + + nodes = [n for n in self.last_session.session.nodes + if all(q in n.pool for q in query)] + return [n + for n in [n.data.get('provision',{}).get('public_hostname',None) for n in nodes] + if n] + + def find_hostnames(self, *query): + ls = self.c.list_sessions() + nodes = [n for s in ls.sessions for n in s.nodes + if all(q in n.pool for q in query)] + return [n + for n in [n.data.get('provision',{}).get('public_hostname',None) for n in nodes] + if n] + + def find_pool_name(self, *query): + return [p.name for p in self.c.list_pools().pools if all(q in p.name for q in query)] + + def request_session(self, *query): + pool = self.find_pool_name(*query) + session = self.c.request_session([{"pool":pool[0], "quantity":"1"}]) + self.last_session = session + return session + +class TmuxWrapper: + def __init__(self, host, session='default-session', **rest): + self.host = host + self.c = Connection(host, **rest) + self.c.run("dnf -y install git tmux") + self.ensure_session(session) + + def ensure_session(self,session='default-session'): + return self.c.run("tmux has-session -t {session} || tmux new -s {session} -d".format(session=session)) + + def run(self, cmd, session='default-session'): + return self.c.run("tmux send-keys -t {session}:0 '{cmd}' ENTER".format(session=session, cmd=cmd)) + + def run_and_notify(self, cmd, session='default-session'): + return self.c.run("tmux send-keys -t {session}:0 '{cmd}; tmux wait-for -S {session}' ENTER".format(session=session, cmd=cmd)) + + def wait_for(self, session='default-session'): + return self.c.run("tmux wait-for {session}".format(session=session)) + + def send_folder(self, path, remote): + with tempfile.NamedTemporaryFile() as t: + local("tar cf {to} -C {path} .".format(path=path, to=t.name)) + print(t.name) + self.c.run("mkdir -p {}".format(remote)) + self.c.put(local=t.name, remote=remote) + self.c.run("cd {path} && tar xf {name}".format(path=remote, name=t.name.split('/')[-1])) + def get_pane(self): + return tmux.c.run("tmux capture-pane -p -t default-session") + def set_repo(self, baseurl="https://composes.stream.centos.org/development/latest-CentOS-Stream/compose/"): + repo = io.StringIO(""" +[baseos-compose] +name=CentOS Stream $releasever - BaseOS +baseurl={baseurl}/BaseOS/$arch/os/ +gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-centosofficial +gpgcheck=0 +repo_gpgcheck=0 +metadata_expire=6h +countme=1 +enabled=1 + +[appstream-compose] +name=CentOS Stream $releasever - AppStream +baseurl={baseurl}/AppStream/$arch/os/ +gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-centosofficial +gpgcheck=0 +repo_gpgcheck=0 +metadata_expire=6h +countme=1 +enabled=1 + """.format(baseurl=baseurl)) + self.c.put(repo, "/tmp/compose.repo") + self.c.run("dnf repolist | awk '(NR>1) {print $1}' | xargs dnf config-manager --disable") + self.c.run("dnf config-manager --add-repo /tmp/compose.repo") + self.c.run("dnf config-manager --enable baseos-compose") + self.c.run("dnf config-manager --enable appstream-compose") + self.c.run("dnf clean all") + + + +def runtests(auth_name, auth_key, query, path=None, compose=None): + d = DuffyWrapper(auth_name=auth_name, auth_key=auth_key) + print("Getting", *query) + hostnames = d.get_hostnames(*query) + if not hostnames: + raise Exception("Didn't manage to find or provision matching machine.") + + hostname = hostnames[0] + print("root@"+hostname) + tmux = TmuxWrapper("root@"+hostname) + if compose: + print("Setting compose", compose) + tmux.set_repo(compose) + if path: + print("Sending local repo",path ) + tmux.send_folder(path, "/opt/t_functional") + else: + tmux.c.run("git clone https://github.com/CentOS/sig-core-t_functional /opt/t_functional") + tmux.ensure_session() + tmux.run("cd /opt/t_functional") + tmux.run("export SYSTEMD_PAGER=") + tmux.run("ls -l") + tmux.c.run("cd /opt//t_functional && /bin/bash runtests.sh 0_common") + return tmux + + +def main(): + parser = ArgumentParser(prog='My App') + parser.add_argument('--arch', help="The duffy query.") + parser.add_argument('--path', help="", default=None, required=False) + parser.add_argument('--release', help="" ) + parser.add_argument('--compose', help="", default=None, required=False) + args = parser.parse_args() + auth_name=os.getenv("DUFFY_AUTH_NAME") + auth_key=os.getenv("DUFFY_AUTH_KEY") + if auth_name and auth_key: + runtests(auth_name, auth_key, ['virt', args.arch, args.release], path=args.path, compose=args.compose) + +if __name__ == "__main__": + main() diff --git a/tests/p_curl/0-install_curl.sh b/tests/p_curl/0-install_curl.sh index 555a439..3503caf 100755 --- a/tests/p_curl/0-install_curl.sh +++ b/tests/p_curl/0-install_curl.sh @@ -4,7 +4,10 @@ t_Log "$0 - installing curl" if [ "$centos_ver" -ge "9" ]; then - t_InstallPackage curl-minimal + binary=$(which curl) + if [ -z $binary ]; then + t_InstallPackage curl-minimal + fi else t_InstallPackage curl fi diff --git a/tests/p_openssh/sshd_user_login-with-key.sh b/tests/p_openssh/sshd_user_login-with-key.sh index f8d41e0..8663906 100755 --- a/tests/p_openssh/sshd_user_login-with-key.sh +++ b/tests/p_openssh/sshd_user_login-with-key.sh @@ -12,7 +12,11 @@ fi for KeyType in $keytypes; do userdel -rf sshtest; useradd sshtest && echo sshtest | passwd --stdin sshtest - runuser -l sshtest -c "echo | ssh-keygen -q -t ${KeyType} -b 1024 -f ~/.ssh/id_${KeyType}" > /dev/null + if [ $centos_ver -ge 8 ]; then + runuser -l sshtest -c "echo | ssh-keygen -q -t ${KeyType} -b 2048 -f ~/.ssh/id_${KeyType}" > /dev/null + else + runuser -l sshtest -c "echo | ssh-keygen -q -t ${KeyType} -b 1024 -f ~/.ssh/id_${KeyType}" > /dev/null + fi runuser -l sshtest -c "cat ~/.ssh/*pub > ~/.ssh/authorized_keys && chmod 600 ~/.ssh/*keys" > /dev/null cp ./tests/p_openssh/_helper_sshd_user_login-with-key.expect /home/sshtest/ && chmod +x /home/sshtest/*.expect diff --git a/tests/p_postgresql/1-config-postgresql.sh b/tests/p_postgresql/1-config-postgresql.sh index 8058422..7c5e7f5 100755 --- a/tests/p_postgresql/1-config-postgresql.sh +++ b/tests/p_postgresql/1-config-postgresql.sh @@ -4,7 +4,7 @@ t_Log "Running $0 - initializing and starting PostgreSQL" t_Log "Initialize PostgreSQL DB " -if (t_GetPkgRel postgresql | grep -q el8) then +if [ $centos_ver -ge 8 ]; then postgresql-setup --initdb elif (t_GetPkgRel postgresql | grep -q el7) then postgresql-setup initdb