From ae8198a11e507c4f4f701aa92c3ae531d140e547 Mon Sep 17 00:00:00 2001

From: Yash Mankad <ymankad@redhat.com>

Date: Wed, 12 Dec 2018 00:14:41 +0000

Subject: [PATCH 13/13] Acceptance tests: add make rule for running them

MIME-Version: 1.0

Content-Type: text/plain; charset=UTF-8

Content-Transfer-Encoding: 8bit

RH-Author: Yash Mankad <ymankad@redhat.com>

Message-id: <9527fefa2d8d1b27d4a647cf8355236b61fb028b.1544573601.git.ymankad@redhat.com>

Patchwork-id: 83439

O-Subject: [RHEL-8.0 qemu-kvm PATCH v2 7/7] Acceptance tests: add make rule for running them

Bugzilla: 1655807

RH-Acked-by: Eduardo Habkost <ehabkost@redhat.com>

RH-Acked-by: John Snow <jsnow@redhat.com>

RH-Acked-by: Philippe Mathieu-Daudé <philmd@redhat.com>

From: Cleber Rosa <crosa@redhat.com>

The acceptance (aka functional, aka Avocado-based) tests are

Python files located in "tests/acceptance" that need to be run

with the Avocado libs and test runner.

Let's provide a convenient way for QEMU developers to run them,

by making use of the tests-venv with the required setup.

Also, while the Avocado test runner will take care of creating a

location to save test results to, it was understood that it's better

if the results are kept within the build tree.

Signed-off-by: Cleber Rosa <crosa@redhat.com>

Acked-by: Stefan Hajnoczi <stefanha@redhat.com>

Acked-by: Wainer dos Santos Moschetta <wainersm@redhat.com>

Reviewed-by: Caio Carrara <ccarrara@redhat.com>

Message-Id: <20181018153134.8493-3-crosa@redhat.com>

Signed-off-by: Eduardo Habkost <ehabkost@redhat.com>

(cherry picked from commit a56931eef343c7564e35bcc05eaed2a469a1b1b8)

Signed-off-by: Yash Mankad <ymankad@redhat.com>

Signed-off-by: Danilo C. L. de Paula <ddepaula@redhat.com>

---

 docs/devel/testing.rst | 43 ++++++++++++++++++++++++++++++++++++++-----

 tests/Makefile.include | 21 +++++++++++++++++++--

 tests/requirements.txt |  1 +

 3 files changed, 58 insertions(+), 7 deletions(-)

diff --git a/docs/devel/testing.rst b/docs/devel/testing.rst

index f33e5a8..db08a80 100644

--- a/docs/devel/testing.rst

+++ b/docs/devel/testing.rst

@@ -524,10 +524,39 @@ Tests based on ``avocado_qemu.Test`` can easily:

    - http://avocado-framework.readthedocs.io/en/latest/api/test/avocado.html#avocado.Test

    - http://avocado-framework.readthedocs.io/en/latest/api/utils/avocado.utils.html

-Installation

-------------

+Running tests

+-------------

+

+You can run the acceptance tests simply by executing:

+

+.. code::

+

+  make check-acceptance

+

+This involves the automatic creation of Python virtual environment

+within the build tree (at ``tests/venv``) which will have all the

+right dependencies, and will save tests results also within the

+build tree (at ``tests/results``).

-To install Avocado and its dependencies, run:

+Note: the build environment must be using a Python 3 stack, and have

+the ``venv`` and ``pip`` packages installed.  If necessary, make sure

+``configure`` is called with ``--python=`` and that those modules are

+available.  On Debian and Ubuntu based systems, depending on the

+specific version, they may be on packages named ``python3-venv`` and

+``python3-pip``.

+

+The scripts installed inside the virtual environment may be used

+without an "activation".  For instance, the Avocado test runner

+may be invoked by running:

+

+ .. code::

+

+  tests/venv/bin/avocado run $OPTION1 $OPTION2 tests/acceptance/

+

+Manual Installation

+-------------------

+

+To manually install Avocado and its dependencies, run:

 .. code::

@@ -668,11 +697,15 @@ The exact QEMU binary to be used on QEMUMachine.

 Uninstalling Avocado

 --------------------

-If you've followed the installation instructions above, you can easily

-uninstall Avocado.  Start by listing the packages you have installed::

+If you've followed the manual installation instructions above, you can

+easily uninstall Avocado.  Start by listing the packages you have

+installed::

   pip list --user

 And remove any package you want with::

   pip uninstall <package_name>

+

+If you've used ``make check-acceptance``, the Python virtual environment where

+Avocado is installed will be cleaned up as part of ``make check-clean``.

diff --git a/tests/Makefile.include b/tests/Makefile.include

index 99a9dcd..1177ca3 100644

--- a/tests/Makefile.include

+++ b/tests/Makefile.include

@@ -10,6 +10,7 @@ check-help:

 	@echo " $(MAKE) check-speed          Run qobject speed tests"

 	@echo " $(MAKE) check-qapi-schema    Run QAPI schema tests"

 	@echo " $(MAKE) check-block          Run block tests"

+	@echo " $(MAKE) check-acceptance     Run all acceptance (functional) tests"

 	@echo " $(MAKE) check-report.html    Generates an HTML test report"

 	@echo " $(MAKE) check-venv           Creates a Python venv for tests"

 	@echo " $(MAKE) check-clean          Clean the tests"

@@ -956,10 +957,15 @@ check-decodetree:

 # Python venv for running tests

-.PHONY: check-venv

+.PHONY: check-venv check-acceptance

 TESTS_VENV_DIR=$(BUILD_DIR)/tests/venv

 TESTS_VENV_REQ=$(SRC_PATH)/tests/requirements.txt

+TESTS_RESULTS_DIR=$(BUILD_DIR)/tests/results

+# Controls the output generated by Avocado when running tests.

+# Any number of command separated loggers are accepted.  For more

+# information please refer to "avocado --help".

+AVOCADO_SHOW=none

 $(shell $(PYTHON) -c 'import sys; assert sys.version_info >= (3,0)' >/dev/null 2>&1)

 ifeq ($(.SHELLSTATUS),0)

@@ -976,8 +982,19 @@ $(TESTS_VENV_DIR):

 	$(error "venv directory for tests requires Python 3")

 endif

+$(TESTS_RESULTS_DIR):

+	$(call quiet-command, mkdir -p $@, \

+            MKDIR, $@)

+

 check-venv: $(TESTS_VENV_DIR)

+check-acceptance: check-venv $(TESTS_RESULTS_DIR)

+	$(call quiet-command, \

+            $(TESTS_VENV_DIR)/bin/python -m avocado \

+            --show=$(AVOCADO_SHOW) run --job-results-dir=$(TESTS_RESULTS_DIR) \

+            --failfast=on $(SRC_PATH)/tests/acceptance, \

+            "AVOCADO", "tests/acceptance")

+

 # Consolidated targets

 .PHONY: check-qapi-schema check-qtest check-unit check check-clean

@@ -992,7 +1009,7 @@ check-clean:

 	rm -rf $(check-unit-y) tests/*.o $(QEMU_IOTESTS_HELPERS-y)

 	rm -rf $(sort $(foreach target,$(SYSEMU_TARGET_LIST), $(check-qtest-$(target)-y)) $(check-qtest-generic-y))

 	rm -f tests/test-qapi-gen-timestamp

-	rm -rf $(TESTS_VENV_DIR)

+	rm -rf $(TESTS_VENV_DIR) $(TESTS_RESULTS_DIR)

 clean: check-clean

diff --git a/tests/requirements.txt b/tests/requirements.txt

index d39f9d1..64c6e27 100644

--- a/tests/requirements.txt

+++ b/tests/requirements.txt

@@ -1,3 +1,4 @@

 # Add Python module requirements, one per line, to be installed

 # in the tests/venv Python virtual environment. For more info,

 # refer to: https://pip.pypa.io/en/stable/user_guide/#id1

+avocado-framework==65.0

-- 

1.8.3.1