From 3bf2f4c13bd334bb4e9e2b05336b88f60fdf687c Mon Sep 17 00:00:00 2001 From: Sebastian Huber Date: Tue, 21 Nov 2023 11:13:16 +0100 Subject: testrunner: New --- rtemsspec/packagebuildfactory.py | 6 + rtemsspec/testrunner.py | 179 +++++++++++++++++++++ .../spec-packagebuild/qdp/test-runner/dummy.yml | 10 ++ .../qdp/test-runner/grmon-manual.yml | 16 ++ .../qdp/test-runner/subprocess.yml | 14 ++ rtemsspec/tests/test_packagebuild.py | 130 ++++++++++++++- spec-qdp/spec/qdp-test-runner-dummy.yml | 22 +++ spec-qdp/spec/qdp-test-runner-grmon-manual.yml | 34 ++++ spec-qdp/spec/qdp-test-runner-subprocess.yml | 26 +++ spec-qdp/spec/qdp-test-runner.yml | 35 ++++ 10 files changed, 471 insertions(+), 1 deletion(-) create mode 100644 rtemsspec/testrunner.py create mode 100644 rtemsspec/tests/spec-packagebuild/qdp/test-runner/dummy.yml create mode 100644 rtemsspec/tests/spec-packagebuild/qdp/test-runner/grmon-manual.yml create mode 100644 rtemsspec/tests/spec-packagebuild/qdp/test-runner/subprocess.yml create mode 100644 spec-qdp/spec/qdp-test-runner-dummy.yml create mode 100644 spec-qdp/spec/qdp-test-runner-grmon-manual.yml create mode 100644 spec-qdp/spec/qdp-test-runner-subprocess.yml create mode 100644 spec-qdp/spec/qdp-test-runner.yml diff --git a/rtemsspec/packagebuildfactory.py b/rtemsspec/packagebuildfactory.py index 22ba8ded..f62fa78d 100644 --- a/rtemsspec/packagebuildfactory.py +++ b/rtemsspec/packagebuildfactory.py @@ -29,6 +29,8 @@ from rtemsspec.directorystate import DirectoryState from rtemsspec.packagebuild import BuildItemFactory, PackageVariant from rtemsspec.reposubset import RepositorySubset from rtemsspec.runactions import RunActions +from rtemsspec.testrunner import DummyTestRunner, GRMONManualTestRunner, \ + SubprocessTestRunner def create_build_item_factory() -> BuildItemFactory: @@ -42,5 +44,9 @@ def create_build_item_factory() -> BuildItemFactory: factory.add_constructor("qdp/directory-state/repository", DirectoryState) factory.add_constructor("qdp/directory-state/unpacked-archive", DirectoryState) + factory.add_constructor("qdp/test-runner/dummy", DummyTestRunner) + factory.add_constructor("qdp/test-runner/grmon-manual", + GRMONManualTestRunner) + factory.add_constructor("qdp/test-runner/subprocess", SubprocessTestRunner) factory.add_constructor("qdp/variant", PackageVariant) return factory diff --git a/rtemsspec/testrunner.py b/rtemsspec/testrunner.py new file mode 100644 index 00000000..94bad5a4 --- /dev/null +++ b/rtemsspec/testrunner.py @@ -0,0 +1,179 @@ +# SPDX-License-Identifier: BSD-2-Clause +""" This module provides a build item to run tests. """ + +# Copyright (C) 2022, 2023 embedded brains GmbH & Co. KG +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. + +import datetime +import logging +import multiprocessing +import os +import queue +import subprocess +from subprocess import run as subprocess_run +import tarfile +import time +import threading +from typing import Any, Dict, List, NamedTuple, Union + +from rtemsspec.items import Item, ItemGetValueContext +from rtemsspec.packagebuild import BuildItem, PackageBuildDirector +from rtemsspec.testoutputparser import augment_report + +Report = Dict[str, Union[str, List[str]]] + + +class Executable(NamedTuple): + """ This data class represents a test executable. """ + path: str + digest: str + timeout: int + + +class TestRunner(BuildItem): + """ Runs the tests. """ + + def __init__(self, director: PackageBuildDirector, item: Item): + super().__init__(director, item) + self._executable = "/dev/null" + self._executables: List[Executable] = [] + self.mapper.add_get_value(f"{self.item.type}:/test-executable", + self._get_test_executable) + self.mapper.add_get_value(f"{self.item.type}:/test-executables-grmon", + self._get_test_executables_grmon) + + def _get_test_executable(self, _ctx: ItemGetValueContext) -> Any: + return self._executable + + def _get_test_executables_grmon(self, _ctx: ItemGetValueContext) -> Any: + return " \\\n".join( + os.path.basename(executable.path) + for executable in self._executables) + + def run_tests(self, executables: List[Executable]) -> List[Report]: + """ + Runs the test executables and produces a log file of the test run. + """ + self._executables = executables + return [] + + +class DummyTestRunner(TestRunner): + """ Cannot run the tests. """ + + def run_tests(self, _executables: List[Executable]) -> List[Report]: + """ Raises an exception. """ + raise IOError("this test runner cannot run tests") + + +class GRMONManualTestRunner(TestRunner): + """ Provides scripts to run the tests using GRMON. """ + + def run_tests(self, executables: List[Executable]) -> List[Report]: + super().run_tests(executables) + base = self["script-base-path"] + dir_name = os.path.basename(base) + grmon_name = f"{base}.grmon" + shell_name = f"{base}.sh" + tar_name = f"{base}.tar.xz" + os.makedirs(os.path.dirname(base), exist_ok=True) + with tarfile.open(tar_name, "w:xz") as tar_file: + with open(grmon_name, "w", encoding="utf-8") as grmon_file: + grmon_file.write(self["grmon-script"]) + tar_file.add(grmon_name, os.path.join(dir_name, "run.grmon")) + with open(shell_name, "w", encoding="utf-8") as shell_file: + shell_file.write(self["shell-script"]) + tar_file.add(shell_name, os.path.join(dir_name, "run.sh")) + for executable in executables: + tar_file.add( + executable.path, + os.path.join(dir_name, os.path.basename(executable.path))) + raise IOError(f"Run the tests provided by {tar_name}") + + +def _now_utc() -> str: + return datetime.datetime.utcnow().isoformat() + + +class _Job: + # pylint: disable=too-few-public-methods + def __init__(self, executable: Executable, command: List[str]): + self.report: Report = { + "executable": executable.path, + "executable-sha512": executable.digest, + "command-line": command + } + self.timeout = executable.timeout + + +def _worker(work_queue: queue.Queue, item: BuildItem): + with open(os.devnull, "rb") as devnull: + while True: + try: + job = work_queue.get_nowait() + except queue.Empty: + return + logging.info("%s: run: %s", item.uid, job.report["command-line"]) + job.report["start-time"] = _now_utc() + begin = time.monotonic() + try: + process = subprocess_run(job.report["command-line"], + check=False, + stdin=devnull, + stdout=subprocess.PIPE, + timeout=job.timeout) + stdout = process.stdout.decode("utf-8") + except subprocess.TimeoutExpired as timeout: + if timeout.stdout is not None: + stdout = timeout.stdout.decode("utf-8") + else: + stdout = "" + except Exception: # pylint: disable=broad-exception-caught + stdout = "" + output = stdout.rstrip().replace("\r\n", "\n").split("\n") + augment_report(job.report, output) + job.report["output"] = output + job.report["duration"] = time.monotonic() - begin + logging.debug("%s: done: %s", item.uid, job.report["executable"]) + work_queue.task_done() + + +class SubprocessTestRunner(TestRunner): + """ Runs the tests in subprocesses. """ + + def run_tests(self, executables: List[Executable]) -> List[Report]: + super().run_tests(executables) + work_queue: queue.Queue[_Job] = \ + queue.Queue() # pylint: disable=unsubscriptable-object + jobs: List[_Job] = [] + for executable in executables: + self._executable = executable.path + job = _Job(executable, self["command"]) + jobs.append(job) + work_queue.put(job) + for _ in range(min(multiprocessing.cpu_count(), len(executables))): + threading.Thread(target=_worker, + args=(work_queue, self), + daemon=True).start() + work_queue.join() + return [job.report for job in jobs] diff --git a/rtemsspec/tests/spec-packagebuild/qdp/test-runner/dummy.yml b/rtemsspec/tests/spec-packagebuild/qdp/test-runner/dummy.yml new file mode 100644 index 00000000..3dfc14e5 --- /dev/null +++ b/rtemsspec/tests/spec-packagebuild/qdp/test-runner/dummy.yml @@ -0,0 +1,10 @@ +SPDX-License-Identifier: CC-BY-SA-4.0 OR BSD-2-Clause +copyrights: +- Copyright (C) 2023 embedded brains GmbH & Co. KG +description: Description. +enabled-by: true +links: [] +params: {} +qdp-type: test-runner +test-runner-type: dummy +type: qdp diff --git a/rtemsspec/tests/spec-packagebuild/qdp/test-runner/grmon-manual.yml b/rtemsspec/tests/spec-packagebuild/qdp/test-runner/grmon-manual.yml new file mode 100644 index 00000000..0325b127 --- /dev/null +++ b/rtemsspec/tests/spec-packagebuild/qdp/test-runner/grmon-manual.yml @@ -0,0 +1,16 @@ +SPDX-License-Identifier: CC-BY-SA-4.0 OR BSD-2-Clause +script-base-path: ${../variant:/prefix-directory}/tests +grmon-script: | + ${.:/test-executables-grmon} +shell-script: | + ${.:/params/x} +copyrights: +- Copyright (C) 2023 embedded brains GmbH & Co. KG +description: Description. +enabled-by: true +links: [] +params: + x: abc +qdp-type: test-runner +test-runner-type: grmon-manual +type: qdp diff --git a/rtemsspec/tests/spec-packagebuild/qdp/test-runner/subprocess.yml b/rtemsspec/tests/spec-packagebuild/qdp/test-runner/subprocess.yml new file mode 100644 index 00000000..ffcd549e --- /dev/null +++ b/rtemsspec/tests/spec-packagebuild/qdp/test-runner/subprocess.yml @@ -0,0 +1,14 @@ +SPDX-License-Identifier: CC-BY-SA-4.0 OR BSD-2-Clause +command: +- foo +- bar +- ${.:/test-executable} +copyrights: +- Copyright (C) 2023 embedded brains GmbH & Co. KG +description: Description. +enabled-by: true +links: [] +params: {} +qdp-type: test-runner +test-runner-type: subprocess +type: qdp diff --git a/rtemsspec/tests/test_packagebuild.py b/rtemsspec/tests/test_packagebuild.py index aad59c63..2aa54228 100644 --- a/rtemsspec/tests/test_packagebuild.py +++ b/rtemsspec/tests/test_packagebuild.py @@ -24,18 +24,23 @@ # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. +import json import logging import os import pytest from pathlib import Path import shutil +import subprocess import tarfile +from typing import NamedTuple from rtemsspec.items import EmptyItem, Item, ItemCache, ItemGetValueContext from rtemsspec.packagebuild import BuildItem, BuildItemMapper, \ build_item_input, PackageBuildDirector from rtemsspec.packagebuildfactory import create_build_item_factory from rtemsspec.specverify import verify +import rtemsspec.testrunner +from rtemsspec.testrunner import Executable from rtemsspec.tests.util import get_and_clear_log from rtemsspec.util import run_command @@ -80,7 +85,21 @@ class _TestItem(BuildItem): super().__init__(director, item, BuildItemMapper(item, recursive=True)) -def test_packagebuild(caplog, tmpdir): +class _Subprocess(NamedTuple): + stdout: bytes + + +def _test_runner_subprocess(command, check, stdin, stdout, timeout): + if command[2] == "a.exe": + raise Exception + if command[2] == "b.exe": + raise subprocess.TimeoutExpired(command[2], timeout, b"") + if command[2] == "c.exe": + raise subprocess.TimeoutExpired(command[2], timeout, None) + return _Subprocess(b"u\r\nv\nw\n") + + +def test_packagebuild(caplog, tmpdir, monkeypatch): tmp_dir = Path(tmpdir) item_cache = _create_item_cache(tmp_dir, Path("spec-packagebuild")) @@ -221,3 +240,112 @@ def test_packagebuild(caplog, tmpdir): assert not os.path.exists(os.path.join(tmpdir, "pkg", "sub-repo", "bsp.c")) director.build_package(None, None) assert os.path.exists(os.path.join(tmpdir, "pkg", "sub-repo", "bsp.c")) + + # Test DummyTestRunner + dummy_runner = director["/qdp/test-runner/dummy"] + with pytest.raises(IOError): + dummy_runner.run_tests([]) + + # Test GRMONManualTestRunner + grmon_manual_runner = director["/qdp/test-runner/grmon-manual"] + exe = tmp_dir / "a.exe" + exe.touch() + with pytest.raises(IOError): + grmon_manual_runner.run_tests([ + Executable( + str(exe), "QvahP3YJU9bvpd7DYxJDkRBLJWbEFMEoH5Ncwu6UtxA" + "_l9EQ1zLW9yQTprx96BTyYE2ew7vV3KECjlRg95Ya6A==", 456) + ]) + with tarfile.open(tmp_dir / "tests.tar.xz", "r:*") as archive: + assert archive.getnames() == [ + "tests/run.grmon", "tests/run.sh", "tests/a.exe" + ] + with archive.extractfile("tests/run.grmon") as src: + assert src.read() == b"a.exe\n" + with archive.extractfile("tests/run.sh") as src: + assert src.read() == b"abc\n" + + # Test SubprocessTestRunner + subprocess_runner = director["/qdp/test-runner/subprocess"] + monkeypatch.setattr(rtemsspec.testrunner, "subprocess_run", + _test_runner_subprocess) + reports = subprocess_runner.run_tests([ + Executable( + "a.exe", "QvahP3YJU9bvpd7DYxJDkRBLJWbEFMEoH5Ncwu6UtxA" + "_l9EQ1zLW9yQTprx96BTyYE2ew7vV3KECjlRg95Ya6A==", 1), + Executable( + "b.exe", "4VgX6KGWuDyG5vmlO4J-rdbHpOJoIIYLn_3oSk2BKAc" + "Au5RXTg1IxhHjiPO6Yzl8u4GsWBh0qc3flRwEFcD8_A==", 2), + Executable( + "c.exe", "YtTC0r1DraKOn9vNGppBAVFVTnI9IqS6jFDRBKVucU_" + "W_dpQF0xtC_mRjGV7t5RSQKhY7l3iDGbeBZJ-lV37bg==", 3), + Executable( + "d.exe", "ZtTC0r1DraKOn9vNGppBAVFVTnI9IqS6jFDRBKVucU_" + "W_dpQF0xtC_mRjGV7t5RSQKhY7l3iDGbeBZJ-lV37bg==", 4) + ]) + monkeypatch.undo() + reports[0]["start-time"] = "c" + reports[0]["duration"] = 2. + reports[1]["start-time"] = "d" + reports[1]["duration"] = 3. + reports[2]["start-time"] = "e" + reports[2]["duration"] = 4. + reports[3]["start-time"] = "f" + reports[3]["duration"] = 5. + assert reports == [{ + "command-line": ["foo", "bar", "a.exe"], + "data-ranges": [], + "duration": + 2.0, + "executable": + "a.exe", + "executable-sha512": + "QvahP3YJU9bvpd7DYxJDkRBLJWbEFMEoH5Ncwu6UtxA_" + "l9EQ1zLW9yQTprx96BTyYE2ew7vV3KECjlRg95Ya6A==", + "info": {}, + "output": [""], + "start-time": + "c" + }, { + "command-line": ["foo", "bar", "b.exe"], + "data-ranges": [], + "duration": + 3., + "executable": + "b.exe", + "executable-sha512": + "4VgX6KGWuDyG5vmlO4J-rdbHpOJoIIYLn_3oSk2BKAcA" + "u5RXTg1IxhHjiPO6Yzl8u4GsWBh0qc3flRwEFcD8_A==", + "info": {}, + "output": [""], + "start-time": + "d" + }, { + "command-line": ["foo", "bar", "c.exe"], + "data-ranges": [], + "duration": + 4., + "executable": + "c.exe", + "executable-sha512": + "YtTC0r1DraKOn9vNGppBAVFVTnI9IqS6jFDRBKVucU_W" + "_dpQF0xtC_mRjGV7t5RSQKhY7l3iDGbeBZJ-lV37bg==", + "info": {}, + "output": [""], + "start-time": + "e" + }, { + "command-line": ["foo", "bar", "d.exe"], + "data-ranges": [], + "duration": + 5., + "executable": + "d.exe", + "executable-sha512": + "ZtTC0r1DraKOn9vNGppBAVFVTnI9IqS6jFDRBKVucU_W" + "_dpQF0xtC_mRjGV7t5RSQKhY7l3iDGbeBZJ-lV37bg==", + "info": {}, + "output": ["u", "v", "w"], + "start-time": + "f" + }] diff --git a/spec-qdp/spec/qdp-test-runner-dummy.yml b/spec-qdp/spec/qdp-test-runner-dummy.yml new file mode 100644 index 00000000..6f5c5834 --- /dev/null +++ b/spec-qdp/spec/qdp-test-runner-dummy.yml @@ -0,0 +1,22 @@ +SPDX-License-Identifier: CC-BY-SA-4.0 OR BSD-2-Clause +copyrights: +- Copyright (C) 2023 embedded brains GmbH & Co. KG +enabled-by: true +links: +- role: spec-member + uid: root +- role: spec-refinement + spec-key: test-runner-type + spec-value: dummy + uid: qdp-test-runner +spec-description: null +spec-example: null +spec-info: + dict: + attributes: {} + description: | + This set of attributes specifies a test runner which cannot run tests. + mandatory-attributes: all +spec-name: Dummy Test Runner Item Type +spec-type: qdp-test-runner-dummy +type: spec diff --git a/spec-qdp/spec/qdp-test-runner-grmon-manual.yml b/spec-qdp/spec/qdp-test-runner-grmon-manual.yml new file mode 100644 index 00000000..26d89669 --- /dev/null +++ b/spec-qdp/spec/qdp-test-runner-grmon-manual.yml @@ -0,0 +1,34 @@ +SPDX-License-Identifier: CC-BY-SA-4.0 OR BSD-2-Clause +copyrights: +- Copyright (C) 2023 embedded brains GmbH & Co. KG +enabled-by: true +links: +- role: spec-member + uid: root +- role: spec-refinement + spec-key: test-runner-type + spec-value: grmon-manual + uid: qdp-test-runner +spec-description: null +spec-example: null +spec-info: + dict: + attributes: + script-base-path: + description: | + It shall be the base path for scripts. + spec-type: str + grmon-script: + description: | + It shall be the GRMON script to run the tests. + spec-type: str + shell-script: + description: | + It shall be the shell script to run the GRMON script. + spec-type: str + description: | + This set of attributes specifies a GRMON manual test runner. + mandatory-attributes: all +spec-name: GRMON Manual Test Runner Item Type +spec-type: qdp-test-runner-grmon-manual +type: spec diff --git a/spec-qdp/spec/qdp-test-runner-subprocess.yml b/spec-qdp/spec/qdp-test-runner-subprocess.yml new file mode 100644 index 00000000..a8846c5b --- /dev/null +++ b/spec-qdp/spec/qdp-test-runner-subprocess.yml @@ -0,0 +1,26 @@ +SPDX-License-Identifier: CC-BY-SA-4.0 OR BSD-2-Clause +copyrights: +- Copyright (C) 2023 embedded brains GmbH & Co. KG +enabled-by: true +links: +- role: spec-member + uid: root +- role: spec-refinement + spec-key: test-runner-type + spec-value: subprocess + uid: qdp-test-runner +spec-description: null +spec-example: null +spec-info: + dict: + attributes: + command: + description: | + It shall be the test runner subprocess command. + spec-type: list-str + description: | + This set of attributes specifies a subprocess test runner. + mandatory-attributes: all +spec-name: Subprocess Test Runner Item Type +spec-type: qdp-test-runner-subprocess +type: spec diff --git a/spec-qdp/spec/qdp-test-runner.yml b/spec-qdp/spec/qdp-test-runner.yml new file mode 100644 index 00000000..0fd7de54 --- /dev/null +++ b/spec-qdp/spec/qdp-test-runner.yml @@ -0,0 +1,35 @@ +SPDX-License-Identifier: CC-BY-SA-4.0 OR BSD-2-Clause +copyrights: +- Copyright (C) 2022 embedded brains GmbH & Co. KG +enabled-by: true +links: +- role: spec-member + uid: root +- role: spec-refinement + spec-key: qdp-type + spec-value: test-runner + uid: qdp-root +spec-description: null +spec-example: null +spec-info: + dict: + attributes: + description: + description: | + It shall be the test runner description. + spec-type: str + params: + description: | + It shall be an optional set of parameters which may be used for + variable subsitution. + spec-type: any + test-runner-type: + description: | + It shall be the test runner type. + spec-type: name + description: | + This set of attributes specifies a test runner. + mandatory-attributes: all +spec-name: Test Runner Item Type +spec-type: qdp-test-runner +type: spec -- cgit v1.2.3