From 61a4f93ed2e6f8353cac2cbd43a52b39cb8ef1c9 Mon Sep 17 00:00:00 2001 From: Sebastian Huber Date: Tue, 21 Nov 2023 11:13:16 +0100 Subject: runtests: New --- rtemsspec/packagebuildfactory.py | 3 + rtemsspec/runtests.py | 115 +++++++++++++++++++++ .../tests/spec-packagebuild/qdp/build/bsp.yml | 19 ++++ .../tests/spec-packagebuild/qdp/package-build.yml | 2 + .../spec-packagebuild/qdp/steps/run-tests.yml | 23 +++++ .../qdp/test-logs/bsp-previous.yml | 15 +++ .../tests/spec-packagebuild/qdp/test-logs/bsp.yml | 15 +++ .../spec-packagebuild/qdp/test-runner/test.yml | 10 ++ .../spec/qdp-test-runner-test.yml | 22 ++++ rtemsspec/tests/test-files/pkg/build/bsp/a.exe | 0 rtemsspec/tests/test-files/pkg/build/bsp/b.exe | 1 + .../tests/test-files/pkg/build/bsp/b.norun.exe | 0 rtemsspec/tests/test_packagebuild.py | 34 +++++- spec-qdp/spec/qdp-input-generic-role.yml | 4 + spec-qdp/spec/qdp-run-tests.yml | 22 ++++ spec-qdp/spec/qdp-test-log.yml | 22 ++++ spec-qdp/spec/qdp-test-procedure-role.yml | 23 +++++ 17 files changed, 328 insertions(+), 2 deletions(-) create mode 100644 rtemsspec/runtests.py create mode 100644 rtemsspec/tests/spec-packagebuild/qdp/build/bsp.yml create mode 100644 rtemsspec/tests/spec-packagebuild/qdp/steps/run-tests.yml create mode 100644 rtemsspec/tests/spec-packagebuild/qdp/test-logs/bsp-previous.yml create mode 100644 rtemsspec/tests/spec-packagebuild/qdp/test-logs/bsp.yml create mode 100644 rtemsspec/tests/spec-packagebuild/qdp/test-runner/test.yml create mode 100644 rtemsspec/tests/spec-packagebuild/spec/qdp-test-runner-test.yml create mode 100644 rtemsspec/tests/test-files/pkg/build/bsp/a.exe create mode 100644 rtemsspec/tests/test-files/pkg/build/bsp/b.exe create mode 100644 rtemsspec/tests/test-files/pkg/build/bsp/b.norun.exe create mode 100644 spec-qdp/spec/qdp-run-tests.yml create mode 100644 spec-qdp/spec/qdp-test-log.yml create mode 100644 spec-qdp/spec/qdp-test-procedure-role.yml diff --git a/rtemsspec/packagebuildfactory.py b/rtemsspec/packagebuildfactory.py index f62fa78d..9b3e7944 100644 --- a/rtemsspec/packagebuildfactory.py +++ b/rtemsspec/packagebuildfactory.py @@ -29,6 +29,7 @@ from rtemsspec.directorystate import DirectoryState from rtemsspec.packagebuild import BuildItemFactory, PackageVariant from rtemsspec.reposubset import RepositorySubset from rtemsspec.runactions import RunActions +from rtemsspec.runtests import RunTests, TestLog from rtemsspec.testrunner import DummyTestRunner, GRMONManualTestRunner, \ SubprocessTestRunner @@ -40,8 +41,10 @@ def create_build_item_factory() -> BuildItemFactory: factory.add_constructor("qdp/build-step/repository-subset", RepositorySubset) factory.add_constructor("qdp/build-step/run-actions", RunActions) + factory.add_constructor("qdp/build-step/run-tests", RunTests) factory.add_constructor("qdp/directory-state/generic", DirectoryState) factory.add_constructor("qdp/directory-state/repository", DirectoryState) + factory.add_constructor("qdp/directory-state/test-log", TestLog) factory.add_constructor("qdp/directory-state/unpacked-archive", DirectoryState) factory.add_constructor("qdp/test-runner/dummy", DummyTestRunner) diff --git a/rtemsspec/runtests.py b/rtemsspec/runtests.py new file mode 100644 index 00000000..a0de436d --- /dev/null +++ b/rtemsspec/runtests.py @@ -0,0 +1,115 @@ +# SPDX-License-Identifier: BSD-2-Clause +""" This module provides a build step to run the RTEMS Tester. """ + +# Copyright (C) 2022 embedded brains GmbH (http://www.embedded-brains.de) +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. + +import datetime +import json +import os +import logging +import time +from typing import Dict, List + +from rtemsspec.directorystate import DirectoryState +from rtemsspec.items import Item +from rtemsspec.packagebuild import BuildItem, PackageBuildDirector +from rtemsspec.testrunner import Executable, Report, TestRunner + + +def _now_utc() -> str: + return datetime.datetime.utcnow().isoformat() + + +class TestLog(DirectoryState): + """ Maintains a test log. """ + + def __init__(self, director: PackageBuildDirector, item: Item): + super().__init__(director, item) + self.reports: List[Report] = [] + + def discard(self) -> None: + try: + with open(self.file, "r", encoding="utf-8") as src: + self.reports = json.load(src)["reports"] + logging.info("%s: loaded test log: %s", self.uid, self.file) + except FileNotFoundError: + self.reports = [] + super().discard() + + def get_reports_by_hash(self) -> Dict[str, Report]: + """ Gets the reports by executable hash. """ + reports_by_hash: Dict[str, Report] = {} + for report in self.reports: + digest = report["executable-sha512"] + assert digest not in reports_by_hash + assert isinstance(digest, str) + reports_by_hash[digest] = report + return reports_by_hash + + +class RunTests(BuildItem): + """ Runs the tests. """ + + def run(self) -> None: + start_time = _now_utc() + begin = time.monotonic() + log = self.output("log") + assert isinstance(log, TestLog) + previous_reports_by_hash = log.get_reports_by_hash() + + # Use previous report if the executable hash did not change + source = self.input("source") + assert isinstance(source, DirectoryState) + reports: List[Report] = [] + executables: List[Executable] = [] + for path, digest in source.files_and_hashes(): + if not path.endswith(".exe") or path.endswith(".norun.exe"): + continue + assert digest + report = previous_reports_by_hash.get(digest, None) + if report is None: + logging.debug("%s: run: %s", self.uid, path) + executables.append(Executable(path, digest, 1800)) + else: + logging.debug("%s: use previous report for: %s", self.uid, + path) + report["executable"] = path + reports.append(report) + + # Run the tests with changed executables + if executables: + runner = self.input("runner") + assert isinstance(runner, TestRunner) + reports.extend(runner.run_tests(executables)) + + # Save the reports + os.makedirs(os.path.dirname(log.file), exist_ok=True) + with open(log.file, "w", encoding="utf-8") as dst: + data = { + "duration": time.monotonic() - begin, + "end-time": _now_utc(), + "reports": sorted(reports, key=lambda x: x["executable"]), + "start-time": start_time + } + json.dump(data, dst, sort_keys=True, indent=2) diff --git a/rtemsspec/tests/spec-packagebuild/qdp/build/bsp.yml b/rtemsspec/tests/spec-packagebuild/qdp/build/bsp.yml new file mode 100644 index 00000000..109b8ed8 --- /dev/null +++ b/rtemsspec/tests/spec-packagebuild/qdp/build/bsp.yml @@ -0,0 +1,19 @@ +SPDX-License-Identifier: CC-BY-SA-4.0 OR BSD-2-Clause +copyrights: +- Copyright (C) 2023 embedded brains GmbH & Co. KG +copyrights-by-license: {} +directory: ${../variant:/build-directory}/bsp +directory-state-type: generic +enabled-by: true +files: +- file: a.exe + hash: null +- file: b.exe + hash: null +- file: b.norun.exe + hash: null +hash: null +links: [] +patterns: [] +qdp-type: directory-state +type: qdp diff --git a/rtemsspec/tests/spec-packagebuild/qdp/package-build.yml b/rtemsspec/tests/spec-packagebuild/qdp/package-build.yml index 02c4b721..2af718ef 100644 --- a/rtemsspec/tests/spec-packagebuild/qdp/package-build.yml +++ b/rtemsspec/tests/spec-packagebuild/qdp/package-build.yml @@ -13,6 +13,8 @@ links: uid: steps/repository-subset - role: build-step uid: steps/run-actions +- role: build-step + uid: steps/run-tests - role: build-step uid: steps/archive qdp-type: package-build diff --git a/rtemsspec/tests/spec-packagebuild/qdp/steps/run-tests.yml b/rtemsspec/tests/spec-packagebuild/qdp/steps/run-tests.yml new file mode 100644 index 00000000..5e165bdf --- /dev/null +++ b/rtemsspec/tests/spec-packagebuild/qdp/steps/run-tests.yml @@ -0,0 +1,23 @@ +SPDX-License-Identifier: CC-BY-SA-4.0 OR BSD-2-Clause +build-step-type: run-tests +copyrights: +- Copyright (C) 2023 embedded brains GmbH & Co. KG +description: | + Run the tests. +enabled-by: run-tests +links: +- role: test-procedure + uid: ../variant +- hash: null + name: source + role: input + uid: ../build/bsp +- hash: null + name: runner + role: input + uid: ../test-runner/test +- name: log + role: output + uid: ../test-logs/bsp +qdp-type: build-step +type: qdp diff --git a/rtemsspec/tests/spec-packagebuild/qdp/test-logs/bsp-previous.yml b/rtemsspec/tests/spec-packagebuild/qdp/test-logs/bsp-previous.yml new file mode 100644 index 00000000..e679c62a --- /dev/null +++ b/rtemsspec/tests/spec-packagebuild/qdp/test-logs/bsp-previous.yml @@ -0,0 +1,15 @@ +SPDX-License-Identifier: CC-BY-SA-4.0 OR BSD-2-Clause +copyrights: +- Copyright (C) 2023 embedded brains GmbH & Co. KG +copyrights-by-license: {} +directory: ${../variant:/deployment-directory} +directory-state-type: generic +enabled-by: true +files: +- file: log-empty.json + hash: null +hash: null +links: [] +patterns: [] +qdp-type: directory-state +type: qdp diff --git a/rtemsspec/tests/spec-packagebuild/qdp/test-logs/bsp.yml b/rtemsspec/tests/spec-packagebuild/qdp/test-logs/bsp.yml new file mode 100644 index 00000000..16838238 --- /dev/null +++ b/rtemsspec/tests/spec-packagebuild/qdp/test-logs/bsp.yml @@ -0,0 +1,15 @@ +SPDX-License-Identifier: CC-BY-SA-4.0 OR BSD-2-Clause +copyrights: +- Copyright (C) 2023 embedded brains GmbH & Co. KG +copyrights-by-license: {} +directory: ${../variant:/deployment-directory} +directory-state-type: test-log +enabled-by: true +files: +- file: test-log-bsp.json + hash: null +hash: null +links: [] +patterns: [] +qdp-type: directory-state +type: qdp diff --git a/rtemsspec/tests/spec-packagebuild/qdp/test-runner/test.yml b/rtemsspec/tests/spec-packagebuild/qdp/test-runner/test.yml new file mode 100644 index 00000000..53dbaa06 --- /dev/null +++ b/rtemsspec/tests/spec-packagebuild/qdp/test-runner/test.yml @@ -0,0 +1,10 @@ +SPDX-License-Identifier: CC-BY-SA-4.0 OR BSD-2-Clause +copyrights: +- Copyright (C) 2023 embedded brains GmbH & Co. KG +description: Description. +enabled-by: true +links: [] +params: {} +qdp-type: test-runner +test-runner-type: test +type: qdp diff --git a/rtemsspec/tests/spec-packagebuild/spec/qdp-test-runner-test.yml b/rtemsspec/tests/spec-packagebuild/spec/qdp-test-runner-test.yml new file mode 100644 index 00000000..23e75a04 --- /dev/null +++ b/rtemsspec/tests/spec-packagebuild/spec/qdp-test-runner-test.yml @@ -0,0 +1,22 @@ +SPDX-License-Identifier: CC-BY-SA-4.0 OR BSD-2-Clause +copyrights: +- Copyright (C) 2023 embedded brains GmbH & Co. KG +enabled-by: true +links: +- role: spec-member + uid: root +- role: spec-refinement + spec-key: test-runner-type + spec-value: test + uid: qdp-test-runner +spec-description: null +spec-example: null +spec-info: + dict: + attributes: {} + description: | + This set of attributes specifies a test test runner. + mandatory-attributes: all +spec-name: Subprocess Test Runner Item Type +spec-type: qdp-test-runner-test +type: spec diff --git a/rtemsspec/tests/test-files/pkg/build/bsp/a.exe b/rtemsspec/tests/test-files/pkg/build/bsp/a.exe new file mode 100644 index 00000000..e69de29b diff --git a/rtemsspec/tests/test-files/pkg/build/bsp/b.exe b/rtemsspec/tests/test-files/pkg/build/bsp/b.exe new file mode 100644 index 00000000..61780798 --- /dev/null +++ b/rtemsspec/tests/test-files/pkg/build/bsp/b.exe @@ -0,0 +1 @@ +b diff --git a/rtemsspec/tests/test-files/pkg/build/bsp/b.norun.exe b/rtemsspec/tests/test-files/pkg/build/bsp/b.norun.exe new file mode 100644 index 00000000..e69de29b diff --git a/rtemsspec/tests/test_packagebuild.py b/rtemsspec/tests/test_packagebuild.py index 2aa54228..fa0309d9 100644 --- a/rtemsspec/tests/test_packagebuild.py +++ b/rtemsspec/tests/test_packagebuild.py @@ -32,7 +32,7 @@ from pathlib import Path import shutil import subprocess import tarfile -from typing import NamedTuple +from typing import List, NamedTuple from rtemsspec.items import EmptyItem, Item, ItemCache, ItemGetValueContext from rtemsspec.packagebuild import BuildItem, BuildItemMapper, \ @@ -40,10 +40,12 @@ from rtemsspec.packagebuild import BuildItem, BuildItemMapper, \ from rtemsspec.packagebuildfactory import create_build_item_factory from rtemsspec.specverify import verify import rtemsspec.testrunner -from rtemsspec.testrunner import Executable +from rtemsspec.testrunner import Executable, Report, TestRunner from rtemsspec.tests.util import get_and_clear_log from rtemsspec.util import run_command +TestRunner.__test__ = False + def _copy_dir(src, dst): dst.mkdir(parents=True, exist_ok=True) @@ -85,6 +87,17 @@ class _TestItem(BuildItem): super().__init__(director, item, BuildItemMapper(item, recursive=True)) +class _TestRunner(TestRunner): + + def run_tests(self, executables: List[Executable]) -> List[Report]: + logging.info("executables: %s", executables) + super().run_tests(executables) + return [{ + "executable": executable.path, + "executable-sha512": executable.digest + } for executable in executables] + + class _Subprocess(NamedTuple): stdout: bytes @@ -349,3 +362,20 @@ def test_packagebuild(caplog, tmpdir, monkeypatch): "start-time": "f" }] + + # Test RunTests + variant["enabled"] = ["run-tests"] + factory.add_constructor("qdp/test-runner/test", _TestRunner) + build_bsp = director["/qdp/build/bsp"] + build_bsp.load() + director.build_package(None, None) + log = get_and_clear_log(caplog) + assert (f"executables: [Executable(path='{build_bsp.directory}" + "/a.exe', digest='z4PhNX7vuL3xVChQ1m2AB9Yg5AULVxXcg_SpIdNs6c5H0NE8" + "XYXysP-DGNKHfuwvY7kxvUdBeoGlODJ6-SfaPg==', timeout=1800), " + f"Executable(path='{build_bsp.directory}/b.exe', " + "digest='hopqxuHQKT10-tB_bZWVKz4B09MVPbZ3p12Ad5g_1OMNtr_Im3YIqT-yZ" + "GkjOp8aCVctaHqcXaeLID6xUQQKFQ==', timeout=1800)]") in log + director.build_package(None, ["/qdp/steps/run-tests"]) + log = get_and_clear_log(caplog) + assert f"use previous report for: {build_bsp.directory}/a.exe" diff --git a/spec-qdp/spec/qdp-input-generic-role.yml b/spec-qdp/spec/qdp-input-generic-role.yml index c69157d1..ff5ab32d 100644 --- a/spec-qdp/spec/qdp-input-generic-role.yml +++ b/spec-qdp/spec/qdp-input-generic-role.yml @@ -29,6 +29,10 @@ links: spec-key: name spec-value: member uid: qdp-input-role +- role: spec-refinement + spec-key: name + spec-value: runner + uid: qdp-input-role - role: spec-refinement spec-key: name spec-value: section diff --git a/spec-qdp/spec/qdp-run-tests.yml b/spec-qdp/spec/qdp-run-tests.yml new file mode 100644 index 00000000..4a08043a --- /dev/null +++ b/spec-qdp/spec/qdp-run-tests.yml @@ -0,0 +1,22 @@ +SPDX-License-Identifier: CC-BY-SA-4.0 OR BSD-2-Clause +copyrights: +- Copyright (C) 2022 embedded brains GmbH & Co. KG +enabled-by: true +links: +- role: spec-member + uid: root +- role: spec-refinement + spec-key: build-step-type + spec-value: run-tests + uid: qdp-build-step +spec-description: null +spec-example: null +spec-info: + dict: + attributes: {} + description: | + This set of attributes specifies a run of tests. + mandatory-attributes: all +spec-name: Run Tests Item Type +spec-type: qdp-run-tests +type: spec diff --git a/spec-qdp/spec/qdp-test-log.yml b/spec-qdp/spec/qdp-test-log.yml new file mode 100644 index 00000000..368a3c7c --- /dev/null +++ b/spec-qdp/spec/qdp-test-log.yml @@ -0,0 +1,22 @@ +SPDX-License-Identifier: CC-BY-SA-4.0 OR BSD-2-Clause +copyrights: +- Copyright (C) 2023 embedded brains GmbH & Co. KG +enabled-by: true +links: +- role: spec-member + uid: root +- role: spec-refinement + spec-key: directory-state-type + spec-value: test-log + uid: qdp-directory-state +spec-description: null +spec-example: null +spec-info: + dict: + attributes: {} + description: | + This set of attributes specifies a test log. + mandatory-attributes: all +spec-name: Test Log Item Type +spec-type: qdp-test-log +type: spec diff --git a/spec-qdp/spec/qdp-test-procedure-role.yml b/spec-qdp/spec/qdp-test-procedure-role.yml new file mode 100644 index 00000000..84a3b88c --- /dev/null +++ b/spec-qdp/spec/qdp-test-procedure-role.yml @@ -0,0 +1,23 @@ +SPDX-License-Identifier: CC-BY-SA-4.0 OR BSD-2-Clause +copyrights: +- Copyright (C) 2023 embedded brains GmbH & Co. KG +enabled-by: true +links: +- role: spec-member + uid: root +- role: spec-refinement + spec-key: role + spec-value: test-procedure + uid: link +spec-description: null +spec-example: null +spec-info: + dict: + attributes: {} + description: | + It defines the test procedure role of links and is used to associate a + test procedure with a variant. + mandatory-attributes: all +spec-name: Test Procedure Link Role +spec-type: qdp-test-procedure-role +type: spec -- cgit v1.2.3