From 6d96924f5b3e5cbfa1d416543d33e45e7eb72176 Mon Sep 17 00:00:00 2001 From: Sebastian Huber Date: Tue, 21 Nov 2023 11:13:16 +0100 Subject: qdp: Add scripts to build a QDP --- Makefile | 56 +++ config/base.yml | 146 ++++++++ config/variant-sparc-gr712rc-smp.yml | 45 +++ config/variant-sparc-gr712rc-uni.yml | 44 +++ config/variant-sparc-gr740-smp.yml | 45 +++ config/variant-sparc-gr740-uni.yml | 44 +++ qdp_workspace.py | 526 +++++++++++++++++++++++++++++ rtemsspec/tests/test_util.py | 18 +- rtemsspec/util.py | 20 ++ spec-qdp/qdp/deployment/archive.yml | 13 + spec-qdp/qdp/deployment/verify-package.yml | 13 + spec-qdp/qdp/package-build.yml | 9 + spec-qdp/qdp/steps/archive.yml | 19 ++ spec-qdp/spec/qdp-dummy.yml | 22 ++ spec-qdp/spec/qdp-uuid.yml | 26 ++ workspace/.gitignore | 3 + 16 files changed, 1045 insertions(+), 4 deletions(-) create mode 100644 config/base.yml create mode 100644 config/variant-sparc-gr712rc-smp.yml create mode 100644 config/variant-sparc-gr712rc-uni.yml create mode 100644 config/variant-sparc-gr740-smp.yml create mode 100644 config/variant-sparc-gr740-uni.yml create mode 100755 qdp_workspace.py create mode 100644 spec-qdp/qdp/deployment/archive.yml create mode 100644 spec-qdp/qdp/deployment/verify-package.yml create mode 100644 spec-qdp/qdp/package-build.yml create mode 100644 spec-qdp/qdp/steps/archive.yml create mode 100644 spec-qdp/spec/qdp-dummy.yml create mode 100644 spec-qdp/spec/qdp-uuid.yml create mode 100644 workspace/.gitignore diff --git a/Makefile b/Makefile index 6627d317..22f9e3c0 100644 --- a/Makefile +++ b/Makefile @@ -32,3 +32,59 @@ env: . env/bin/activate && pip install --upgrade pip && pip install -r requirements.txt echo -e "#!/bin/sh\n$$(which python3-config) "'$$@' > env/bin/python3-config chmod +x env/bin/python3-config + +PREFIX = /opt/rtems + +PACKAGE_VERSION = 0 + +RTEMS_API = 6 + +LOG_LEVEL = DEBUG + +GR712RC_SMP_PKG = $(PREFIX)/rtems-$(RTEMS_API)-sparc-gr712rc-smp-$(PACKAGE_VERSION) + +GR712RC_SMP_LOG = $(GR712RC_SMP_PKG)-log.txt + +gr712rc-smp-clean: + rm -rf $(GR712RC_SMP_PKG) $(GR712RC_SMP_LOG) + +gr712rc-smp-update: + ./qdp_workspace.py --prefix $(PREFIX) --log-file=$(GR712RC_SMP_LOG) --log-level=$(LOG_LEVEL) config/base.yml config/variant-sparc-gr712rc-smp.yml + +gr712rc-smp-new: gr712rc-smp-clean gr712rc-smp-update + +GR712RC_UNI_PKG = $(PREFIX)/rtems-$(RTEMS_API)-sparc-gr712rc-uni-$(PACKAGE_VERSION) + +GR712RC_UNI_LOG = $(GR712RC_UNI_PKG)-log.txt + +gr712rc-uni-clean: + rm -rf $(GR712RC_UNI_PKG) $(GR712RC_UNI_LOG) + +gr712rc-uni-update: + ./qdp_workspace.py --prefix $(PREFIX) --log-file=$(GR712RC_UNI_LOG) --log-level=$(LOG_LEVEL) config/base.yml config/variant-sparc-gr712rc-uni.yml + +gr712rc-uni-new: gr712rc-uni-clean gr712rc-uni-update + +GR740_SMP_PKG = $(PREFIX)/rtems-$(RTEMS_API)-sparc-gr740-smp-$(PACKAGE_VERSION) + +GR740_SMP_LOG = $(GR740_SMP_PKG)-log.txt + +gr740-smp-clean: + rm -rf $(GR740_SMP_PKG) $(GR740_SMP_LOG) + +gr740-smp-update: + ./qdp_workspace.py --prefix $(PREFIX) --log-file=$(GR740_SMP_LOG) --log-level=$(LOG_LEVEL) config/base.yml config/variant-sparc-gr740-smp.yml + +gr740-smp-new: gr740-smp-clean gr740-smp-update + +GR740_UNI_PKG = $(PREFIX)/rtems-$(RTEMS_API)-sparc-gr740-uni-$(PACKAGE_VERSION) + +GR740_UNI_LOG = $(GR740_UNI_PKG)-log.txt + +gr740-uni-clean: + rm -rf $(GR740_UNI_PKG) $(GR740_UNI_LOG) + +gr740-uni-update: + ./qdp_workspace.py --prefix $(PREFIX) --log-file=$(GR740_UNI_LOG) --log-level=$(LOG_LEVEL) config/base.yml config/variant-sparc-gr740-uni.yml + +gr740-uni-new: gr740-uni-clean gr740-uni-update diff --git a/config/base.yml b/config/base.yml new file mode 100644 index 00000000..b01672ba --- /dev/null +++ b/config/base.yml @@ -0,0 +1,146 @@ +workspace-actions: +- action-name: base-load-items + action-type: load-items + action-when: 1000 + enabled-by: true + paths: + - ${.:/toolchain-directory}/spec-spec + - ${.:/toolchain-directory}/spec-glossary + - ${.:/toolchain-directory}/spec-qdp + - ${.:/toolchain-directory}/spec + set-types: + - type: qdp/variant + uid: /qdp/variant +- action-name: base-deployment-directory + action-type: make-deployment-directory + action-when: 3000 + enabled-by: true +- action-name: base-workspace-items-load + action-type: load-workspace-items + action-when: 3000 + enabled-by: true + path: ${/qdp/variant:/build-directory}/spec + set-types: + - type: qdp/variant + uid: /qdp/variant +- action-name: base-make-uuid + action-type: make-uuid-item + action-when: 3000 + enabled-by: true + uid: /qdp/uuid +- action-name: base-gitignore + action-type: copy-directory + action-when: 4000 + copyrights-by-license: {} + destination-directory: ${../variant:/deployment-directory} + enabled-by: true + files: + - file: .gitignore + hash: null + links: [] + patterns: [] + source-directory: ${.:/toolchain-directory}/workspace + uid: /qdp/source/gitignore +- action-name: qt-modules + action-type: copy-directory + action-when: 4000 + copyrights-by-license: {} + destination-directory: ${../variant:/build-directory} + enabled-by: true + files: [] + links: [] + patterns: + - exclude: + - '*/.*' + include: rtemsspec/*.py + - exclude: [] + include: qdp_build.py + source-directory: ${.:/toolchain-directory} + uid: /qdp/source/qt-modules +- action-name: base-rtems + action-type: git-clone + action-when: 4000 + branch: qdp + commit: 42c9cdf35f6aa27f41d20b9b170d6e4e83a76913 + copyrights-by-license: + description: | + RTEMS and all third-party software distributed with RTEMS which may be + linked to the application is licensed under permissive open source + licenses. This means that the licenses do not propagate to the + application software. Most of the original RTEMS code is now under the + BSD-2-Clause license. Some code of RTEMS is under a legacy license, the + modified GPL-2.0 or later license with an exception for static linking. + It exposes no license requirements on application code. RTMES is a + collection of software from several sources. Each file may have its own + copyright/license that is embedded in the source file. + files: + - LICENSE + - LICENSE.Apache-2.0 + - LICENSE.BSD-2-Clause + - LICENSE.BSD-3-Clause + - LICENSE.CC-BY-SA-4.0 + - LICENSE.Freescale + - LICENSE.GPL-2.0 + - LICENSE.JFFS2 + - LICENSE.LLVM + description: | + This repository contains the RTEMS sources. It is used to provide the BSPs + shipped with the QDP. + destination-directory: ${../variant:/deployment-directory}/src/rtems + directory-state-invalidates: [] + enabled-by: true + links: + - role: repository + uid: ../variant + - hash: null + name: member + role: input-to + uid: ../steps/archive + origin-branch: master + origin-commit: 71c024eaca2b16c32447a0d9d712310717d17af8 + origin-commit-url: https://git.rtems.org/rtems/commit/?id=${.:/origin-commit} + origin-fetch: [] + origin-url: git://git.rtems.org/rtems.git + post-clone-commands: [] + source-directory: ${.:/toolchain-directory}/modules/rtems + uid: /qdp/source/rtems +- action-name: base-rtems-load-spec + action-type: load-items + action-when: 4000 + enabled-by: true + paths: + - ${/qdp/variant:/deployment-directory}/src/rtems/spec + set-types: [] +- action-name: base-rtems-docs + action-type: git-clone + action-when: 4000 + branch: qdp + commit: 2c88912893ebbcc3b9fa14d4fcc100c42252d0df + copyrights-by-license: {} + description: | + This repository contains the RTEMS Documentation sources. It is used to + provide the RTEMS Documentation shipped with the QDP. + destination-directory: ${../variant:/deployment-directory}/src/rtems-docs + directory-state-invalidates: [] + enabled-by: true + links: + - role: repository + uid: ../variant + - hash: null + name: member + role: input-to + uid: ../steps/archive + origin-branch: master + origin-commit: 2c88912893ebbcc3b9fa14d4fcc100c42252d0df + origin-commit-url: https://git.rtems.org/rtems-docs/commit/?id=${.:/origin-commit} + origin-fetch: [] + origin-url: git://git.rtems.org/rtems-docs.git + post-clone-commands: [] + source-directory: ${.:/toolchain-directory}/modules/rtems-docs + uid: /qdp/source/rtems-docs +- action-name: base-workspace-items-finalize + action-type: finalize-workspace-items + action-when: 6000 + enabled-by: true + spec-type-root-uid: /spec/root + verify: true diff --git a/config/variant-sparc-gr712rc-smp.yml b/config/variant-sparc-gr712rc-smp.yml new file mode 100644 index 00000000..384e0666 --- /dev/null +++ b/config/variant-sparc-gr712rc-smp.yml @@ -0,0 +1,45 @@ +workspace-actions: +- action-name: sparc-gr712rc-smp + action-type: make-item + action-when: 500 + data: + SPDX-License-Identifier: CC-BY-SA-4.0 OR BSD-2-Clause + arch: sparc + bsp: gr712rc + bsp-family: leon3 + build-directory: ${.:/deployment-directory}/build + config: smp + copyrights: + - Copyright (C) 2020, 2023 embedded brains GmbH & Co. KG + deployment-directory: ${.:/prefix-directory}/${.:/package-directory} + enabled: + - ${.:/arch} + - bsps/${.:/arch}/${.:/bsp-family} + - ${.:/arch}/${.:/bsp} + - RTEMS_QUAL + - RTEMS_SMP + - __GNUC__ + - target/evaluation-board + - target-hash/cpI09Ju6orF2eoJcmJi4igeIarypsRNwUxTrZSs9LMg= + - target/simulator + - target-hash/qYOFDHUGg5--JyB28V7llk_t6WYeA3VAogeqwGLZeCM= + enabled-by: true + ident: ${.:/arch}/${.:/bsp}${.:/config/slash}/${.:/package-version} + links: + - role: package-build + uid: package-build + name: ${.:/arch}-${.:/bsp}${.:/config/dash}-${.:/package-version} + package-directory: rtems-${.:/rtems-version}-${.:/name} + package-version: '0' + params: + makefile-run-command: sparc-rtems$$(RTEMS_API)-sis -${.:sis-target} -extirq + ${.:sis-extirq} -dumbio -r $$< + sis-cpus: '2' + sis-extirq: '12' + sis-target: leon3 + prefix-directory: /opt/rtems + qdp-type: variant + rtems-version: '6' + type: qdp + enabled-by: true + uid: /qdp/variant diff --git a/config/variant-sparc-gr712rc-uni.yml b/config/variant-sparc-gr712rc-uni.yml new file mode 100644 index 00000000..b1ee37f3 --- /dev/null +++ b/config/variant-sparc-gr712rc-uni.yml @@ -0,0 +1,44 @@ +workspace-actions: +- action-name: sparc-gr712rc-uni + action-type: make-item + action-when: 500 + data: + SPDX-License-Identifier: CC-BY-SA-4.0 OR BSD-2-Clause + arch: sparc + bsp: gr712rc + bsp-family: leon3 + build-directory: ${.:/deployment-directory}/build + config: uni + copyrights: + - Copyright (C) 2020, 2023 embedded brains GmbH & Co. KG + deployment-directory: ${.:/prefix-directory}/${.:/package-directory} + enabled: + - ${.:/arch} + - bsps/${.:/arch}/${.:/bsp-family} + - ${.:/arch}/${.:/bsp} + - RTEMS_QUAL + - __GNUC__ + - target/evaluation-board + - target-hash/cpI09Ju6orF2eoJcmJi4igeIarypsRNwUxTrZSs9LMg= + - target/simulator + - target-hash/qYOFDHUGg5--JyB28V7llk_t6WYeA3VAogeqwGLZeCM= + enabled-by: true + ident: ${.:/arch}/${.:/bsp}${.:/config/slash}/${.:/package-version} + links: + - role: package-build + uid: package-build + name: ${.:/arch}-${.:/bsp}${.:/config/dash}-${.:/package-version} + package-directory: rtems-${.:/rtems-version}-${.:/name} + package-version: '0' + params: + makefile-run-command: sparc-rtems$$(RTEMS_API)-sis -${.:sis-target} -extirq + ${.:sis-extirq} -dumbio -r $$< + sis-cpus: '1' + sis-extirq: '12' + sis-target: leon3 + prefix-directory: /opt/rtems + qdp-type: variant + rtems-version: '6' + type: qdp + enabled-by: true + uid: /qdp/variant diff --git a/config/variant-sparc-gr740-smp.yml b/config/variant-sparc-gr740-smp.yml new file mode 100644 index 00000000..bd2ce838 --- /dev/null +++ b/config/variant-sparc-gr740-smp.yml @@ -0,0 +1,45 @@ +workspace-actions: +- action-name: sparc-gr740-smp + action-type: make-item + action-when: 500 + data: + SPDX-License-Identifier: CC-BY-SA-4.0 OR BSD-2-Clause + arch: sparc + bsp: gr740 + bsp-family: leon3 + build-directory: ${.:/deployment-directory}/build + config: smp + copyrights: + - Copyright (C) 2020, 2023 embedded brains GmbH & Co. KG + deployment-directory: ${.:/prefix-directory}/${.:/package-directory} + enabled: + - ${.:/arch} + - bsps/${.:/arch}/${.:/bsp-family} + - ${.:/arch}/${.:/bsp} + - RTEMS_QUAL + - RTEMS_SMP + - __GNUC__ + - target/evaluation-board + - target-hash/c1ZkBOsUIJ-siPI7pK7knk0z6uni1pxOFlZ2eLDflYc= + - target/simulator + - target-hash/_xQeTNJwSla2bVbhWPVcI0emLk2bE_GVQfvzt9CN84k= + enabled-by: true + ident: ${.:/arch}/${.:/bsp}${.:/config/slash}/${.:/package-version} + links: + - role: package-build + uid: package-build + name: ${.:/arch}-${.:/bsp}${.:/config/dash}-${.:/package-version} + package-directory: rtems-${.:/rtems-version}-${.:/name} + package-version: '0' + params: + makefile-run-command: sparc-rtems$$(RTEMS_API)-sis -${.:sis-target} -extirq + ${.:sis-extirq} -dumbio -r $$< + sis-cpus: '4' + sis-extirq: '10' + sis-target: gr740 + prefix-directory: /opt/rtems + qdp-type: variant + rtems-version: '6' + type: qdp + enabled-by: true + uid: /qdp/variant diff --git a/config/variant-sparc-gr740-uni.yml b/config/variant-sparc-gr740-uni.yml new file mode 100644 index 00000000..9433818e --- /dev/null +++ b/config/variant-sparc-gr740-uni.yml @@ -0,0 +1,44 @@ +workspace-actions: +- action-name: sparc-gr740-uni + action-type: make-item + action-when: 500 + data: + SPDX-License-Identifier: CC-BY-SA-4.0 OR BSD-2-Clause + arch: sparc + bsp: gr740 + bsp-family: leon3 + build-directory: ${.:/deployment-directory}/build + config: uni + copyrights: + - Copyright (C) 2020, 2023 embedded brains GmbH & Co. KG + deployment-directory: ${.:/prefix-directory}/${.:/package-directory} + enabled: + - ${.:/arch} + - bsps/${.:/arch}/${.:/bsp-family} + - ${.:/arch}/${.:/bsp} + - RTEMS_QUAL + - __GNUC__ + - target/evaluation-board + - target-hash/c1ZkBOsUIJ-siPI7pK7knk0z6uni1pxOFlZ2eLDflYc= + - target/simulator + - target-hash/_xQeTNJwSla2bVbhWPVcI0emLk2bE_GVQfvzt9CN84k= + enabled-by: true + ident: ${.:/arch}/${.:/bsp}${.:/config/slash}/${.:/package-version} + links: + - role: package-build + uid: package-build + name: ${.:/arch}-${.:/bsp}${.:/config/dash}-${.:/package-version} + package-directory: rtems-${.:/rtems-version}-${.:/name} + package-version: '0' + params: + makefile-run-command: sparc-rtems$$(RTEMS_API)-sis -${.:sis-target} -extirq + ${.:sis-extirq} -dumbio -r $$< + sis-cpus: '1' + sis-extirq: '10' + sis-target: gr740 + prefix-directory: /opt/rtems + qdp-type: variant + rtems-version: '6' + type: qdp + enabled-by: true + uid: /qdp/variant diff --git a/qdp_workspace.py b/qdp_workspace.py new file mode 100755 index 00000000..0f38429a --- /dev/null +++ b/qdp_workspace.py @@ -0,0 +1,526 @@ +#!/usr/bin/env python +# SPDX-License-Identifier: BSD-2-Clause +""" Creates a QDP workspace directory according to the configuration file. """ + +# Copyright (C) 2020, 2023 embedded brains GmbH & Co. KG +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. + +import copy +import logging +import os +import subprocess +import sys +from typing import Any, Callable, Dict, List, Optional, Set +import uuid + +from rtemsspec.items import Item, ItemCache, JSONItemCache, is_enabled +from rtemsspec.directorystate import DirectoryState +from rtemsspec.packagebuild import BuildItem, BuildItemFactory, \ + PackageBuildDirector +from rtemsspec.packagebuildfactory import create_build_item_factory +from rtemsspec.specverify import verify +from rtemsspec.util import create_build_argument_parser, hash_file, \ + init_logging, load_config, run_command + + +class _ConfigItem(BuildItem): + + def __init__(self, director: PackageBuildDirector, item: Item, + factory: BuildItemFactory): + super().__init__(director, item) + workspace_cache_config: Dict[str, Any] = { + "enabled": [], + "initialize-links": False, + "paths": [], + "resolve-proxies": False, + "spec-type-root-uid": None + } + self.workspace_cache = JSONItemCache(workspace_cache_config) + self.workspace_director = PackageBuildDirector(self.workspace_cache, + factory) + + def is_enabled(self, enabled_by: Any) -> bool: + """ + Returns true, if the enabled by expression evaluates to true for the + enabled set of the configuration item, otherwise returns false. + """ + try: + enabled_set = self.enabled_set + except KeyError: + enabled_set = [] + return is_enabled(self.substitute(enabled_set), enabled_by) + + def set_file(self, item: Item) -> None: + """ Sets the file of the item. """ + file = os.path.join(self["spec-directory"], f"{item.uid[1:]}.json") + os.makedirs(os.path.dirname(file), exist_ok=True) + item.file = file + + def try_save(self, build_item: BuildItem, _reason: str) -> None: + """ Tries to set the file of the item and save it. """ + if "spec-directory" in self: + self.set_file(build_item.item) + build_item.item.save() + self.item.cache[build_item.uid].data.clear() + self.item.cache[build_item.uid].data.update(build_item.item.data) + else: + logging.info("%s: cannot save item", build_item.uid) + + +def _create_item( + config: _ConfigItem, + action: Dict[str, Any], + data: Dict[str, Any], + type_name: str, + prepare: Optional[Callable[[_ConfigItem, Dict[str, Any]], None]] = None +) -> Optional[BuildItem]: + uid = action["uid"] + item = config.item.cache.create_item(uid, data) + item["_type"] = type_name + if prepare is not None: + prepare(config, action) + workspace_item = config.workspace_cache.create_item( + uid, copy.deepcopy(data)) + workspace_item["_type"] = type_name + build_item = config.workspace_director[uid] + return build_item + + +def _make_root_data(config: _ConfigItem, type_name: str) -> Any: + return { + "SPDX-License-Identifier": config.variant["SPDX-License-Identifier"], + "copyrights": config.variant["copyrights"], + "enabled-by": True, + "links": [], + "qdp-type": type_name, + "type": "qdp" + } + + +def _make_directory_state_data(config: _ConfigItem, directory: str, + directory_state_type: str) -> Any: + data = _make_root_data(config, "directory-state") + data["copyrights-by-license"] = {} + data["directory"] = directory + data["directory-state-type"] = directory_state_type + data["files"] = [] + data["hash"] = None + data["patterns"] = [] + return data + + +def _load_directory(config: _ConfigItem, action: Dict[str, Any]) -> None: + directory_state = config.director[action["uid"]] + assert isinstance(directory_state, DirectoryState) + directory_state.load() + directory_state["patterns"] = [] + + +def _action_copy_directory(config: _ConfigItem, action: Dict[str, + Any]) -> None: + source_directory = config.substitute(action["source-directory"]) + data = _make_directory_state_data(config, source_directory, "generic") + data["copyrights-by-license"] = action["copyrights-by-license"] + data["patterns"] = action["patterns"] + data["files"] = action["files"] + data["links"] = action["links"] + workspace_directory_state = _create_item(config, action, data, + "qdp/directory-state/generic", + _load_directory) + assert isinstance(workspace_directory_state, DirectoryState) + directory_state = config.director[action["uid"]] + assert isinstance(directory_state, DirectoryState) + workspace_directory_state["directory"] = action["destination-directory"] + workspace_directory_state.clear() + workspace_directory_state.lazy_clone(directory_state) + config.try_save(workspace_directory_state, "Update directory state") + + +def _copy_file(config: _ConfigItem, action: Dict[str, Any], + the_type: str) -> None: + source_file = config.substitute(action["source-file"]) + data = _make_directory_state_data(config, os.path.dirname(source_file), + the_type) + data["files"] = [{"file": os.path.basename(source_file), "hash": None}] + data["links"] = action["links"] + workspace_directory_state = _create_item( + config, action, data, f"qdp/directory-state/{the_type}", + _load_directory) + assert isinstance(workspace_directory_state, DirectoryState) + directory_state = config.director[action["uid"]] + assert isinstance(directory_state, DirectoryState) + workspace_directory_state["directory"] = action["destination-directory"] + workspace_directory_state.clear() + workspace_directory_state.copy_file(source_file, + action["destination-file"]) + workspace_directory_state.load() + config.try_save(workspace_directory_state, "Update directory state") + + +def _action_copy_file(config: _ConfigItem, action: Dict[str, Any]) -> None: + _copy_file(config, action, "generic") + + +def _action_copy_test_log(config: _ConfigItem, action: Dict[str, Any]) -> None: + _copy_file(config, action, "test-log") + + +def _git_clone(config: _ConfigItem, action: Dict[str, Any], + repository: DirectoryState) -> None: + source_directory = config.substitute(action["source-directory"]) + status = run_command(["git", "fetch", "origin"], source_directory) + assert status == 0 + branch = action["branch"] + commit = action["commit"] + status = run_command(["git", "branch", "-f", branch, commit], + source_directory) + assert status == 0 + destination_directory = repository.directory + status = run_command([ + "git", "clone", "--branch", branch, "--single-branch", + f"file://{source_directory}", destination_directory + ], source_directory) + assert status == 0 + origin_url = action["origin-url"] + if origin_url: + status = run_command( + ["git", "remote", "add", "tmp", + config.substitute(origin_url)], destination_directory) + assert status == 0 + status = run_command(["git", "remote", "remove", "origin"], + destination_directory) + assert status == 0 + status = run_command(["git", "remote", "rename", "tmp", "origin"], + destination_directory) + assert status == 0 + for fetch in action["origin-fetch"]: + status = run_command( + ["git", "fetch", "origin", + config.substitute(fetch)], destination_directory) + assert status == 0 + origin_branch = action["origin-branch"] + origin_commit = action["origin-commit"] + if origin_branch and origin_commit: + status = run_command( + ["git", "checkout", "-b", origin_branch, origin_commit], + destination_directory) + assert status == 0 + status = run_command(["git", "checkout", branch], + destination_directory) + assert status == 0 + status = run_command([ + "git", "symbolic-ref", "refs/remotes/origin/HEAD", + f"refs/remotes/origin/{origin_branch}" + ], destination_directory) + assert status == 0 + for command in action["post-clone-commands"]: + status = run_command(config.substitute(command), destination_directory) + assert status == 0 + repository.load() + config.try_save(repository, "Clone Git repository") + + +def _action_git_clone(config: _ConfigItem, action: Dict[str, Any]) -> None: + data = _make_directory_state_data(config, action["destination-directory"], + "repository") + data["patterns"] = [{"include": "**/*", "exclude": []}] + for key in [ + "branch", "commit", "copyrights-by-license", "description", + "links", "origin-branch", "origin-commit", "origin-commit-url", + "origin-url" + ]: + data[key] = action[key] + repository = _create_item(config, action, data, + "qdp/directory-state/repository") + assert isinstance(repository, DirectoryState) + destination_directory = repository.directory + assert not os.path.exists(destination_directory) + _git_clone(config, action, repository) + + +def _add_item(config: _ConfigItem, action: Dict[str, Any], + data: Dict[str, Any]) -> None: + config.item.cache.create_item(action["uid"], data) + + +def _action_add_item(config: _ConfigItem, action: Dict[str, Any]) -> None: + source_file = config.substitute(action["source"]) + data = config.item.cache.load_data(source_file, action["uid"]) + _add_item(config, action, data) + + +def _action_make_item(config: _ConfigItem, action: Dict[str, Any]) -> None: + _add_item(config, action, action["data"]) + + +def _action_make_uuid_item(config: _ConfigItem, action: Dict[str, + Any]) -> None: + data = _make_root_data(config, "uuid") + data["uuid"] = str(uuid.uuid4()) + _add_item(config, action, data) + + +def _set_types(item_cache: ItemCache, action: Dict[str, Any]) -> None: + for set_type in action["set-types"]: + item_cache[set_type["uid"]]["_type"] = set_type["type"] + + +def _action_load_items(config: _ConfigItem, action: Dict[str, Any]) -> None: + action_name = action["action-name"] + item_cache = config.item.cache + for path in action["paths"]: + path = config.substitute(path) + logging.info("%s: load items from: %s", action_name, path) + item_cache.load_items(path) + _set_types(item_cache, action) + config.director.clear() + config.workspace_director.clear() + + +def _new_workspace_items(config: _ConfigItem, action_name: str, + new: Set[str]) -> None: + for uid in sorted(new): + logging.debug("%s: new item: %s", action_name, uid) + data = config.item.cache[uid].data + item = config.workspace_cache.create_item(uid, copy.deepcopy(data)) + item["_type"] = data["_type"] + config.set_file(item) + item.save() + + +def _action_load_workspace_items(config: _ConfigItem, + action: Dict[str, Any]) -> None: + action_name = action["action-name"] + path = config.substitute(action["path"]) + config["spec-directory"] = path + config.variant.item["enabled"] = config.variant["enabled"] + logging.info("%s: add workspace items to: %s", action_name, path) + new = set(config.item.cache.keys()) + _new_workspace_items(config, action_name, new) + _set_types(config.workspace_cache, action) + + +def _action_finalize_workspace_items(config: _ConfigItem, + action: Dict[str, Any]) -> None: + action_name = action["action-name"] + new = set(config.item.cache.keys()) + _new_workspace_items(config, action_name, new) + logging.info("%s: initialize workspace item links", action_name) + config.workspace_cache.initialize_links() + enabled_set = config.variant["enabled"] + logging.info("%s: set workspace enabled: %s", action_name, enabled_set) + config.workspace_cache.set_enabled(enabled_set) + logging.info("%s: set workspace item types", action_name) + config.workspace_cache.set_types(action["spec-type-root-uid"]) + config.workspace_director.clear() + if action["verify"] and config["spec-verify"]: + logging.info("%s: verify workspace items", action_name) + logger = logging.getLogger() + level = logger.getEffectiveLevel() + logger.setLevel(logging.ERROR) + verify_config = {"root-type": action["spec-type-root-uid"]} + status = verify(verify_config, config.workspace_cache) + assert status.critical == 0 + assert status.error == 0 + logger.setLevel(level) + logging.debug("%s: finished verifying workspace items", action_name) + + +def _action_make_deployment_directory(config: _ConfigItem, + _action: Dict[str, Any]) -> None: + deployment_directory = config.variant["deployment-directory"] + assert not os.path.exists(deployment_directory) + logging.info("workspace: create deployment directory: %s", + deployment_directory) + os.makedirs(deployment_directory) + + +def _create_symbolic_links(action: Dict[str, Any], + unpacked_archive: DirectoryState) -> None: + for symbolic_link in action["archive-symbolic-links"]: + link = unpacked_archive.substitute(symbolic_link["link"]) + link_path = os.path.join(unpacked_archive.directory, link) + target = unpacked_archive.substitute(symbolic_link["target"]) + target = os.path.relpath(target, os.path.dirname(link_path)) + logging.info("%s: create symbolic link from '%s' to '%s'", + action["action-name"], link_path, target) + os.symlink(target, link_path) + unpacked_archive.add_files([link]) + + +def _apply_patches(config: _ConfigItem, action: Dict[str, Any], + unpacked_archive: DirectoryState) -> None: + for patch in action["archive-patches"]: + if patch["type"] == "inline": + source = "-" + stdin = config.substitute(patch["patch"]).encode("utf-8") + logging.info("%s: apply inline patch: %s", action["action-name"], + stdin) + else: + assert patch["type"] == "file" + source = patch["file"] + stdin = None + logging.info("%s: apply patch: %s", action["action-name"], source) + command = [ + "git", "apply", "--apply", "--numstat", "--directory", + unpacked_archive.directory, "-p", "1", "--unsafe-paths", source + ] + output = subprocess.check_output(command, + stdin=stdin, + cwd=config["toolchain-directory"]) + unpacked_archive.add_files([ + line.decode("utf-8").split("\t")[2] + for line in output.splitlines() + ]) + + +def _action_unpack_archive(config: _ConfigItem, action: Dict[str, + Any]) -> None: + data = _make_directory_state_data(config, action["destination-directory"], + "unpacked-archive") + archive_file = config.substitute(action["archive-file"]) + data["archive-file"] = os.path.basename(archive_file) + for key in [ + "archive-hash", "archive-patches", "archive-symbolic-links", + "archive-url", "copyrights-by-license", "description", + "enabled-by", "links" + ]: + data[key] = action[key] + unpacked_archive = _create_item(config, action, data, + "qdp/directory-state/unpacked-archive") + assert isinstance(unpacked_archive, DirectoryState) + unpacked_archive.add_tarfile_members(archive_file, + unpacked_archive.directory, True) + assert hash_file(archive_file) == unpacked_archive["archive-hash"] + unpacked_archive.compact() + _create_symbolic_links(action, unpacked_archive) + _apply_patches(config, action, unpacked_archive) + unpacked_archive.load() + config.try_save(unpacked_archive, "Unpack archive") + + +def _create_dummy(config: _ConfigItem, action: Dict[str, Any]) -> None: + data = _make_root_data(config, "dummy") + data["enabled-by"] = action["enabled-by"] + dummy = _create_item(config, action, data, "qdp/dummy") + if dummy is None: + return + config.try_save(dummy, "Add dummy item") + + +_ACTIONS = { + "add-item": _action_add_item, + "copy-directory": _action_copy_directory, + "copy-file": _action_copy_file, + "copy-test-log": _action_copy_test_log, + "finalize-workspace-items": _action_finalize_workspace_items, + "git-clone": _action_git_clone, + "load-items": _action_load_items, + "load-workspace-items": _action_load_workspace_items, + "make-deployment-directory": _action_make_deployment_directory, + "make-item": _action_make_item, + "make-uuid-item": _action_make_uuid_item, + "unpack-archive": _action_unpack_archive +} + +_NEEDS_DUMMY = [ + "add-item", "copy-directory", "copy-file", "copy-test-log", "git-clone", + "make-item", "unpack-archive" +] + + +def _run_actions(config: _ConfigItem, when: int, + actions: List[Dict[str, Any]]) -> None: + logging.info("workspace: run actions at: %i", when) + for action in actions: + action_type = action["action-type"] + if config.is_enabled(action["enabled-by"]): + logging.info("%s: run action", action["action-name"]) + _ACTIONS[action_type](config, action) + else: + logging.info("%s: action is disabled", action["action-name"]) + if action_type in _NEEDS_DUMMY: + _create_dummy(config, action) + + +def _toolchain_commit(toolchain_directory: str) -> str: + stdout: List[str] = [] + status = run_command(["git", "rev-parse", "HEAD"], toolchain_directory, + stdout) + assert status == 0 + return stdout[0].strip() + + +def main(argv: List[str]) -> None: + """ + Creates a QDP workspace directory according to the configuration files. + """ + parser = create_build_argument_parser() + parser.add_argument("--prefix", + help="the prefix directory", + default="/tmp/qdp") + parser.add_argument("--cache-directory", + help="the config cache directory", + default="config-cache") + parser.add_argument('config_files', nargs='+') + args = parser.parse_args(argv) + init_logging(args) + config_cache_config: Dict[str, Any] = { + "cache-directory": os.path.abspath(args.cache_directory), + "enabled": [], + "resolve-proxies": False, + "initialize-links": False, + "paths": [], + "spec-type-root-uid": None, + } + config_cache = ItemCache(config_cache_config) + factory = create_build_item_factory() + director = PackageBuildDirector(config_cache, factory) + config_item = Item( + config_cache, "/qdp/config", { + "SPDX-License-Identifier": + "CC-BY-SA-4.0 OR BSD-2-Clause", + "copyrights": + ["Copyright (C) 2020, 2023 embedded brains GmbH & Co. KG"] + }) + config_item["_type"] = "config" + config = _ConfigItem(director, config_item, factory) + config["prefix-directory"] = os.path.abspath(args.prefix) + config["spec-verify"] = not args.no_spec_verify + toolchain_directory = os.path.abspath(os.path.dirname(__file__)) + logging.info("workspace: toolchain directory: %s", toolchain_directory) + config["toolchain-directory"] = toolchain_directory + config["toolchain-commit"] = _toolchain_commit(toolchain_directory) + config["enabled"] = [] + actions_at: Dict[int, List[Dict[str, Any]]] = {} + for file in args.config_files: + logging.info("workspace: load configuration from: %s", file) + for action in load_config(file)["workspace-actions"]: + actions_at.setdefault(action["action-when"], []).append(action) + for when, actions in sorted(actions_at.items()): + _run_actions(config, when, actions) + config.workspace_director.build_package(args.only, args.force) + + +if __name__ == "__main__": # pragma: no cover + main(sys.argv[1:]) diff --git a/rtemsspec/tests/test_util.py b/rtemsspec/tests/test_util.py index 3eab1641..0d4db2b3 100644 --- a/rtemsspec/tests/test_util.py +++ b/rtemsspec/tests/test_util.py @@ -28,7 +28,8 @@ import os import logging from rtemsspec.util import copy_file, copy_files, create_argument_parser, \ - base64_to_hex, init_logging, load_config, run_command + create_build_argument_parser, base64_to_hex, init_logging, load_config, \ + run_command from rtemsspec.tests.util import get_and_clear_log @@ -79,13 +80,22 @@ DEBUG A""" def test_args(): - parser = create_argument_parser() + parser = create_build_argument_parser() args = parser.parse_args([]) - init_logging(args) assert args.log_level == "INFO" assert args.log_file is None + assert args.only is None + assert args.force is None + assert not args.no_spec_verify + init_logging(args) log_file = "log.txt" - args = parser.parse_args(["--log-level=DEBUG", f"--log-file={log_file}"]) + args = parser.parse_args([ + "--log-level=DEBUG", f"--log-file={log_file}", "--only", "abc", + "--force", "def", "--no-spec-verify" + ]) assert args.log_level == "DEBUG" assert args.log_file == log_file + assert args.only == ["abc"] + assert args.force == ["def"] + assert args.no_spec_verify init_logging(args) diff --git a/rtemsspec/util.py b/rtemsspec/util.py index a1f46372..296c6137 100644 --- a/rtemsspec/util.py +++ b/rtemsspec/util.py @@ -188,6 +188,26 @@ def create_argument_parser( return parser +def create_build_argument_parser( + default_log_level: str = "INFO") -> argparse.ArgumentParser: + """ Creates an argument parser with default build options. """ + parser = create_argument_parser(default_log_level) + parser.add_argument('--only', + type=str, + nargs='*', + default=None, + help="build only these steps") + parser.add_argument('--force', + type=str, + nargs='*', + default=None, + help="force to build these steps") + parser.add_argument('--no-spec-verify', + action="store_true", + help="do not verify the specification") + return parser + + def init_logging(args: argparse.Namespace) -> None: """ Initializes the logging module. """ handlers: List[Any] = [logging.StreamHandler()] diff --git a/spec-qdp/qdp/deployment/archive.yml b/spec-qdp/qdp/deployment/archive.yml new file mode 100644 index 00000000..823071e8 --- /dev/null +++ b/spec-qdp/qdp/deployment/archive.yml @@ -0,0 +1,13 @@ +SPDX-License-Identifier: CC-BY-SA-4.0 OR BSD-2-Clause +copyrights: +- Copyright (C) 2020 embedded brains GmbH & Co. KG +copyrights-by-license: {} +directory: ${../variant:/build-directory} +directory-state-type: generic +enabled-by: true +files: [] +hash: null +links: [] +patterns: [] +qdp-type: directory-state +type: qdp diff --git a/spec-qdp/qdp/deployment/verify-package.yml b/spec-qdp/qdp/deployment/verify-package.yml new file mode 100644 index 00000000..34ca7d60 --- /dev/null +++ b/spec-qdp/qdp/deployment/verify-package.yml @@ -0,0 +1,13 @@ +SPDX-License-Identifier: CC-BY-SA-4.0 OR BSD-2-Clause +copyrights: +- Copyright (C) 2021 embedded brains GmbH & Co. KG +copyrights-by-license: {} +directory: ${../variant:/deployment-directory} +directory-state-type: generic +enabled-by: true +files: [] +hash: null +links: [] +patterns: [] +qdp-type: directory-state +type: qdp diff --git a/spec-qdp/qdp/package-build.yml b/spec-qdp/qdp/package-build.yml new file mode 100644 index 00000000..1e999798 --- /dev/null +++ b/spec-qdp/qdp/package-build.yml @@ -0,0 +1,9 @@ +SPDX-License-Identifier: CC-BY-SA-4.0 OR BSD-2-Clause +copyrights: +- Copyright (C) 2020 embedded brains GmbH & Co. KG +enabled-by: true +links: +- role: build-step + uid: steps/archive +qdp-type: package-build +type: qdp diff --git a/spec-qdp/qdp/steps/archive.yml b/spec-qdp/qdp/steps/archive.yml new file mode 100644 index 00000000..019542f7 --- /dev/null +++ b/spec-qdp/qdp/steps/archive.yml @@ -0,0 +1,19 @@ +SPDX-License-Identifier: CC-BY-SA-4.0 OR BSD-2-Clause +archive-file: rtems-${../variant:/rtems-version}-${../variant:/name}.tar.xz +archive-strip-prefix: ${../variant:/prefix-directory}/ +build-step-type: archive +copyrights: +- Copyright (C) 2020, 2021 embedded brains GmbH & Co. KG +description: | + Packs all deployed components into the archive file handed over to end users. +enabled-by: true +links: +- name: archive + role: output + uid: ../deployment/archive +- name: verify-package + role: output + uid: ../deployment/verify-package +qdp-type: build-step +type: qdp +verification-script: verify_package.py diff --git a/spec-qdp/spec/qdp-dummy.yml b/spec-qdp/spec/qdp-dummy.yml new file mode 100644 index 00000000..c1051559 --- /dev/null +++ b/spec-qdp/spec/qdp-dummy.yml @@ -0,0 +1,22 @@ +SPDX-License-Identifier: CC-BY-SA-4.0 OR BSD-2-Clause +copyrights: +- Copyright (C) 2023 embedded brains GmbH & Co. KG +enabled-by: true +links: +- role: spec-member + uid: root +- role: spec-refinement + spec-key: qdp-type + spec-value: dummy + uid: qdp-root +spec-description: null +spec-example: null +spec-info: + dict: + attributes: {} + description: | + This set of attributes specifies a dummy. + mandatory-attributes: all +spec-name: Dummy Item Type +spec-type: qdp-dummy +type: spec diff --git a/spec-qdp/spec/qdp-uuid.yml b/spec-qdp/spec/qdp-uuid.yml new file mode 100644 index 00000000..6cad9270 --- /dev/null +++ b/spec-qdp/spec/qdp-uuid.yml @@ -0,0 +1,26 @@ +SPDX-License-Identifier: CC-BY-SA-4.0 OR BSD-2-Clause +copyrights: +- Copyright (C) 2023 embedded brains GmbH & Co. KG +enabled-by: true +links: +- role: spec-member + uid: root +- role: spec-refinement + spec-key: qdp-type + spec-value: uuid + uid: qdp-root +spec-description: null +spec-example: null +spec-info: + dict: + attributes: + uuid: + description: | + It shall be the UUID. + spec-type: str + description: | + This set of attributes provides an UUID. + mandatory-attributes: all +spec-name: UUID Item Type +spec-type: qdp-uuid +type: spec diff --git a/workspace/.gitignore b/workspace/.gitignore new file mode 100644 index 00000000..ca53c44c --- /dev/null +++ b/workspace/.gitignore @@ -0,0 +1,3 @@ +b-* +*.pyc +__pycache__ -- cgit v1.2.3