From 5b516a7c8f0d05ccf77f44dd8882f8688bdd5887 Mon Sep 17 00:00:00 2001 From: Sebastian Huber Date: Tue, 21 Nov 2023 11:13:16 +0100 Subject: packagebuild: New --- rtemsspec/packagebuild.py | 412 +++++++++++++++++++++ rtemsspec/packagebuildfactory.py | 34 ++ rtemsspec/tests/spec-packagebuild/qdp/output/a.yml | 7 + rtemsspec/tests/spec-packagebuild/qdp/output/b.yml | 7 + .../tests/spec-packagebuild/qdp/package-build.yml | 13 + rtemsspec/tests/spec-packagebuild/qdp/steps/a.yml | 18 + rtemsspec/tests/spec-packagebuild/qdp/steps/b.yml | 18 + rtemsspec/tests/spec-packagebuild/qdp/steps/c.yml | 36 ++ rtemsspec/tests/spec-packagebuild/qdp/variant.yml | 23 ++ .../spec-packagebuild/spec/qdp-test-input.yml | 30 ++ .../spec-packagebuild/spec/qdp-test-output.yml | 22 ++ .../tests/spec-packagebuild/spec/qdp-test.yml | 28 ++ rtemsspec/tests/test_packagebuild.py | 160 ++++++++ spec-qdp/spec/qdp-build-step-role.yml | 23 ++ spec-qdp/spec/qdp-build-step.yml | 31 ++ spec-qdp/spec/qdp-input-generic-role.yml | 73 ++++ spec-qdp/spec/qdp-input-role.yml | 37 ++ spec-qdp/spec/qdp-optional-hash.yml | 21 ++ spec-qdp/spec/qdp-output-role.yml | 28 ++ spec-qdp/spec/qdp-package-build-role.yml | 23 ++ spec-qdp/spec/qdp-package-build.yml | 24 ++ spec-qdp/spec/qdp-root.yml | 26 ++ spec-qdp/spec/qdp-variant.yml | 88 +++++ 23 files changed, 1182 insertions(+) create mode 100644 rtemsspec/packagebuild.py create mode 100644 rtemsspec/packagebuildfactory.py create mode 100644 rtemsspec/tests/spec-packagebuild/qdp/output/a.yml create mode 100644 rtemsspec/tests/spec-packagebuild/qdp/output/b.yml create mode 100644 rtemsspec/tests/spec-packagebuild/qdp/package-build.yml create mode 100644 rtemsspec/tests/spec-packagebuild/qdp/steps/a.yml create mode 100644 rtemsspec/tests/spec-packagebuild/qdp/steps/b.yml create mode 100644 rtemsspec/tests/spec-packagebuild/qdp/steps/c.yml create mode 100644 rtemsspec/tests/spec-packagebuild/qdp/variant.yml create mode 100644 rtemsspec/tests/spec-packagebuild/spec/qdp-test-input.yml create mode 100644 rtemsspec/tests/spec-packagebuild/spec/qdp-test-output.yml create mode 100644 rtemsspec/tests/spec-packagebuild/spec/qdp-test.yml create mode 100644 rtemsspec/tests/test_packagebuild.py create mode 100644 spec-qdp/spec/qdp-build-step-role.yml create mode 100644 spec-qdp/spec/qdp-build-step.yml create mode 100644 spec-qdp/spec/qdp-input-generic-role.yml create mode 100644 spec-qdp/spec/qdp-input-role.yml create mode 100644 spec-qdp/spec/qdp-optional-hash.yml create mode 100644 spec-qdp/spec/qdp-output-role.yml create mode 100644 spec-qdp/spec/qdp-package-build-role.yml create mode 100644 spec-qdp/spec/qdp-package-build.yml create mode 100644 spec-qdp/spec/qdp-root.yml create mode 100644 spec-qdp/spec/qdp-variant.yml diff --git a/rtemsspec/packagebuild.py b/rtemsspec/packagebuild.py new file mode 100644 index 00000000..dd7db469 --- /dev/null +++ b/rtemsspec/packagebuild.py @@ -0,0 +1,412 @@ +# SPDX-License-Identifier: BSD-2-Clause +""" This module provides the basic support to build a QDP. """ + +# Copyright (C) 2021 EDISOFT (https://www.edisoft.pt/) +# Copyright (C) 2020, 2023 embedded brains GmbH & Co. KG +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. + +import copy +import itertools +import logging +import re +from typing import Any, Dict, Iterator, List, Optional, Tuple, Type + +from rtemsspec.items import data_digest, Item, ItemCache, \ + ItemGetValueContext, ItemGetValue, ItemMapper, is_enabled, \ + Link, link_is_enabled +from rtemsspec.sphinxcontent import SphinxMapper + +_SINGLE_SUBSTITUTION = re.compile( + r"^\${[a-zA-Z0-9._/-]+(:[a-zA-Z0-9._/-]+)?(:[^${}]*)?}$") + + +def _get_input_links(item: Item) -> Iterator[Link]: + yield from itertools.chain(item.links_to_parents("input"), + item.links_to_children("input-to")) + + +def build_item_input(item: Item, name: str) -> Item: + """ Returns the first input item with the name. """ + for link in _get_input_links(item): + if link["name"] == name: + return link.item + raise KeyError + + +def _get_spec(ctx: ItemGetValueContext) -> Any: + return ctx.item.spec + + +class BuildItemMapper(SphinxMapper): + """ + The build item mapper provides a method to get a link to the primary + documentation place of the item. + """ + + def __init__(self, item: Item, recursive: bool = False): + super().__init__(item, recursive) + for type_name in item.cache.items_by_type: + self.add_get_value(f"{type_name}:/spec", _get_spec) + + def get_link(self, _item: Item) -> str: + """ Returns a link to the primary documentation place of the item. """ + raise NotImplementedError + + +class BuildItem(): + """ This is the base class for build steps. """ + + # pylint: disable=too-many-public-methods + @classmethod + def prepare_factory(cls, _factory: "BuildItemFactory", + _type_name: str) -> None: + """ Prepares the build item factory for the type. """ + + def __init__(self, + director: "PackageBuildDirector", + item: Item, + mapper: Optional[BuildItemMapper] = None): + if mapper is None: + mapper = BuildItemMapper(item, recursive=True) + self.director = director + self.item = item + self.mapper = mapper + director.factory.add_get_values_to_mapper(self.mapper) + self._did_run = False + + def __contains__(self, key: str) -> bool: + return key in self.item + + def __getitem__(self, key: str) -> Any: + return self.substitute(self.item[key]) + + def __setitem__(self, key: str, value: Any) -> None: + self.item[key] = value + + @property + def uid(self) -> str: + """ Returns the UID of the build item. """ + return self.item.uid + + @property + def variant(self) -> "BuildItem": + """ Returns the variant build item. """ + return self.director["/qdp/variant"] + + @property + def enabled_set(self) -> List["str"]: + """ Is the enabled set of the variant item. """ + return self.director["/qdp/variant"]["enabled"] + + @property + def enabled(self) -> bool: + """ + Is true, if the build item is enabled using the enabled set of the + variant item, otherwise false. + """ + return is_enabled(self.enabled_set, self["enabled-by"]) + + def build(self, force: bool) -> None: + """ Runs the build if necessary. """ + self._did_run = False + self.mapper.item = self.item + logging.info("%s: check if build is necessary", self.uid) + necessary = self.is_build_necessary() + if necessary: + logging.info("%s: build is necessary", self.uid) + if force: + logging.info("%s: build is forced", self.uid) + if force or necessary: + logging.info("%s: discard outputs", self.uid) + self.discard_outputs() + logging.info("%s: run", self.uid) + self.run() + self._did_run = True + logging.info("%s: refresh outputs", self.uid) + self.refresh_outputs() + logging.info("%s: refresh input links", self.uid) + self.refresh_input_links() + logging.info("%s: refresh", self.uid) + self.refresh() + logging.info("%s: commit", self.uid) + self.commit("Finish build step") + logging.info("%s: finished", self.uid) + else: + logging.info("%s: build is unnecessary", self.uid) + + def has_changed(self, link: Link) -> bool: + """ + Returns true, if the build item state changed with respect to the state + of the link, otherwise false. + """ + return self._did_run or link["hash"] is None or self.digest != link[ + "hash"] + + def is_build_necessary(self) -> bool: + """ Returns true, if the build is necessary, otherwise false. """ + necessary = False + for link in itertools.chain( + self.item.links_to_parents("input", + is_link_enabled=link_is_enabled), + self.item.links_to_children("input-to", + is_link_enabled=link_is_enabled)): + if not link.item.is_enabled(self.enabled_set): + logging.info("%s: input is disabled: %s", self.uid, + link.item.uid) + continue + build_item = self.director[link.item.uid] + if link["hash"] is None: + logging.info("%s: input is new: %s", self.uid, build_item.uid) + if build_item.has_changed(link): + logging.info("%s: input has changed: %s", self.uid, + build_item.uid) + necessary = True + else: + logging.info("%s: input has not changed: %s", self.uid, + build_item.uid) + return necessary + + def discard(self) -> None: + """ Discards the data associated with the build item. """ + + def discard_outputs(self) -> None: + """ Discards all outputs of the build item. """ + for item in self.item.parents("output", + is_link_enabled=link_is_enabled): + if not item.is_enabled(self.enabled_set): + logging.info("%s: output is disabled: %s", self.uid, item.uid) + continue + self.director[item.uid].discard() + + def clear(self) -> None: + """ Clears the state of the build item. """ + + def refresh(self) -> None: + """ Refreshes the build item state. """ + + def commit(self, _reason: str) -> None: + """ Commits the build item state. """ + for item in self.item.children("input-to"): + item.save() + self.item.save() + + @property + def digest(self) -> str: + """ Is the hash of the build item. """ + data = self.item.data + data["links"] = copy.deepcopy(data["links"]) + for link in data["links"]: + if link["role"] == "input-to": + link["hash"] = None + return data_digest(data) + + def refresh_link(self, link: Link) -> None: + """ Refreshes the link to reflect the state of the build item. """ + link["hash"] = self.digest + + def refresh_outputs(self) -> None: + """ Refreshes all outputs of the build item. """ + for item in self.item.parents("output"): + self.director[item.uid].refresh() + + def refresh_input_links(self) -> None: + """ Refreshes all input links of the build item. """ + for link in _get_input_links(self.item): + self.director[link.item.uid].refresh_link(link) + + def run(self): + """ Runs the build item tasks. """ + + def substitute(self, data: Any, item: Optional[Item] = None) -> Any: + """ + Recursively substitutes the data using the item mapper of the build + step. + """ + if item is None: + item = self.item + if isinstance(data, str): + return self.mapper.substitute(data, item) + if isinstance(data, list): + new_list: List[Any] = [] + for element in data: + if isinstance(element, str): + match = _SINGLE_SUBSTITUTION.search(element) + if match: + new_item, _, new_element = self.mapper.map( + element[2:-1], item) + if isinstance(new_element, list): + new_list.extend( + self.mapper.substitute(new_element_2, new_item) + for new_element_2 in new_element) + else: + new_list.append( + self.mapper.substitute(new_element, new_item)) + else: + new_list.append(self.mapper.substitute(element, item)) + else: + new_list.append(self.substitute(element)) + return new_list + if isinstance(data, dict): + new_dict: Dict[Any, Any] = {} + for key, value in data.items(): + new_dict[key] = self.substitute(value) + return new_dict + return data + + def input(self, name: str) -> "BuildItem": + """ + Returns the first directory state dependency and the expected hash + associated with the name. + """ + for link in _get_input_links(self.item): + if link["name"] == name: + return self.director[link.item.uid] + raise KeyError + + def inputs(self, name: Optional[str] = None) -> Iterator["BuildItem"]: + """ Yields the inputs associated with the name. """ + for link in _get_input_links(self.item): + if name is None or link["name"] == name: + yield self.director[link.item.uid] + + def input_links(self, name: Optional[str] = None) -> Iterator[Link]: + """ Yields the inputs associated with the name. """ + for link in _get_input_links(self.item): + if name is None or link["name"] == name: + yield link + + def output(self, name: str) -> "BuildItem": + """ + Returns the first directory state production associated with the + name. + """ + for link in self.item.links_to_parents( + "output", is_link_enabled=link_is_enabled): + if link["name"] == name: + if link.item.is_enabled(self.enabled_set): + return self.director[link.item.uid] + logging.info("%s: output is disabled: %s", self.uid, + link.item.uid) + raise ValueError + raise KeyError + + +def _get_dash(ctx: ItemGetValueContext) -> str: + return f"-{ctx.value}" if ctx.value else "" + + +def _get_slash(ctx: ItemGetValueContext) -> str: + return f"/{ctx.value}" if ctx.value else "" + + +class PackageVariant(BuildItem): + """ This is the class represents a package variant. """ + + @classmethod + def prepare_factory(cls, factory: "BuildItemFactory", + type_name: str) -> None: + BuildItem.prepare_factory(factory, type_name) + factory.add_get_value(f"{type_name}:/config/dash", _get_dash) + factory.add_get_value(f"{type_name}:/config/slash", _get_slash) + + +class BuildItemFactory: + """ + The build item factory can create build items for registered build item + types. + """ + + def __init__(self) -> None: + """ Initializes the dictionary of build steps """ + self._build_step_map: Dict[str, Type[BuildItem]] = {} + self._get_values: List[Tuple[str, ItemGetValue]] = [] + + def add_constructor(self, type_name: str, cls: Type[BuildItem]): + """ Associates the build item constructor with the type name. """ + self._build_step_map[type_name] = cls + cls.prepare_factory(self, type_name) + + def create(self, director: "PackageBuildDirector", + item: Item) -> BuildItem: + """ + Creates a build item for the item. + + The new build item will be assocated with the build director. + """ + return self._build_step_map.get(item.type, BuildItem)(director, item) + + def add_get_value(self, type_path_key: str, + get_value: ItemGetValue) -> None: + """ Adds a get value method for the type key path. """ + self._get_values.append((type_path_key, get_value)) + + def add_get_values_to_mapper(self, mapper: ItemMapper) -> None: + """ Adds add registered get value methods to the mapper. """ + for type_path_key, get_value in self._get_values: + mapper.add_get_value(type_path_key, get_value) + + +class PackageBuildDirector: + """ + The package build director contains the package build state and runs the + build. + """ + + # pylint: disable=too-few-public-methods + def __init__(self, item_cache: ItemCache, factory: BuildItemFactory): + self._item_cache = item_cache + self.factory = factory + self._build_items: Dict[str, BuildItem] = {} + + def __getitem__(self, uid: str) -> BuildItem: + item = self._build_items.get(uid, None) + if item is not None: + return item + logging.info("%s: create build item", uid) + item = self.factory.create(self, self._item_cache[uid]) + self._build_items[uid] = item + return item + + def clear(self) -> None: + """ Clears the build items of the director. """ + self._build_items.clear() + + def build_package(self, only: Optional[List[str]], + force: Optional[List[str]]): + """ Builds the package """ + if force is None: + force = [] + build_steps = self._item_cache["/qdp/variant"].parent("package-build") + enabled_set = self["/qdp/variant"]["enabled"] + logging.info("%s: build the package", build_steps.uid) + for step in build_steps.parents("build-step", + is_link_enabled=link_is_enabled): + if not step.is_enabled(enabled_set): + logging.info("%s: is disabled", step.uid) + continue + builder = self[step.uid] + if only is not None and step.uid not in only: + logging.info("%s: build is skipped", step.uid) + continue + builder.build(step.uid in force) + logging.info("%s: finished building package", build_steps.uid) diff --git a/rtemsspec/packagebuildfactory.py b/rtemsspec/packagebuildfactory.py new file mode 100644 index 00000000..d419e28f --- /dev/null +++ b/rtemsspec/packagebuildfactory.py @@ -0,0 +1,34 @@ +# SPDX-License-Identifier: BSD-2-Clause +""" This module provides the default build item factory. """ + +# Copyright (C) 2023 embedded brains GmbH & Co. KG +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. + +from rtemsspec.packagebuild import BuildItemFactory, PackageVariant + + +def create_build_item_factory() -> BuildItemFactory: + """ Creates the default build item factory. """ + factory = BuildItemFactory() + factory.add_constructor("qdp/variant", PackageVariant) + return factory diff --git a/rtemsspec/tests/spec-packagebuild/qdp/output/a.yml b/rtemsspec/tests/spec-packagebuild/qdp/output/a.yml new file mode 100644 index 00000000..985b5ff8 --- /dev/null +++ b/rtemsspec/tests/spec-packagebuild/qdp/output/a.yml @@ -0,0 +1,7 @@ +SPDX-License-Identifier: CC-BY-SA-4.0 OR BSD-2-Clause +copyrights: +- Copyright (C) 2023 embedded brains GmbH & Co. KG +enabled-by: true +links: [] +qdp-type: test-output +type: qdp diff --git a/rtemsspec/tests/spec-packagebuild/qdp/output/b.yml b/rtemsspec/tests/spec-packagebuild/qdp/output/b.yml new file mode 100644 index 00000000..e738d86e --- /dev/null +++ b/rtemsspec/tests/spec-packagebuild/qdp/output/b.yml @@ -0,0 +1,7 @@ +SPDX-License-Identifier: CC-BY-SA-4.0 OR BSD-2-Clause +copyrights: +- Copyright (C) 2023 embedded brains GmbH & Co. KG +enabled-by: false +links: [] +qdp-type: test-output +type: qdp diff --git a/rtemsspec/tests/spec-packagebuild/qdp/package-build.yml b/rtemsspec/tests/spec-packagebuild/qdp/package-build.yml new file mode 100644 index 00000000..1b451572 --- /dev/null +++ b/rtemsspec/tests/spec-packagebuild/qdp/package-build.yml @@ -0,0 +1,13 @@ +SPDX-License-Identifier: CC-BY-SA-4.0 OR BSD-2-Clause +copyrights: +- Copyright (C) 2020 embedded brains GmbH & Co. KG +enabled-by: true +links: +- role: build-step + uid: steps/a +- role: build-step + uid: steps/b +- role: build-step + uid: steps/c +qdp-type: package-build +type: qdp diff --git a/rtemsspec/tests/spec-packagebuild/qdp/steps/a.yml b/rtemsspec/tests/spec-packagebuild/qdp/steps/a.yml new file mode 100644 index 00000000..bf480a60 --- /dev/null +++ b/rtemsspec/tests/spec-packagebuild/qdp/steps/a.yml @@ -0,0 +1,18 @@ +SPDX-License-Identifier: CC-BY-SA-4.0 OR BSD-2-Clause +build-step-type: test +copyrights: +- Copyright (C) 2023 embedded brains GmbH & Co. KG +description: | + A. +enabled-by: true +links: +- hash: null + name: variant + role: input + uid: ../variant +- hash: null + name: bar + role: input-to + uid: c +qdp-type: build-step +type: qdp diff --git a/rtemsspec/tests/spec-packagebuild/qdp/steps/b.yml b/rtemsspec/tests/spec-packagebuild/qdp/steps/b.yml new file mode 100644 index 00000000..0094c5f1 --- /dev/null +++ b/rtemsspec/tests/spec-packagebuild/qdp/steps/b.yml @@ -0,0 +1,18 @@ +SPDX-License-Identifier: CC-BY-SA-4.0 OR BSD-2-Clause +build-step-type: test +copyrights: +- Copyright (C) 2023 embedded brains GmbH & Co. KG +description: | + B. +enabled-by: false +links: +- hash: null + name: variant + role: input + uid: ../variant +- hash: null + name: bla + role: input-to + uid: c +qdp-type: build-step +type: qdp diff --git a/rtemsspec/tests/spec-packagebuild/qdp/steps/c.yml b/rtemsspec/tests/spec-packagebuild/qdp/steps/c.yml new file mode 100644 index 00000000..7bf9f893 --- /dev/null +++ b/rtemsspec/tests/spec-packagebuild/qdp/steps/c.yml @@ -0,0 +1,36 @@ +SPDX-License-Identifier: CC-BY-SA-4.0 OR BSD-2-Clause +build-step-type: test-mapper +copyrights: +- Copyright (C) 2023 embedded brains GmbH & Co. KG +description: | + C. +enabled-by: true +links: +- hash: null + name: variant + role: input + uid: ../variant +- hash: null + name: foo + role: input + uid: a +- name: blub + role: output + uid: ../output/a +- name: moo + role: output + uid: ../output/b +qdp-type: build-step +type: qdp +values: + a: a + b: + - b1 + - b2 + c: c + list: + - ${.:/values/a} + - ${.:/values/b} + - - d + - e + - ${.:/values/c} diff --git a/rtemsspec/tests/spec-packagebuild/qdp/variant.yml b/rtemsspec/tests/spec-packagebuild/qdp/variant.yml new file mode 100644 index 00000000..07271d61 --- /dev/null +++ b/rtemsspec/tests/spec-packagebuild/qdp/variant.yml @@ -0,0 +1,23 @@ +SPDX-License-Identifier: CC-BY-SA-4.0 OR BSD-2-Clause +arch: sparc +bsp: gr712rc +bsp-family: leon3 +build-directory: ${.:/deployment-directory}/build +config: smp +copyrights: +- Copyright (C) 2020 embedded brains GmbH & Co. KG +deployment-directory: ${.:/prefix-directory}/${.:/package-directory} +enabled: [] +enabled-by: true +ident: ${.:/arch}/${.:/bsp}${.:/config/slash}/${.:/package-version} +links: +- role: package-build + uid: package-build +name: ${.:/arch}-${.:/bsp}${.:/config/dash}-${.:/package-version} +package-directory: pkg +package-version: '4' +params: {} +prefix-directory: ${.:/tmpdir} +qdp-type: variant +rtems-version: '6' +type: qdp diff --git a/rtemsspec/tests/spec-packagebuild/spec/qdp-test-input.yml b/rtemsspec/tests/spec-packagebuild/spec/qdp-test-input.yml new file mode 100644 index 00000000..06d41dac --- /dev/null +++ b/rtemsspec/tests/spec-packagebuild/spec/qdp-test-input.yml @@ -0,0 +1,30 @@ +SPDX-License-Identifier: CC-BY-SA-4.0 OR BSD-2-Clause +copyrights: +- Copyright (C) 2023 embedded brains GmbH & Co. KG +enabled-by: true +links: +- role: spec-member + uid: root +- role: spec-refinement + spec-key: name + spec-value: bar + uid: qdp-input-role +- role: spec-refinement + spec-key: name + spec-value: bla + uid: qdp-input-role +- role: spec-refinement + spec-key: name + spec-value: foo + uid: qdp-input-role +spec-description: null +spec-example: null +spec-info: + dict: + attributes: {} + description: | + Test input. + mandatory-attributes: all +spec-name: Test Input Item Type +spec-type: qdp-test-input +type: spec diff --git a/rtemsspec/tests/spec-packagebuild/spec/qdp-test-output.yml b/rtemsspec/tests/spec-packagebuild/spec/qdp-test-output.yml new file mode 100644 index 00000000..e9dc7809 --- /dev/null +++ b/rtemsspec/tests/spec-packagebuild/spec/qdp-test-output.yml @@ -0,0 +1,22 @@ +SPDX-License-Identifier: CC-BY-SA-4.0 OR BSD-2-Clause +copyrights: +- Copyright (C) 2023 embedded brains GmbH & Co. KG +enabled-by: true +links: +- role: spec-member + uid: root +- role: spec-refinement + spec-key: qdp-type + spec-value: test-output + uid: qdp-root +spec-description: null +spec-example: null +spec-info: + dict: + attributes: {} + description: | + Test output. + mandatory-attributes: all +spec-name: Test Output Item Type +spec-type: qdp-test-output +type: spec diff --git a/rtemsspec/tests/spec-packagebuild/spec/qdp-test.yml b/rtemsspec/tests/spec-packagebuild/spec/qdp-test.yml new file mode 100644 index 00000000..690b8385 --- /dev/null +++ b/rtemsspec/tests/spec-packagebuild/spec/qdp-test.yml @@ -0,0 +1,28 @@ +SPDX-License-Identifier: CC-BY-SA-4.0 OR BSD-2-Clause +copyrights: +- Copyright (C) 2023 embedded brains GmbH & Co. KG +enabled-by: true +links: +- role: spec-member + uid: root +- role: spec-refinement + spec-key: build-step-type + spec-value: test + uid: qdp-build-step +- role: spec-refinement + spec-key: build-step-type + spec-value: test-mapper + uid: qdp-build-step +spec-description: null +spec-example: null +spec-info: + dict: + attributes: + values: + description: Values. + spec-type: any + description: Test. + mandatory-attributes: none +spec-name: Test Item Type +spec-type: qdp-test +type: spec diff --git a/rtemsspec/tests/test_packagebuild.py b/rtemsspec/tests/test_packagebuild.py new file mode 100644 index 00000000..48ee2c84 --- /dev/null +++ b/rtemsspec/tests/test_packagebuild.py @@ -0,0 +1,160 @@ +# SPDX-License-Identifier: BSD-2-Clause +""" Unit tests for the rtemsspec.packagebuild module. """ + +# Copyright (C) 2023 embedded brains GmbH & Co. KG +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. + +import logging +import os +import pytest +from pathlib import Path +import shutil + +from rtemsspec.items import EmptyItem, Item, ItemCache, ItemGetValueContext +from rtemsspec.packagebuild import BuildItem, BuildItemMapper, \ + build_item_input, PackageBuildDirector +from rtemsspec.packagebuildfactory import create_build_item_factory +from rtemsspec.specverify import verify +from rtemsspec.tests.util import get_and_clear_log + + +def _copy_dir(src, dst): + dst.mkdir(parents=True, exist_ok=True) + for item in os.listdir(src): + s = src / item + d = dst / item + if s.is_dir(): + _copy_dir(s, d) + else: + shutil.copy2(str(s), str(d)) + + +def _create_item_cache(tmp_dir: Path, spec_dir: Path) -> ItemCache: + spec_dst = tmp_dir / Path("pkg/build/spec") + test_dir = Path(__file__).parent + _copy_dir(test_dir / spec_dir, spec_dst) + _copy_dir(test_dir.parent.parent / "spec-spec", spec_dst) + _copy_dir(test_dir.parent.parent / "spec-qdp" / "spec", spec_dst / "spec") + cache_dir = os.path.join(tmp_dir, "cache") + config = { + "cache-directory": os.path.normpath(cache_dir), + "paths": [str(spec_dst.absolute())], + "spec-type-root-uid": "/spec/root" + } + return ItemCache(config) + + +def test_builditemmapper(): + mapper = BuildItemMapper(EmptyItem()) + with pytest.raises(NotImplementedError): + mapper.get_link(mapper.item) + + +class _TestItem(BuildItem): + + def __init__(self, director: PackageBuildDirector, item: Item): + super().__init__(director, item, BuildItemMapper(item, recursive=True)) + + +def test_packagebuild(caplog, tmpdir): + tmp_dir = Path(tmpdir) + item_cache = _create_item_cache(tmp_dir, Path("spec-packagebuild")) + + caplog.set_level(logging.WARN) + verify_config = {"root-type": "/spec/root"} + status = verify(verify_config, item_cache) + assert status.critical == 0 + assert status.error == 0 + + caplog.set_level(logging.DEBUG) + factory = create_build_item_factory() + factory.add_constructor("qdp/build-step/test-mapper", _TestItem) + + def get_tmpdir(_ctx: ItemGetValueContext) -> str: + return str(tmp_dir.absolute()) + + factory.add_get_value("qdp/variant:/tmpdir", get_tmpdir) + director = PackageBuildDirector(item_cache, factory) + director.clear() + prefix_dir = Path(director["/qdp/variant"]["prefix-directory"]) + + director.build_package(None, None) + log = get_and_clear_log(caplog) + assert "INFO /qdp/steps/a: create build item" in log + assert "INFO /qdp/steps/b: create build item" not in log + assert "INFO /qdp/steps/b: is disabled" in log + assert "INFO /qdp/steps/c: output is disabled: /qdp/output/b" in log + + director.build_package(None, ["/qdp/steps/a"]) + log = get_and_clear_log(caplog) + assert "INFO /qdp/steps/a: build is forced" in log + assert "INFO /qdp/steps/c: input has changed: /qdp/steps/a" in log + + director.build_package([], None) + log = get_and_clear_log(caplog) + assert "INFO /qdp/steps/a: build is skipped" in log + + director.build_package(None, None) + log = get_and_clear_log(caplog) + assert "INFO /qdp/steps/a: build is unnecessary" in log + assert "INFO /qdp/steps/c: build is unnecessary" in log + assert "INFO /qdp/steps/c: input is disabled: /qdp/steps/b" in log + + c = director["/qdp/steps/c"] + assert isinstance(c, _TestItem) + c["foo"] = "bar" + c["blub"] = "${.:/foo}" + assert c["foo"] == "bar" + assert "foo" in c + assert "nil" not in c + assert c["blub"] == "bar" + assert c.substitute(c.item["blub"], c.item) == "bar" + assert c.substitute("${/qdp/variant:/spec}") == "spec:/qdp/variant" + assert c.variant.uid == "/qdp/variant" + variant_config = c.variant["config"] + c.variant["config"] = "" + assert c.variant["name"] == "sparc-gr712rc-4" + assert c.variant["ident"] == "sparc/gr712rc/4" + c.variant["config"] = variant_config + assert c.variant["name"] == "sparc-gr712rc-smp-4" + assert c.variant["ident"] == "sparc/gr712rc/smp/4" + assert c.enabled_set == [] + assert c.enabled + assert build_item_input(c.item, "foo").uid == "/qdp/steps/a" + assert build_item_input(c.item, "bar").uid == "/qdp/steps/a" + with pytest.raises(KeyError): + build_item_input(c.item, "blub") + assert c.input("foo").uid == "/qdp/steps/a" + assert list(c.input_links("foo"))[0].item.uid == "/qdp/steps/a" + with pytest.raises(KeyError): + c.input("nix") + assert [item.uid for item in c.inputs() + ] == ["/qdp/variant", "/qdp/steps/a", "/qdp/steps/a"] + assert [item.uid for item in c.inputs("foo")] == ["/qdp/steps/a"] + assert c.output("blub").uid == "/qdp/output/a" + with pytest.raises(KeyError): + c.output("nix") + with pytest.raises(ValueError): + c.output("moo") + assert c["values"]["list"] == ["a", "b1", "b2", ["d", "e"], "c"] + c.clear() diff --git a/spec-qdp/spec/qdp-build-step-role.yml b/spec-qdp/spec/qdp-build-step-role.yml new file mode 100644 index 00000000..0ba67293 --- /dev/null +++ b/spec-qdp/spec/qdp-build-step-role.yml @@ -0,0 +1,23 @@ +SPDX-License-Identifier: CC-BY-SA-4.0 OR BSD-2-Clause +copyrights: +- Copyright (C) 2020 embedded brains GmbH & Co. KG +enabled-by: true +links: +- role: spec-member + uid: root +- role: spec-refinement + spec-key: role + spec-value: build-step + uid: link +spec-description: null +spec-example: null +spec-info: + dict: + attributes: {} + description: | + It defines the build step role of links and is used to define a build + step. + mandatory-attributes: all +spec-name: Build Step Link Role +spec-type: qdp-build-step-role +type: spec diff --git a/spec-qdp/spec/qdp-build-step.yml b/spec-qdp/spec/qdp-build-step.yml new file mode 100644 index 00000000..26af6e18 --- /dev/null +++ b/spec-qdp/spec/qdp-build-step.yml @@ -0,0 +1,31 @@ +SPDX-License-Identifier: CC-BY-SA-4.0 OR BSD-2-Clause +copyrights: +- Copyright (C) 2020 embedded brains GmbH & Co. KG +enabled-by: true +links: +- role: spec-member + uid: root +- role: spec-refinement + spec-key: qdp-type + spec-value: build-step + uid: qdp-root +spec-description: null +spec-example: null +spec-info: + dict: + attributes: + build-step-type: + description: | + It shall be the build step type. + spec-type: name + description: + description: | + It shall be the description of the build step. + spec-type: str + description: | + This set of attributes specifies a build step to produce a component of + the QDP. + mandatory-attributes: all +spec-name: Build Step Item Type +spec-type: qdp-build-step +type: spec diff --git a/spec-qdp/spec/qdp-input-generic-role.yml b/spec-qdp/spec/qdp-input-generic-role.yml new file mode 100644 index 00000000..c69157d1 --- /dev/null +++ b/spec-qdp/spec/qdp-input-generic-role.yml @@ -0,0 +1,73 @@ +SPDX-License-Identifier: CC-BY-SA-4.0 OR BSD-2-Clause +copyrights: +- Copyright (C) 2022 embedded brains GmbH & Co. KG +enabled-by: true +links: +- role: spec-member + uid: root +- role: spec-refinement + spec-key: name + spec-value: archive + uid: qdp-input-role +- role: spec-refinement + spec-key: name + spec-value: build + uid: qdp-input-role +- role: spec-refinement + spec-key: name + spec-value: config + uid: qdp-input-role +- role: spec-refinement + spec-key: name + spec-value: example + uid: qdp-input-role +- role: spec-refinement + spec-key: name + spec-value: log + uid: qdp-input-role +- role: spec-refinement + spec-key: name + spec-value: member + uid: qdp-input-role +- role: spec-refinement + spec-key: name + spec-value: section + uid: qdp-input-role +- role: spec-refinement + spec-key: name + spec-value: subsection + uid: qdp-input-role +- role: spec-refinement + spec-key: name + spec-value: source + uid: qdp-input-role +- role: spec-refinement + spec-key: name + spec-value: spec + uid: qdp-input-role +- role: spec-refinement + spec-key: name + spec-value: test-log + uid: qdp-input-role +- role: spec-refinement + spec-key: name + spec-value: tools + uid: qdp-input-role +- role: spec-refinement + spec-key: name + spec-value: variant + uid: qdp-input-role +- role: spec-refinement + spec-key: name + spec-value: verify-package + uid: qdp-input-role +spec-description: null +spec-example: null +spec-info: + dict: + attributes: {} + description: null + mandatory-attributes: all +spec-name: Generic Input Link Role +spec-type: qdp-input-generic-role +type: spec diff --git a/spec-qdp/spec/qdp-input-role.yml b/spec-qdp/spec/qdp-input-role.yml new file mode 100644 index 00000000..1448ef85 --- /dev/null +++ b/spec-qdp/spec/qdp-input-role.yml @@ -0,0 +1,37 @@ +SPDX-License-Identifier: CC-BY-SA-4.0 OR BSD-2-Clause +copyrights: +- Copyright (C) 2020 embedded brains GmbH & Co. KG +enabled-by: true +links: +- role: spec-member + uid: root +- role: spec-refinement + spec-key: role + spec-value: input + uid: link +- role: spec-refinement + spec-key: role + spec-value: input-to + uid: link +spec-description: null +spec-example: null +spec-info: + dict: + attributes: + hash: + description: | + If the value is present, then it shall be the expected hash of the + input, otherwise the input dependency is not yet initialized. + spec-type: qdp-optional-hash + name: + description: | + If shall be the name of the input dependency. The name may be used + to distinguish it from other dependencies of the same item. + spec-type: str + description: | + It defines the input role of links and is used to define an input + dependency of a build item. + mandatory-attributes: all +spec-name: Input Link Role +spec-type: qdp-input-role +type: spec diff --git a/spec-qdp/spec/qdp-optional-hash.yml b/spec-qdp/spec/qdp-optional-hash.yml new file mode 100644 index 00000000..93f585a2 --- /dev/null +++ b/spec-qdp/spec/qdp-optional-hash.yml @@ -0,0 +1,21 @@ +SPDX-License-Identifier: CC-BY-SA-4.0 OR BSD-2-Clause +copyrights: +- Copyright (C) 2020, 2022 embedded brains GmbH & Co. KG +enabled-by: true +links: +- role: spec-member + uid: root +spec-description: null +spec-example: null +spec-info: + none: null + str: + assert: + - re: ^[A-Za-z0-9+_=-]{44}$ + - re: ^[A-Za-z0-9+_=-]{88}$ + description: | + If the value is present, then it shall be a SHA256 or SHA512 hash value + encoded in base64url. +spec-name: Optional Hash +spec-type: qdp-optional-hash +type: spec diff --git a/spec-qdp/spec/qdp-output-role.yml b/spec-qdp/spec/qdp-output-role.yml new file mode 100644 index 00000000..01d27025 --- /dev/null +++ b/spec-qdp/spec/qdp-output-role.yml @@ -0,0 +1,28 @@ +SPDX-License-Identifier: CC-BY-SA-4.0 OR BSD-2-Clause +copyrights: +- Copyright (C) 2020 embedded brains GmbH & Co. KG +enabled-by: true +links: +- role: spec-member + uid: root +- role: spec-refinement + spec-key: role + spec-value: output + uid: link +spec-description: null +spec-example: null +spec-info: + dict: + attributes: + name: + description: | + If shall be the name of the output production. The name may be used + to distinguish it from other productions of the same item. + spec-type: str + description: | + It defines the output production role of links and is used to define that + an item is the producer of an output. + mandatory-attributes: all +spec-name: Output Link Role +spec-type: qdp-output-role +type: spec diff --git a/spec-qdp/spec/qdp-package-build-role.yml b/spec-qdp/spec/qdp-package-build-role.yml new file mode 100644 index 00000000..51745960 --- /dev/null +++ b/spec-qdp/spec/qdp-package-build-role.yml @@ -0,0 +1,23 @@ +SPDX-License-Identifier: CC-BY-SA-4.0 OR BSD-2-Clause +copyrights: +- Copyright (C) 2020 embedded brains GmbH & Co. KG +enabled-by: true +links: +- role: spec-member + uid: root +- role: spec-refinement + spec-key: role + spec-value: package-build + uid: link +spec-description: null +spec-example: null +spec-info: + dict: + attributes: {} + description: | + It defines the package build role of links and is used to define the + package build of a variant. + mandatory-attributes: all +spec-name: Package Build Link Role +spec-type: qdp-package-build-role +type: spec diff --git a/spec-qdp/spec/qdp-package-build.yml b/spec-qdp/spec/qdp-package-build.yml new file mode 100644 index 00000000..8c073f7f --- /dev/null +++ b/spec-qdp/spec/qdp-package-build.yml @@ -0,0 +1,24 @@ +SPDX-License-Identifier: CC-BY-SA-4.0 OR BSD-2-Clause +copyrights: +- Copyright (C) 2020 embedded brains GmbH & Co. KG +enabled-by: true +links: +- role: spec-member + uid: root +- role: spec-refinement + spec-key: qdp-type + spec-value: package-build + uid: qdp-root +spec-description: null +spec-example: null +spec-info: + dict: + attributes: {} + description: | + This set of attributes specifies the package build process. There shall + be links to ${qdp-build-step:/spec-name} items. The links shall have the + ${qdp-build-step-role:/spec-name}. + mandatory-attributes: all +spec-name: Package Build Item Type +spec-type: qdp-package-build +type: spec diff --git a/spec-qdp/spec/qdp-root.yml b/spec-qdp/spec/qdp-root.yml new file mode 100644 index 00000000..b40968ae --- /dev/null +++ b/spec-qdp/spec/qdp-root.yml @@ -0,0 +1,26 @@ +SPDX-License-Identifier: CC-BY-SA-4.0 OR BSD-2-Clause +copyrights: +- Copyright (C) 2022 embedded brains GmbH & Co. KG +enabled-by: true +links: +- role: spec-member + uid: root +- role: spec-refinement + spec-key: type + spec-value: qdp + uid: root +spec-description: null +spec-example: null +spec-info: + dict: + attributes: + qdp-type: + description: | + It shall be the QDP item type. + spec-type: name + description: | + This set of attributes specifies a QDP item. + mandatory-attributes: all +spec-name: QDP Root Item Type +spec-type: qdp-root +type: spec diff --git a/spec-qdp/spec/qdp-variant.yml b/spec-qdp/spec/qdp-variant.yml new file mode 100644 index 00000000..1fe39c3b --- /dev/null +++ b/spec-qdp/spec/qdp-variant.yml @@ -0,0 +1,88 @@ +SPDX-License-Identifier: CC-BY-SA-4.0 OR BSD-2-Clause +copyrights: +- Copyright (C) 2020 embedded brains GmbH & Co. KG +enabled-by: true +links: +- role: spec-member + uid: root +- role: spec-refinement + spec-key: qdp-type + spec-value: variant + uid: qdp-root +spec-description: | + Items of this type shall have the following links: + + * There shall be exactly one link to a ${qdp-package-build:/spec-name} item + with the ${qdp-package-build-role:/spec-name}. This link defines the + package build process. +spec-example: null +spec-info: + dict: + attributes: + arch: + description: | + It shall be the name of the target architecture. + spec-type: str + bsp: + description: | + It shall be the name of the Board Support Package (BSP). + spec-type: str + bsp-family: + description: | + It shall be the name of the BSP family. + spec-type: str + build-directory: + description: | + It shall be the path to the package build directory. + spec-type: str + config: + description: | + It shall be the BSP configuration name. It may be the empty string, + if the BSP has no specific configuration. + spec-type: str + deployment-directory: + description: | + It shall be the path to the package deployment directory. + spec-type: str + enabled: + description: | + It shall be the expression which defines under which conditions + the specification items or parts of it are enabled. + spec-type: enabled-by + ident: + description: | + It shall be the package-specific identifier. + spec-type: str + name: + description: | + It shall be the name for package-specific file or directory names. + spec-type: str + package-directory: + description: | + It shall be the package directory. + spec-type: str + package-version: + description: | + It shall be the package version. + spec-type: str + params: + description: | + It shall be an optional set of parameters which may be used for + variable subsitution. + spec-type: any + prefix-directory: + description: | + It shall be the path to the package prefix directory. The deployment + directory should start with this prefix. The prefix should be + stripped from members of the package archive. + spec-type: str + rtems-version: + description: | + It shall be the RTEMS version. + spec-type: str + description: | + This set of attributes specifies a package variant configuration. + mandatory-attributes: all +spec-name: Variant Item Type +spec-type: qdp-variant +type: spec -- cgit v1.2.3