From c109b53f1feb1fb4e401fee45bbcb6eb5f90d7b5 Mon Sep 17 00:00:00 2001 From: Chris Johns Date: Wed, 3 Apr 2024 15:20:11 +1100 Subject: sb: Add sb-rtems-pkg to update the RTEMS package hashes and checksums --- source-builder/sb-rtems-pkg | 29 +++++ source-builder/sb/download.py | 5 +- source-builder/sb/git.py | 12 ++ source-builder/sb/rtemspkg.py | 287 ++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 332 insertions(+), 1 deletion(-) create mode 100755 source-builder/sb-rtems-pkg create mode 100644 source-builder/sb/rtemspkg.py diff --git a/source-builder/sb-rtems-pkg b/source-builder/sb-rtems-pkg new file mode 100755 index 0000000..99ed26c --- /dev/null +++ b/source-builder/sb-rtems-pkg @@ -0,0 +1,29 @@ +#! /usr/bin/env python +# +# RTEMS Tools Project (http://www.rtems.org/) +# Copyright 2024 Chris Johns (chrisj@rtems.org) +# All rights reserved. +# +# This file is part of the RTEMS Tools package in 'rtems-tools'. +# +# Permission to use, copy, modify, and/or distribute this software for any +# purpose with or without fee is hereby granted, provided that the above +# copyright notice and this permission notice appear in all copies. +# +# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +from __future__ import print_function + +try: + import sb.rtemspkg + sb.rtemspkg.run() +except ImportError: + import sys + print("Incorrect Source Builder installation", file = sys.stderr) + sys.exit(1) diff --git a/source-builder/sb/download.py b/source-builder/sb/download.py index 6e6f9f2..0201675 100644 --- a/source-builder/sb/download.py +++ b/source-builder/sb/download.py @@ -85,6 +85,8 @@ def _hash_check(file_, absfile, macros, remove = True): hash = hash.split() if len(hash) != 2: raise error.internal('invalid hash format: %s' % (file_)) + if hash[0] == 'NO-HASH': + return not failed try: hashlib_algorithms = hashlib.algorithms except: @@ -479,7 +481,8 @@ def _git_downloader(url, local, config, opts): else: repo.clean(['-f', '-d']) repo.reset('--hard') - repo.checkout('master') + default_branch = repo.default_branch() + repo.checkout(default_branch) for a in us[1:]: _as = a.split('=') if _as[0] == 'branch' or _as[0] == 'checkout': diff --git a/source-builder/sb/git.py b/source-builder/sb/git.py index 237e690..0aa4da9 100644 --- a/source-builder/sb/git.py +++ b/source-builder/sb/git.py @@ -226,6 +226,18 @@ class repo: hash = l1[len('commit '):] return hash + def default_branch(self): + ec, output = self._run(['remote', 'show']) + if ec == 0: + origin = output.split('\n')[0] + ec, output = self._run(['remote', 'show', origin]) + if ec == 0: + for l in output.split('\n'): + l = l.strip() + if l.startswith('HEAD branch: '): + return l[len('HEAD branch: '):] + return None + if __name__ == '__main__': import os.path import sys diff --git a/source-builder/sb/rtemspkg.py b/source-builder/sb/rtemspkg.py new file mode 100644 index 0000000..492eb59 --- /dev/null +++ b/source-builder/sb/rtemspkg.py @@ -0,0 +1,287 @@ +# +# RTEMS Tools Project (http://www.rtems.org/) +# Copyright 2024 Chris Johns (chrisj@rtems.org) +# All rights reserved. +# +# This file is part of the RTEMS Tools package in 'rtems-tools'. +# +# Permission to use, copy, modify, and/or distribute this software for any +# purpose with or without fee is hereby granted, provided that the above +# copyright notice and this permission notice appear in all copies. +# +# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +# +# This code builds a package compiler tool suite given a tool set. A tool +# set lists the various tools. These are specific tool configurations. +# + +from __future__ import print_function + +import argparse +import base64 +import copy +import datetime +import hashlib +import os +import sys + +try: + from . import build + from . import download + from . import error + from . import git + from . import log + from . import path + from . import simhost + from . import version +except KeyboardInterrupt: + print('abort: user terminated', file=sys.stderr) + sys.exit(1) +except: + raise + +# +# RTEMS Packages we maintian a git hash of in the RSB +# +rpc_label = 0 +rpc_config = 1 +rpc_version = 2 +rpc_repo = 3 +rpc_repo_name = 4 +rpc_branch = 5 +rpc_snapshot = 6 +rpc_package = 7 +rtems_pkg_cfgs = [ + [ + 'RTEMS Tools', 'tools/rtems-tools-%{rtems_version}.cfg', + 'rtems_tools_version', 'git://git.rtems.org/rtems-tools', + 'rtems-tools.git', 'master', + 'https://git.rtems.org/rtems-tools/snapshot/rtems-tools-%{rtems_tools_version}.tar.bz2', + 'rtems-tools-%{rtems_tools_version}.tar.bz2' + ], + [ + 'RTEMS Kernel', 'tools/rtems-kernel-%{rtems_version}.cfg', + 'rtems_kernel_version', 'git://git.rtems.org/rtems', 'rtems.git', + 'master', + 'https://git.rtems.org/rtems/snapshot/rtems-%{rtems_kernel_version}.tar.bz2', + 'rtems-kernel-%{rtems_kernel_version}.tar.bz2' + ], + [ + 'RTEMS LibBSD', 'tools/rtems-libbsd-%{rtems_version}.cfg', + 'rtems_libbsd_version', 'git://git.rtems.org/rtems-libbsd', + 'rtems-libbsd.git', '6-freebsd-12', + 'https://git.rtems.org/rtems-libbsd/snapshot/rtems-libbsd-%{rtems_libbsd_version}.tar.%{rtems_libbsd_ext}', + 'rtems-libbsd-%{rtems_libbsd_version}.tar.%{rtems_libbsd_ext}' + ], + [ + 'RTEMS Net Legacy', 'tools/rtems-net-legacy-%{rtems_version}.cfg', + 'rtems_net_version', 'git://git.rtems.org/rtems-net-legacy', + 'rtems-net-legacy.git', 'main', + 'https://git.rtems.org/rtems-net-legacy/snapshot/rtems-net-legacy-%{rtems_net_version}.tar.%{rtems_net_ext}', + 'rtems-net-legacy-%{rtems_net_version}.tar.%{rtems_net_ext}' + ], + [ + 'RTEMS Net Services', 'net/net-services-1.cfg', + 'rtems_net_services_version', 'git://git.rtems.org/rtems-net-services', + 'rtems-net-services.git', 'master', + 'https://git.rtems.org/rtems-net-services/snapshot/rtems-net-services-%{rtems_net_services_version}.tar.%{rtems_net_services_ext}', + 'rtems-net-services-%{rtems_net_services_version}.tar.%{rtems_net_services_ext}' + ], +] + + +def clean_line(line): + line = line[0:-1] + b = line.find('#') + if b >= 0: + line = line[1:b] + ('\\' if line[-1] == '\\' else '') + return line.strip() + + +def clean_and_pack(line, last_line): + leading_ws = ' ' if len(line) > 0 and line[0].isspace() else '' + line = clean_line(line) + if len(last_line) > 0: + line = last_line + leading_ws + line + return line + + +def config_patch(configdir, config, version_label, config_hash, repo_hash, + checksum): + for cd in configdir.split(':'): + cf = path.join(cd, config) + if path.exists(cf): + try: + with open(cf) as f: + lines = f.readlines() + except IOError as err: + raise error.general('config: %s: read error: %s' % + (config, str(err))) + new_config = [] + new_lines = [] + last_line = '' + for line in lines: + new_lines += [line] + line = clean_and_pack(line, last_line) + if len(line) > 0: + if line[-1] == '\\': + last_line = line[:-1] + continue + last_line = '' + if version_label in line and not 'rsb_version' in line: + if line.startswith('%define ' + version_label): + new_lines = [ + '%define ' + version_label + ' ' + repo_hash + + os.linesep + ] + elif line.startswith('%hash '): + ls = line.split() + if len(ls) != 4: + raise error.general('invalid %hash: ' + line) + new_lines = [ + ' '.join(ls[0:3]) + ' \\' + os.linesep, + ' ' + checksum + os.linesep + ] + new_config += new_lines + new_lines = [] + try: + with open(cf, 'w') as f: + f.writelines(new_config) + except IOError as err: + raise error.general('config: %s: write error: %s' % + (config, str(err))) + return + raise error.general('could not find: ' + config) + + +def checksum_sha512_base64(tarball): + hasher = hashlib.new('sha512') + try: + with open(path.host(tarball), 'rb') as f: + hasher.update(f.read()) + except IOError as err: + log.notice('hash: %s: read error: %s' % (file_, str(err))) + except: + raise + raise error.general('cannot hash the tar file') + hash_hex = hasher.hexdigest() + hash_base64 = base64.b64encode(hasher.digest()).decode('utf-8') + return hash_base64 + + +def run(args=sys.argv): + ec = 0 + output = [] + try: + # + # The RSB options support cannot be used because it loads the defaults + # for the host which we cannot do here. + # + description = 'RTEMS Track Dependencies a build set has for all hosts.' + + argsp = argparse.ArgumentParser(prog='sb-rtems-pkg', + description=description) + argsp.add_argument('--rtems-version', + help='Set the RTEMS version.', + type=str, + default=version.version()) + argsp.add_argument('--log', + help='Log file.', + type=str, + default=simhost.log_default('rtems-pkg')) + argsp.add_argument('--trace', + help='Enable trace logging for debugging.', + action='store_true') + argsp.add_argument('--dry-run', + help='Dry run, do not update the configurations', + action='store_true') + argsp.add_argument('bsets', nargs='*', help='Build sets.') + + argopts = argsp.parse_args(args[1:]) + + simhost.load_log(argopts.log) + log.notice('RTEMS Source Builder - RTEMS Package Update, %s' % + (version.string())) + log.tracing = argopts.trace + + opts = simhost.load_options(args, argopts, extras=['--with-download']) + opts.defaults['_rsb_getting_source'] = '1' + opts.defaults[ + 'rtems_waf_build_root_suffix'] = '%{waf_build_root_suffix}' + opts.defaults['rtems_version'] = argopts.rtems_version + + for cfg in rtems_pkg_cfgs: + b = None + try: + bopts = copy.copy(opts) + bmacros = copy.copy(opts.defaults) + b = build.build(cfg[rpc_config], False, bopts, bmacros) + git_hash_key = b.macros.find(cfg[rpc_version]) + if len(git_hash_key) == 0: + raise error.general(cfg[rpc_label] + + ': cannot find version macro') + source_dir = b.macros.expand('%{_sourcedir}') + config_hash = b.macros.expand('%{' + cfg[rpc_version] + '}') + repo_path = path.join(source_dir, cfg[rpc_repo_name]) + download.get_file( + cfg[rpc_repo] + '?fetch?checkout=' + cfg[rpc_branch], + repo_path, bopts, b) + repo = git.repo(repo_path) + repo_hash = repo.head() + if config_hash != repo_hash: + update = True + update_str = 'UPDATE' + else: + update = False + update_str = 'matching' + print(cfg[rpc_label] + ': ' + update_str + ' config:' + + config_hash + ' repo:' + repo_hash) + b.macros[cfg[rpc_version]] = repo_hash + tarball = b.macros.expand(cfg[rpc_package]) + b.macros.set_write_map('hashes') + b.macros[tarball] = 'NO-HASH NO-HASH' + b.macros.unset_write_map() + tarball_path = path.join(source_dir, + b.macros.expand(cfg[rpc_package])) + download.get_file(b.macros.expand(cfg[rpc_snapshot]), + tarball_path, bopts, b) + tarball_hash = checksum_sha512_base64(tarball_path) + if update and not argopts.dry_run: + config_patch(b.macros.expand('%{_configdir}'), + b.macros.expand(cfg[rpc_config]), + cfg[rpc_version], config_hash, repo_hash, + tarball_hash) + del b + except error.general as gerr: + log.stderr(str(gerr)) + log.stderr('Configuration load FAILED') + b = None + except error.general as gerr: + log.stderr(str(gerr)) + log.stderr('Build FAILED') + ec = 1 + except error.internal as ierr: + log.stderr(str(ierr)) + log.stderr('Internal Build FAILED') + ec = 1 + except error.exit as eerr: + pass + except KeyboardInterrupt: + log.notice('abort: user terminated') + ec = 1 + except: + raise + log.notice('abort: unknown error') + ec = 1 + sys.exit(ec) + + +if __name__ == "__main__": + run() -- cgit v1.2.3