Complete Yocto mirror with license table for TQMa6UL (2038-compliance)

- 264 license table entries with exact download URLs (224/264 resolved)
- Complete sources/ directory with all BitBake recipes
- Build configuration: tqma6ul-multi-mba6ulx, spaetzle (musl)
- Full traceability for Softwarefreigabeantrag
- GCC 13.4.0, Linux 6.6.102, U-Boot 2023.04, musl 1.2.4
- License distribution: GPL-2.0 (24), MIT (23), GPL-2.0+ (18), BSD-3 (16)

sources/meta-arm/scripts/layer-overview.py

@@ -0,0 +1,74 @@
+#! /usr/bin/env python3
+
+"""
+Print an overview of the layer to help writing release notes.
+
+Output includes sublayers, machines, recipes.
+"""
+
+import argparse
+import sys
+
+# TODO:
+# - More human-readable output
+# - Diff mode, give two revisions and list the changes
+
+def is_layer(path):
+    """
+    Determine if this path looks like a layer (is a directory and contains conf/layer.conf).
+    """
+    return path.is_dir() and (path / "conf" / "layer.conf").exists()
+
+
+def print_layer(layer):
+    """
+    Print a summary of the layer.
+    """
+    print(layer.name)
+
+    machines = sorted(p for p in layer.glob("conf/machine/*.conf"))
+    if machines:
+        print(" Machines")
+        for m in machines:
+            print(f"  {m.stem}")
+        print()
+
+    recipes = sorted((p for p in layer.glob("recipes-*/*/*.bb")), key=lambda p:p.name)
+    if recipes:
+        print(" Recipes")
+        for r in recipes:
+            if "_" in r.stem:
+                pn, pv = r.stem.rsplit("_", 1)
+                print(f"  {pn} {pv}")
+            else:
+                print(f"  {r.stem}")
+        print()
+
+
+parser = argparse.ArgumentParser()
+parser.add_argument("repository")
+parser.add_argument("revision", nargs="?")
+args = parser.parse_args()
+
+if args.revision:
+    import gitpathlib
+    base = gitpathlib.GitPath(args.repository, args.revision)
+else:
+    import pathlib
+    base = pathlib.Path(args.repository)
+
+if is_layer(base):
+    print_layer(base)
+else:
+    sublayers = sorted(p for p in base.glob("meta-*") if is_layer(p))
+    if sublayers:
+        print("Sub-Layers")
+        for l in sublayers:
+            print(f" {l.name}")
+        print()
+
+        for layer in sublayers:
+            print_layer(layer)
+    else:
+        print(f"No layers found in {base}", file=sys.stderr)
+        sys.exit(1)

sources/meta-arm/scripts/machine-summary-overview.txt.jinja

@@ -0,0 +1,12 @@
+Machine Overview
+Generated at {{ timestamp }}.
+{% for machine, data in data|dictsort %}
+
+MACHINE: {{ machine }}
+{% for recipe in recipes|sort %}
+{% if recipe in data %}
+{% set details = data[recipe] %}
+{{ details.recipe }}: {{ details.version }}
+{% endif %}
+{% endfor %}
+{% endfor %}

sources/meta-arm/scripts/machine-summary.py

@@ -0,0 +1,239 @@
+#! /usr/bin/env python3
+
+import argparse
+import datetime
+import os
+import pathlib
+import re
+import sys
+
+import jinja2
+
+def trim_pv(pv):
+    """
+    Strip anything after +git from the PV
+    """
+    return "".join(pv.partition("+git")[:2])
+
+def needs_update(version, upstream):
+    """
+    Do a dumb comparison to determine if the version needs to be updated.
+    """
+    if "+git" in version:
+        # strip +git and see if this is a post-release snapshot
+        version = version.replace("+git", "")
+    return version != upstream
+
+def safe_patches(patches):
+    for info in patches:
+        if info["status"] in ("Denied", "Pending", "Unknown"):
+            return False
+    return True
+
+def layer_path(layername: str, d) -> pathlib.Path:
+    """
+    Return the path to the specified layer, or None if the layer isn't present.
+    """
+    if not hasattr(layer_path, "cache"):
+        # Don't use functools.lru_cache as we don't want d changing to invalidate the cache
+        layer_path.cache = {}
+
+    if layername in layer_path.cache:
+        return layer_path.cache[layername]
+
+    bbpath = d.getVar("BBPATH").split(":")
+    pattern = d.getVar('BBFILE_PATTERN_' + layername)
+    for path in reversed(sorted(bbpath)):
+        if re.match(pattern, path + "/"):
+            layer_path.cache[layername] = pathlib.Path(path)
+            return layer_path.cache[layername]
+    return None
+
+def get_url_for_patch(layer: str, localpath: pathlib.Path, d) -> str:
+    relative = localpath.relative_to(layer_path(layer, d))
+
+    # TODO: use layerindexlib
+    # TODO: assumes default branch
+    if layer == "core":
+        return f"https://git.openembedded.org/openembedded-core/tree/meta/{relative}"
+    elif layer in ("meta-arm", "meta-arm-bsp", "arm-toolchain"):
+        return f"https://git.yoctoproject.org/meta-arm/tree/{layer}/{relative}"
+    else:
+        print(f"WARNING: Don't know web URL for layer {layer}", file=sys.stderr)
+        return None
+
+def extract_patch_info(src_uri, d):
+    """
+    Parse the specified patch entry from a SRC_URI and return (base name, layer name, status) tuple
+    """
+    import bb.fetch, bb.utils
+
+    info = {}
+    localpath = pathlib.Path(bb.fetch.decodeurl(src_uri)[2])
+    info["name"] = localpath.name
+    info["layer"] = bb.utils.get_file_layer(str(localpath), d)
+    info["url"] = get_url_for_patch(info["layer"], localpath, d)
+
+    status = "Unknown"
+    with open(localpath, errors="ignore") as f:
+        m = re.search(r"^[\t ]*Upstream[-_ ]Status:?[\t ]*(\w*)", f.read(), re.IGNORECASE | re.MULTILINE)
+        if m:
+            # TODO: validate
+            status = m.group(1)
+    info["status"] = status
+    return info
+
+def harvest_data(machines, recipes):
+    import bb.tinfoil
+    with bb.tinfoil.Tinfoil() as tinfoil:
+        tinfoil.prepare(config_only=True)
+        corepath = layer_path("core", tinfoil.config_data)
+        sys.path.append(os.path.join(corepath, "lib"))
+    import oe.recipeutils
+    import oe.patch
+
+    # Queue of recipes that we're still looking for upstream releases for
+    to_check = list(recipes)
+
+    # Upstream releases
+    upstreams = {}
+    # Machines to recipes to versions
+    versions = {}
+
+    for machine in machines:
+        print(f"Gathering data for {machine}...")
+        os.environ["MACHINE"] = machine
+        with bb.tinfoil.Tinfoil() as tinfoil:
+            versions[machine] = {}
+
+            tinfoil.prepare(quiet=2)
+            for recipe in recipes:
+                try:
+                    d = tinfoil.parse_recipe(recipe)
+                except bb.providers.NoProvider:
+                    continue
+
+                if recipe in to_check:
+                    try:
+                        info = oe.recipeutils.get_recipe_upstream_version(d)
+                        upstreams[recipe] = info["version"]
+                        to_check.remove(recipe)
+                    except (bb.providers.NoProvider, KeyError):
+                        pass
+
+                details = versions[machine][recipe] = {}
+                details["recipe"] = d.getVar("PN")
+                details["version"] = trim_pv(d.getVar("PV"))
+                details["fullversion"] = d.getVar("PV")
+                details["patches"] = [extract_patch_info(p, d) for p in oe.patch.src_patches(d)]
+                details["patched"] = bool(details["patches"])
+                details["patches_safe"] = safe_patches(details["patches"])
+
+    # Now backfill the upstream versions
+    for machine in versions:
+        for recipe in versions[machine]:
+            data = versions[machine][recipe]
+            data["upstream"] = upstreams[recipe]
+            data["needs_update"] = needs_update(data["version"], data["upstream"])
+    return upstreams, versions
+
+# TODO can this be inferred from the list of recipes in the layer
+recipes = ("virtual/kernel",
+           "sbsa-acs",
+           "scp-firmware",
+           "trusted-firmware-a",
+           "trusted-firmware-m",
+           "edk2-firmware",
+           "u-boot",
+           "optee-os",
+           "optee-ftpm",
+           "hafnium",
+           "boot-wrapper-aarch64",
+           "gator-daemon",
+           "gn",
+           "opencsd",
+           "gcc-aarch64-none-elf-native",
+           "gcc-arm-none-eabi-native")
+
+
+class Format:
+    """
+    The name of this format
+    """
+    name = None
+    """
+    Registry of names to classes
+    """
+    registry = {}
+
+    def __init_subclass__(cls, **kwargs):
+        super().__init_subclass__(**kwargs)
+        assert cls.name
+        cls.registry[cls.name] = cls
+
+    @classmethod
+    def get_format(cls, name):
+        return cls.registry[name]()
+
+    def render(self, context, output: pathlib.Path):
+        pass
+
+    def get_template(self, name):
+        template_dir = os.path.dirname(os.path.abspath(__file__))
+        env = jinja2.Environment(
+            loader=jinja2.FileSystemLoader(template_dir),
+            extensions=['jinja2.ext.i18n'],
+            autoescape=jinja2.select_autoescape(),
+            trim_blocks=True,
+            lstrip_blocks=True
+        )
+
+        # We only need i18n for plurals
+        env.install_null_translations()
+
+        return env.get_template(name)
+
+class TextOverview(Format):
+    name = "overview.txt"
+
+    def render(self, context, output: pathlib.Path):
+        with open(output, "wt") as f:
+            f.write(self.get_template(f"machine-summary-overview.txt.jinja").render(context))
+
+class HtmlUpdates(Format):
+    name = "report"
+
+    def render(self, context, output: pathlib.Path):
+        if output.exists() and not output.is_dir():
+            print(f"{output} is not a directory", file=sys.stderr)
+            sys.exit(1)
+
+        if not output.exists():
+            output.mkdir(parents=True)
+
+        with open(output / "index.html", "wt") as f:
+            f.write(self.get_template(f"report-index.html.jinja").render(context))
+
+        subcontext = context.copy()
+        del subcontext["data"]
+        for machine, subdata in context["data"].items():
+            subcontext["machine"] = machine
+            subcontext["data"] = subdata
+            with open(output / f"{machine}.html", "wt") as f:
+                f.write(self.get_template(f"report-details.html.jinja").render(subcontext))
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="machine-summary")
+    parser.add_argument("machines", nargs="+", help="machine names", metavar="MACHINE")
+    parser.add_argument("-t", "--type", required=True, choices=Format.registry.keys())
+    parser.add_argument("-o", "--output", type=pathlib.Path, required=True)
+    args = parser.parse_args()
+
+    context = {}
+    # TODO: include git describe for meta-arm
+    context["timestamp"] = str(datetime.datetime.now().strftime("%c"))
+    context["recipes"] = sorted(recipes)
+    context["releases"], context["data"] = harvest_data(args.machines, recipes)
+
+    formatter = Format.get_format(args.type)
+    formatter.render(context, args.output)

sources/meta-arm/scripts/report-base.html.jinja

@@ -0,0 +1,35 @@
+<!DOCTYPE html>
+<html>
+  <head>
+    <title>{% block title %}{% endblock %}</title>
+    <meta name="viewport" content="width=device-width, initial-scale=1">
+    <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/bulma@0.9.1/css/bulma.min.css">
+  </head>
+  <body>
+    <section class="section">
+      {# TODO use position: sticky to glue this to the top #}
+      <nav class="breadcrumb is-large">
+        <ul>
+          <li class="{{ "is-active" if machine is undefined }}">
+            <a href="index.html">Recipe Report</a>
+          </li>
+          {% if machine is defined %}
+          <li class="is-active">
+            <a href="#">{{machine}}</a>
+          </li>
+          {% endif %}
+        </ul>
+      </nav>
+
+      <div class="content">
+        {% block content %}{% endblock %}
+      </div>
+    </section>
+
+    <footer class="footer">
+      <div class="content has-text-centered">
+        Generated by <code>machine-summary</code> at {{ timestamp }}.
+      </div>
+    </footer>
+  </body>
+</html>

sources/meta-arm/scripts/report-details.html.jinja

@@ -0,0 +1,64 @@
+{% extends "report-base.html.jinja" %}
+{% block title %}Recipe Report for {{ machine }}{% endblock %}
+
+{# Write a tag element using the Upstream-Status to determine the class. #}
+{% macro make_patch_tag(status) -%}
+  {% set status = status.split()[0] %}
+  {% if status in ("Unknown", "Pending") %}
+    {% set class = "is-danger" %}
+  {% elif status in ("Backport", "Accepted", "Inappropriate", "Denied") %}
+    {% set class = "is-success" %}
+  {% elif status in ("Submitted",) %}
+    {% set class = "is-info" %}
+  {% else %}
+    {% set class = "is-info" %}
+  {% endif %}
+  <span class="tag {{ class }}">{{ status }}</span>
+{%- endmacro %}
+
+{% block content %}
+  <!-- TODO table of contents -->
+
+  {% for name, data in data|dictsort if data.needs_update or data.patched %}
+  <h2 class="title is-4">
+    {{ data.recipe }} {{ data.fullversion }}
+    {% if name != data.recipe %}
+      (provides {{ name }})
+    {% endif %}
+    {% if data.needs_update %}<span class="tag is-danger">Upgrade Needed</span>{% endif %}
+    <a id="recipe-{{ data.recipe }}" class="has-text-grey-lighter">#</a>
+  </h2>
+
+  {% if data.needs_update %}
+  <p>
+    Recipe is version {{ data.fullversion }}, latest upstream release is <strong>{{ data.upstream }}</strong>.
+  </p>
+  {% endif%}
+
+  {% if data.patched %}
+  <table class="table is-striped is-bordered">
+    <thead>
+      <tr>
+        <th>Patch</th>
+        <th style="width: 20em">Layer</th>
+        <th style="width: 10em">Status</th>
+      </tr>
+    </thead>
+    <tbody>
+      {% for pinfo in data.patches %}
+      <tr>
+        <td>
+          {% if pinfo.url %}<a href="{{pinfo.url}}">{% endif %}
+          {{ pinfo.name }}
+          {% if pinfo.url %}</a>{% endif %}
+        </td>
+        <td>{{ pinfo.layer }}</td>
+        <!-- TODO: tooltip with full status? -->
+        <td class="has-text-centered">{{ make_patch_tag(pinfo.status)}}</td>
+      </tr>
+      {% endfor %}
+    </tbody>
+  </table>
+  {% endif %}
+  {% endfor %}
+{% endblock %}

sources/meta-arm/scripts/report-index.html.jinja

@@ -0,0 +1,50 @@
+{% extends "report-base.html.jinja" %}
+{% block title %}Recipe Report{% endblock %}
+
+{% block content %}
+  <table class="table is-striped">
+    <thead>
+      <tr>
+        <th>Machine</th>
+        {% for recipe in recipes|sort %}
+        <th>{{ recipe }} ({{releases[recipe]|default("?")}})</th>
+        {% endfor %}
+      </tr>
+    </thead>
+    <tbody>
+      {% for machine, data in data|dictsort %}
+      <tr>
+        <th><a href="{{machine}}.html">{{ machine }}</a></th>
+        {% for recipe in recipes|sort %}
+          {% if recipe in data %}
+            {% set details = data[recipe] %}
+            <td style="text-align: center">
+            <a href="{{machine}}.html#recipe-{{details.recipe}}">
+              {{ details.recipe if details.recipe != recipe}}
+              {{ details.version }}
+            </a>
+            {% if details.patches or details.needs_update %}
+            <br>
+            {% if details.patches %}
+              <span class="tag {{ "is-info" if details.patches_safe else "is-danger" }}">
+                {% trans count=details.patches|length %}
+                  {{ count }} Patch
+                {% pluralize %}
+                  {{ count }} Patches
+                {% endtrans %}
+              </span>
+            {% endif %}
+            {% if details.needs_update %}
+              <span class="tag is-danger">Upgrade</span>
+            {% endif %}
+            {% endif %}
+            </td>
+          {% else %}
+            <td>-</td>
+          {% endif %}
+        {% endfor %}
+      </tr>
+      {% endfor %}
+    </tbody>
+  </table>
+{% endblock %}

sources/meta-arm/scripts/runfvp

@@ -0,0 +1,109 @@
+#! /usr/bin/env python3
+
+import itertools
+import os
+import pathlib
+import signal
+import sys
+import threading
+
+import logging
+logger = logging.getLogger("RunFVP")
+
+# Add meta-arm/lib/ to path
+libdir = pathlib.Path(__file__).parents[1] / "meta-arm" / "lib"
+sys.path.insert(0, str(libdir))
+
+from fvp import conffile, terminal, runner
+
+def parse_args(arguments):
+    import argparse
+    terminals = terminal.terminals
+
+    parser = argparse.ArgumentParser(description="Run images in a FVP")
+    parser.add_argument("config", nargs="?", help="Machine name or path to .fvpconf file")
+    group = parser.add_mutually_exclusive_group()
+    group.add_argument("-t", "--terminals", choices=terminals.all_terminals(), default=terminals.preferred_terminal(), help="Automatically start terminals (default: %(default)s)")
+    group.add_argument("-c", "--console", action="store_true", help="Attach the first uart to stdin/stdout")
+    parser.add_argument("--verbose", action="store_true", help="Output verbose logging")
+    parser.usage = f"{parser.format_usage().strip()} -- [ arguments passed to FVP ]"
+    # TODO option for telnet vs netcat
+
+    # If the arguments contains -- then everything after it should be passed to the FVP binary directly.
+    if "--" in arguments:
+        i = arguments.index("--")
+        fvp_args = arguments[i+1:]
+        arguments = arguments[:i]
+    else:
+        fvp_args = []
+
+    args = parser.parse_args(args=arguments)
+    logging.basicConfig(level=args.verbose and logging.DEBUG or logging.WARNING,
+                        format='\033[G%(levelname)s: %(message)s')
+
+    # If we're hooking up the console, don't start any terminals
+    if args.console:
+        args.terminals = "none"
+
+    logger.debug(f"Parsed arguments: {vars(args)}")
+    logger.debug(f"FVP arguments: {fvp_args}")
+    return args, fvp_args
+
+def start_fvp(args, fvpconf, extra_args):
+    fvp = runner.FVPRunner(logger)
+    try:
+        fvp.start(fvpconf, extra_args, args.terminals)
+
+        if args.console:
+            config = fvp.getConfig()
+            expected_terminal = config["consoles"].get("default")
+            if expected_terminal is None:
+                logger.error("--console used but FVP_CONSOLE not set in machine configuration")
+                return 1
+            port_stdout, log_stdout = itertools.tee(fvp.stdout, 2)
+            parser = runner.ConsolePortParser(port_stdout)
+            port = parser.parse_port(expected_terminal)
+
+            def debug_log():
+                for line in log_stdout:
+                    line = line.strip().decode(errors='ignore')
+                    logger.debug(f'FVP output: {line}')
+            log_thread = threading.Thread(None, debug_log)
+            log_thread.start()
+
+            telnet = fvp.create_telnet(port)
+            telnet.wait()
+            logger.debug(f"Telnet quit, cancelling tasks")
+        else:
+            for line in fvp.stdout:
+                print(line.strip().decode(errors='ignore'))
+
+    finally:
+        return fvp.stop()
+
+
+def runfvp(cli_args):
+    args, extra_args = parse_args(cli_args)
+    if args.config and pathlib.Path(args.config).exists():
+        config_file = args.config
+    else:
+        config_file = conffile.find(args.config)
+    return start_fvp(args, config_file, extra_args)
+
+
+if __name__ == "__main__":
+    try:
+        # Set the process group so that it's possible to kill runfvp and
+        # everything it spawns easily.
+        # Ignore permission errors happening when spawned from an other process
+        # for example run from except
+        try:
+            os.setpgid(0, 0)
+        except PermissionError:
+            pass
+        if sys.stdin.isatty():
+            signal.signal(signal.SIGTTOU, signal.SIG_IGN)
+            os.tcsetpgrp(sys.stdin.fileno(), os.getpgrp())
+        sys.exit(runfvp(sys.argv[1:]))
+    except KeyboardInterrupt:
+        pass