Complete Yocto mirror with license table for TQMa6UL (2038-compliance)

- 264 license table entries with exact download URLs (224/264 resolved)
- Complete sources/ directory with all BitBake recipes
- Build configuration: tqma6ul-multi-mba6ulx, spaetzle (musl)
- Full traceability for Softwarefreigabeantrag
- GCC 13.4.0, Linux 6.6.102, U-Boot 2023.04, musl 1.2.4
- License distribution: GPL-2.0 (24), MIT (23), GPL-2.0+ (18), BSD-3 (16)

sources/poky/meta/lib/oe/rootfs.py

@@ -0,0 +1,438 @@
+#
+# Copyright OpenEmbedded Contributors
+#
+# SPDX-License-Identifier: GPL-2.0-only
+#
+from abc import ABCMeta, abstractmethod
+from oe.utils import execute_pre_post_process
+from oe.package_manager import *
+from oe.manifest import *
+import oe.path
+import shutil
+import os
+import subprocess
+import re
+
+class Rootfs(object, metaclass=ABCMeta):
+    """
+    This is an abstract class. Do not instantiate this directly.
+    """
+
+    def __init__(self, d, progress_reporter=None, logcatcher=None):
+        self.d = d
+        self.pm = None
+        self.image_rootfs = self.d.getVar('IMAGE_ROOTFS')
+        self.deploydir = self.d.getVar('IMGDEPLOYDIR')
+        self.progress_reporter = progress_reporter
+        self.logcatcher = logcatcher
+
+        self.install_order = Manifest.INSTALL_ORDER
+
+    @abstractmethod
+    def _create(self):
+        pass
+
+    @abstractmethod
+    def _get_delayed_postinsts(self):
+        pass
+
+    @abstractmethod
+    def _save_postinsts(self):
+        pass
+
+    @abstractmethod
+    def _log_check(self):
+        pass
+
+    def _log_check_common(self, type, match):
+        # Ignore any lines containing log_check to avoid recursion, and ignore
+        # lines beginning with a + since sh -x may emit code which isn't
+        # actually executed, but may contain error messages
+        excludes = [ 'log_check', r'^\+' ]
+        if hasattr(self, 'log_check_expected_regexes'):
+            excludes.extend(self.log_check_expected_regexes)
+        # Insert custom log_check excludes
+        excludes += [x for x in (self.d.getVar("IMAGE_LOG_CHECK_EXCLUDES") or "").split(" ") if x]
+        excludes = [re.compile(x) for x in excludes]
+        r = re.compile(match)
+        log_path = self.d.expand("${T}/log.do_rootfs")
+        messages = []
+        with open(log_path, 'r') as log:
+            for line in log:
+                if self.logcatcher and self.logcatcher.contains(line.rstrip()):
+                    continue
+                for ee in excludes:
+                    m = ee.search(line)
+                    if m:
+                        break
+                if m:
+                    continue
+
+                m = r.search(line)
+                if m:
+                    messages.append('[log_check] %s' % line)
+        if messages:
+            if len(messages) == 1:
+                msg = '1 %s message' % type
+            else:
+                msg = '%d %s messages' % (len(messages), type)
+            msg = '[log_check] %s: found %s in the logfile:\n%s' % \
+                (self.d.getVar('PN'), msg, ''.join(messages))
+            if type == 'error':
+                bb.fatal(msg)
+            else:
+                bb.warn(msg)
+
+    def _log_check_warn(self):
+        self._log_check_common('warning', '^(warn|Warn|WARNING:)')
+
+    def _log_check_error(self):
+        self._log_check_common('error', self.log_check_regex)
+
+    def _insert_feed_uris(self):
+        if bb.utils.contains("IMAGE_FEATURES", "package-management",
+                         True, False, self.d):
+            self.pm.insert_feeds_uris(self.d.getVar('PACKAGE_FEED_URIS') or "",
+                self.d.getVar('PACKAGE_FEED_BASE_PATHS') or "",
+                self.d.getVar('PACKAGE_FEED_ARCHS'))
+
+
+    """
+    The _cleanup() method should be used to clean-up stuff that we don't really
+    want to end up on target. For example, in the case of RPM, the DB locks.
+    The method is called, once, at the end of create() method.
+    """
+    @abstractmethod
+    def _cleanup(self):
+        pass
+
+    def _setup_dbg_rootfs(self, package_paths):
+        gen_debugfs = self.d.getVar('IMAGE_GEN_DEBUGFS') or '0'
+        if gen_debugfs != '1':
+           return
+
+        bb.note("  Renaming the original rootfs...")
+        try:
+            shutil.rmtree(self.image_rootfs + '-orig')
+        except:
+            pass
+        bb.utils.rename(self.image_rootfs, self.image_rootfs + '-orig')
+
+        bb.note("  Creating debug rootfs...")
+        bb.utils.mkdirhier(self.image_rootfs)
+
+        bb.note("  Copying back package database...")
+        for path in package_paths:
+            bb.utils.mkdirhier(self.image_rootfs + os.path.dirname(path))
+            if os.path.isdir(self.image_rootfs + '-orig' + path):
+                shutil.copytree(self.image_rootfs + '-orig' + path, self.image_rootfs + path, symlinks=True)
+            elif os.path.isfile(self.image_rootfs + '-orig' + path):
+                shutil.copyfile(self.image_rootfs + '-orig' + path, self.image_rootfs + path)
+
+        # Copy files located in /usr/lib/debug or /usr/src/debug
+        for dir in ["/usr/lib/debug", "/usr/src/debug"]:
+            src = self.image_rootfs + '-orig' + dir
+            if os.path.exists(src):
+                dst = self.image_rootfs + dir
+                bb.utils.mkdirhier(os.path.dirname(dst))
+                shutil.copytree(src, dst)
+
+        # Copy files with suffix '.debug' or located in '.debug' dir.
+        for root, dirs, files in os.walk(self.image_rootfs + '-orig'):
+            relative_dir = root[len(self.image_rootfs + '-orig'):]
+            for f in files:
+                if f.endswith('.debug') or '/.debug' in relative_dir:
+                    bb.utils.mkdirhier(self.image_rootfs + relative_dir)
+                    shutil.copy(os.path.join(root, f),
+                                self.image_rootfs + relative_dir)
+
+        bb.note("  Install complementary '*-dbg' packages...")
+        self.pm.install_complementary('*-dbg')
+
+        if self.d.getVar('PACKAGE_DEBUG_SPLIT_STYLE') == 'debug-with-srcpkg':
+            bb.note("  Install complementary '*-src' packages...")
+            self.pm.install_complementary('*-src')
+
+        """
+        Install additional debug packages. Possibility to install additional packages,
+        which are not automatically installed as complementary package of
+        standard one, e.g. debug package of static libraries.
+        """
+        extra_debug_pkgs = self.d.getVar('IMAGE_INSTALL_DEBUGFS')
+        if extra_debug_pkgs:
+            bb.note("  Install extra debug packages...")
+            self.pm.install(extra_debug_pkgs.split(), True)
+
+        bb.note("  Removing package database...")
+        for path in package_paths:
+            if os.path.isdir(self.image_rootfs + path):
+                shutil.rmtree(self.image_rootfs + path)
+            elif os.path.isfile(self.image_rootfs + path):
+                os.remove(self.image_rootfs + path)
+
+        bb.note("  Rename debug rootfs...")
+        try:
+            shutil.rmtree(self.image_rootfs + '-dbg')
+        except:
+            pass
+        bb.utils.rename(self.image_rootfs, self.image_rootfs + '-dbg')
+
+        bb.note("  Restoring original rootfs...")
+        bb.utils.rename(self.image_rootfs + '-orig', self.image_rootfs)
+
+    def _exec_shell_cmd(self, cmd):
+        try:
+            subprocess.check_output(cmd, stderr=subprocess.STDOUT)
+        except subprocess.CalledProcessError as e:
+            return("Command '%s' returned %d:\n%s" % (e.cmd, e.returncode, e.output))
+
+        return None
+
+    def create(self):
+        bb.note("###### Generate rootfs #######")
+        pre_process_cmds = self.d.getVar("ROOTFS_PREPROCESS_COMMAND")
+        post_process_cmds = self.d.getVar("ROOTFS_POSTPROCESS_COMMAND")
+        rootfs_post_install_cmds = self.d.getVar('ROOTFS_POSTINSTALL_COMMAND')
+
+        def make_last(command, commands):
+            commands = commands.split()
+            if command in commands:
+                commands.remove(command)
+                commands.append(command)
+            return "".join(commands)
+
+        # We want this to run as late as possible, in particular after
+        # systemd_sysusers_create and set_user_group. Using :append is not enough
+        make_last("tidy_shadowutils_files", post_process_cmds)
+        make_last("rootfs_reproducible", post_process_cmds)
+
+        execute_pre_post_process(self.d, pre_process_cmds)
+
+        if self.progress_reporter:
+            self.progress_reporter.next_stage()
+
+        # call the package manager dependent create method
+        self._create()
+
+        sysconfdir = self.image_rootfs + self.d.getVar('sysconfdir')
+        bb.utils.mkdirhier(sysconfdir)
+        with open(sysconfdir + "/version", "w+") as ver:
+            ver.write(self.d.getVar('BUILDNAME') + "\n")
+
+        execute_pre_post_process(self.d, rootfs_post_install_cmds)
+
+        self.pm.run_intercepts()
+
+        execute_pre_post_process(self.d, post_process_cmds)
+
+        if self.progress_reporter:
+            self.progress_reporter.next_stage()
+
+        if bb.utils.contains("IMAGE_FEATURES", "read-only-rootfs",
+                         True, False, self.d) and \
+           not bb.utils.contains("IMAGE_FEATURES",
+                         "read-only-rootfs-delayed-postinsts",
+                         True, False, self.d):
+            delayed_postinsts = self._get_delayed_postinsts()
+            if delayed_postinsts is not None:
+                bb.fatal("The following packages could not be configured "
+                         "offline and rootfs is read-only: %s" %
+                         delayed_postinsts)
+
+        if self.d.getVar('USE_DEVFS') != "1":
+            self._create_devfs()
+
+        self._uninstall_unneeded()
+
+        if self.progress_reporter:
+            self.progress_reporter.next_stage()
+
+        self._insert_feed_uris()
+
+        self._run_ldconfig()
+
+        if self.d.getVar('USE_DEPMOD') != "0":
+            self._generate_kernel_module_deps()
+
+        self._cleanup()
+        self._log_check()
+
+        if self.progress_reporter:
+            self.progress_reporter.next_stage()
+
+
+    def _uninstall_unneeded(self):
+        # Remove the run-postinsts package if no delayed postinsts are found
+        delayed_postinsts = self._get_delayed_postinsts()
+        if delayed_postinsts is None:
+            if os.path.exists(self.d.expand("${IMAGE_ROOTFS}${sysconfdir}/init.d/run-postinsts")) or os.path.exists(self.d.expand("${IMAGE_ROOTFS}${systemd_system_unitdir}/run-postinsts.service")):
+                self.pm.remove(["run-postinsts"])
+
+        image_rorfs = bb.utils.contains("IMAGE_FEATURES", "read-only-rootfs",
+                                        True, False, self.d) and \
+                      not bb.utils.contains("IMAGE_FEATURES",
+                                        "read-only-rootfs-delayed-postinsts",
+                                        True, False, self.d)
+
+        image_rorfs_force = self.d.getVar('FORCE_RO_REMOVE')
+
+        if image_rorfs or image_rorfs_force == "1":
+            # Remove components that we don't need if it's a read-only rootfs
+            unneeded_pkgs = self.d.getVar("ROOTFS_RO_UNNEEDED").split()
+            pkgs_installed = image_list_installed_packages(self.d)
+            # Make sure update-alternatives is removed last. This is
+            # because its database has to available while uninstalling
+            # other packages, allowing alternative symlinks of packages
+            # to be uninstalled or to be managed correctly otherwise.
+            provider = self.d.getVar("VIRTUAL-RUNTIME_update-alternatives")
+            pkgs_to_remove = sorted([pkg for pkg in pkgs_installed if pkg in unneeded_pkgs], key=lambda x: x == provider)
+
+            # update-alternatives provider is removed in its own remove()
+            # call because all package managers do not guarantee the packages
+            # are removed in the order they given in the list (which is
+            # passed to the command line). The sorting done earlier is
+            # utilized to implement the 2-stage removal.
+            if len(pkgs_to_remove) > 1:
+                self.pm.remove(pkgs_to_remove[:-1], False)
+            if len(pkgs_to_remove) > 0:
+                self.pm.remove([pkgs_to_remove[-1]], False)
+
+        if delayed_postinsts:
+            self._save_postinsts()
+            if image_rorfs:
+                bb.warn("There are post install scripts "
+                        "in a read-only rootfs")
+
+        post_uninstall_cmds = self.d.getVar("ROOTFS_POSTUNINSTALL_COMMAND")
+        execute_pre_post_process(self.d, post_uninstall_cmds)
+
+        runtime_pkgmanage = bb.utils.contains("IMAGE_FEATURES", "package-management",
+                                              True, False, self.d)
+        if not runtime_pkgmanage:
+            # Remove the package manager data files
+            self.pm.remove_packaging_data()
+
+    def _run_ldconfig(self):
+        if self.d.getVar('LDCONFIGDEPEND'):
+            bb.note("Executing: ldconfig -r " + self.image_rootfs + " -c new -v -X")
+            self._exec_shell_cmd(['ldconfig', '-r', self.image_rootfs, '-c',
+                                  'new', '-v', '-X'])
+
+        image_rorfs = bb.utils.contains("IMAGE_FEATURES", "read-only-rootfs",
+                                        True, False, self.d)
+        ldconfig_in_features = bb.utils.contains("DISTRO_FEATURES", "ldconfig",
+                                                 True, False, self.d)
+        if image_rorfs or not ldconfig_in_features:
+            ldconfig_cache_dir = os.path.join(self.image_rootfs, "var/cache/ldconfig")
+            if os.path.exists(ldconfig_cache_dir):
+                bb.note("Removing ldconfig auxiliary cache...")
+                shutil.rmtree(ldconfig_cache_dir)
+
+    def _check_for_kernel_modules(self, modules_dir):
+        for root, dirs, files in os.walk(modules_dir, topdown=True):
+            for name in files:
+                found_ko = name.endswith((".ko", ".ko.gz", ".ko.xz", ".ko.zst"))
+                if found_ko:
+                    return found_ko
+        return False
+
+    def _generate_kernel_module_deps(self):
+        modules_dir = os.path.join(self.image_rootfs, 'lib', 'modules')
+        # if we don't have any modules don't bother to do the depmod
+        if not self._check_for_kernel_modules(modules_dir):
+            bb.note("No Kernel Modules found, not running depmod")
+            return
+
+        pkgdatadir = self.d.getVar('PKGDATA_DIR')
+
+        # PKGDATA_DIR can include multiple kernels so we run depmod for each
+        # one of them.
+        for direntry in os.listdir(pkgdatadir):
+            match = re.match('(.*)-depmod', direntry)
+            if not match:
+                continue
+            kernel_package_name = match.group(1)
+
+            kernel_abi_ver_file = oe.path.join(pkgdatadir, direntry, kernel_package_name + '-abiversion')
+            if not os.path.exists(kernel_abi_ver_file):
+                bb.fatal("No kernel-abiversion file found (%s), cannot run depmod, aborting" % kernel_abi_ver_file)
+
+            with open(kernel_abi_ver_file) as f:
+                kernel_ver = f.read().strip(' \n')
+
+            versioned_modules_dir = os.path.join(self.image_rootfs, modules_dir, kernel_ver)
+
+            bb.utils.mkdirhier(versioned_modules_dir)
+
+            bb.note("Running depmodwrapper for %s ..." % versioned_modules_dir)
+            if self._exec_shell_cmd(['depmodwrapper', '-a', '-b', self.image_rootfs, kernel_ver, kernel_package_name]):
+                bb.fatal("Kernel modules dependency generation failed")
+
+    """
+    Create devfs:
+    * IMAGE_DEVICE_TABLE is the old name to an absolute path to a device table file
+    * IMAGE_DEVICE_TABLES is a new name for a file, or list of files, seached
+      for in the BBPATH
+    If neither are specified then the default name of files/device_table-minimal.txt
+    is searched for in the BBPATH (same as the old version.)
+    """
+    def _create_devfs(self):
+        devtable_list = []
+        devtable = self.d.getVar('IMAGE_DEVICE_TABLE')
+        if devtable is not None:
+            devtable_list.append(devtable)
+        else:
+            devtables = self.d.getVar('IMAGE_DEVICE_TABLES')
+            if devtables is None:
+                devtables = 'files/device_table-minimal.txt'
+            for devtable in devtables.split():
+                devtable_list.append("%s" % bb.utils.which(self.d.getVar('BBPATH'), devtable))
+
+        for devtable in devtable_list:
+            self._exec_shell_cmd(["makedevs", "-r",
+                                  self.image_rootfs, "-D", devtable])
+
+
+def get_class_for_type(imgtype):
+    import importlib
+    mod = importlib.import_module('oe.package_manager.' + imgtype + '.rootfs')
+    return mod.PkgRootfs
+
+def variable_depends(d, manifest_dir=None):
+    img_type = d.getVar('IMAGE_PKGTYPE')
+    cls = get_class_for_type(img_type)
+    return cls._depends_list()
+
+def create_rootfs(d, manifest_dir=None, progress_reporter=None, logcatcher=None):
+    env_bkp = os.environ.copy()
+
+    img_type = d.getVar('IMAGE_PKGTYPE')
+
+    cls = get_class_for_type(img_type)
+    cls(d, manifest_dir, progress_reporter, logcatcher).create()
+    os.environ.clear()
+    os.environ.update(env_bkp)
+
+
+def image_list_installed_packages(d, rootfs_dir=None):
+    # Theres no rootfs for baremetal images
+    if bb.data.inherits_class('baremetal-image', d):
+        return ""
+
+    if not rootfs_dir:
+        rootfs_dir = d.getVar('IMAGE_ROOTFS')
+
+    img_type = d.getVar('IMAGE_PKGTYPE')
+
+    import importlib
+    cls = importlib.import_module('oe.package_manager.' + img_type)
+    return cls.PMPkgsList(d, rootfs_dir).list_pkgs()
+
+if __name__ == "__main__":
+    """
+    We should be able to run this as a standalone script, from outside bitbake
+    environment.
+    """
+    """
+    TBD
+    """