Complete Yocto mirror with license table for TQMa6UL (2038-compliance)

- 264 license table entries with exact download URLs (224/264 resolved)
- Complete sources/ directory with all BitBake recipes
- Build configuration: tqma6ul-multi-mba6ulx, spaetzle (musl)
- Full traceability for Softwarefreigabeantrag
- GCC 13.4.0, Linux 6.6.102, U-Boot 2023.04, musl 1.2.4
- License distribution: GPL-2.0 (24), MIT (23), GPL-2.0+ (18), BSD-3 (16)

sources/poky/scripts/lib/wic/__init__.py

@@ -0,0 +1,10 @@
+#!/usr/bin/env python3
+#
+# Copyright (c) 2007 Red Hat, Inc.
+# Copyright (c) 2011 Intel, Inc.
+#
+# SPDX-License-Identifier: GPL-2.0-only
+#
+
+class WicError(Exception):
+    pass

sources/poky/scripts/lib/wic/canned-wks/common.wks.inc

@@ -0,0 +1,3 @@
+# This file is included into 3 canned wks files from this directory
+part /boot --source bootimg-pcbios --ondisk sda --label boot --active --align 1024
+part / --source rootfs --use-uuid --fstype=ext4 --label platform --align 1024

sources/poky/scripts/lib/wic/canned-wks/directdisk-bootloader-config.cfg

@@ -0,0 +1,27 @@
+# This is an example configuration file for syslinux.
+TIMEOUT 50
+ALLOWOPTIONS 1
+SERIAL 0 115200
+PROMPT 0
+
+UI vesamenu.c32
+menu title Select boot options
+menu tabmsg Press [Tab] to edit, [Return] to select
+
+DEFAULT Graphics console boot
+
+LABEL Graphics console boot
+KERNEL /vmlinuz
+APPEND label=boot rootwait
+
+LABEL Serial console boot
+KERNEL /vmlinuz
+APPEND label=boot rootwait console=ttyS0,115200
+
+LABEL Graphics console install
+KERNEL /vmlinuz
+APPEND label=install rootwait
+
+LABEL Serial console install
+KERNEL /vmlinuz
+APPEND label=install rootwait console=ttyS0,115200

sources/poky/scripts/lib/wic/canned-wks/directdisk-bootloader-config.wks

@@ -0,0 +1,8 @@
+# short-description: Create a 'pcbios' direct disk image with custom bootloader config
+# long-description: Creates a partitioned legacy BIOS disk image that the user
+# can directly dd to boot media. The bootloader configuration source is a user file.
+
+include common.wks.inc
+
+bootloader --configfile="directdisk-bootloader-config.cfg"
+

sources/poky/scripts/lib/wic/canned-wks/directdisk-gpt.wks

@@ -0,0 +1,10 @@
+# short-description: Create a 'pcbios' direct disk image
+# long-description: Creates a partitioned legacy BIOS disk image that the user
+# can directly dd to boot media.
+
+
+part /boot --source bootimg-pcbios --ondisk sda --label boot --active --align 1024
+part / --source rootfs --ondisk sda --fstype=ext4 --label platform --align 1024 --use-uuid
+
+bootloader  --ptable gpt --timeout=0  --append="rootwait rootfstype=ext4 video=vesafb vga=0x318 console=tty0 console=ttyS0,115200n8"
+

sources/poky/scripts/lib/wic/canned-wks/directdisk-multi-rootfs.wks

@@ -0,0 +1,23 @@
+# short-description: Create multi rootfs image using rootfs plugin
+# long-description: Creates a partitioned disk image with two rootfs partitions
+# using rootfs plugin.
+#
+# Partitions can use either
+#   - indirect rootfs references to image recipe(s):
+#     wic create directdisk-multi-indirect-recipes -e core-image-minimal \
+#         --rootfs-dir rootfs1=core-image-minimal
+#         --rootfs-dir rootfs2=core-image-minimal-dev
+#
+#   - or paths to rootfs directories:
+#     wic create directdisk-multi-rootfs \
+#         --rootfs-dir rootfs1=tmp/work/qemux86_64-poky-linux/core-image-minimal/1.0-r0/rootfs/
+#         --rootfs-dir rootfs2=tmp/work/qemux86_64-poky-linux/core-image-minimal-dev/1.0-r0/rootfs/
+#
+#   - or any combinations of -r and --rootfs command line options
+
+part /boot --source bootimg-pcbios --ondisk sda --label boot --active --align 1024
+part / --source rootfs --rootfs-dir=rootfs1 --ondisk sda --fstype=ext4 --label platform --align 1024
+part /rescue --source rootfs --rootfs-dir=rootfs2 --ondisk sda --fstype=ext4 --label secondary --align 1024
+
+bootloader  --timeout=0  --append="rootwait rootfstype=ext4 video=vesafb vga=0x318 console=tty0 console=ttyS0,115200n8"
+

sources/poky/scripts/lib/wic/canned-wks/directdisk.wks

@@ -0,0 +1,8 @@
+# short-description: Create a 'pcbios' direct disk image
+# long-description: Creates a partitioned legacy BIOS disk image that the user
+# can directly dd to boot media.
+
+include common.wks.inc
+
+bootloader  --timeout=0  --append="rootwait rootfstype=ext4 video=vesafb vga=0x318 console=tty0 console=ttyS0,115200n8"
+

sources/poky/scripts/lib/wic/canned-wks/efi-bootdisk.wks.in

@@ -0,0 +1,3 @@
+bootloader --ptable gpt
+part /boot --source rootfs --rootfs-dir=${IMAGE_ROOTFS}/boot --fstype=vfat --label boot --active --align 1024 --use-uuid --overhead-factor 1.1
+part / --source rootfs --fstype=ext4 --label root --align 1024 --exclude-path boot/

sources/poky/scripts/lib/wic/canned-wks/mkefidisk.wks

@@ -0,0 +1,11 @@
+# short-description: Create an EFI disk image
+# long-description: Creates a partitioned EFI disk image that the user
+# can directly dd to boot media.
+
+part /boot --source bootimg-efi --sourceparams="loader=grub-efi" --ondisk sda --label msdos --active --align 1024
+
+part / --source rootfs --ondisk sda --fstype=ext4 --label platform --align 1024 --use-uuid
+
+part swap --ondisk sda --size 44 --label swap1 --fstype=swap
+
+bootloader --ptable gpt --timeout=5 --append="rootfstype=ext4 console=ttyS0,115200 console=tty0"

sources/poky/scripts/lib/wic/canned-wks/mkhybridiso.wks

@@ -0,0 +1,7 @@
+# short-description: Create a hybrid ISO image
+# long-description: Creates an EFI and legacy bootable hybrid ISO image
+# which can be used on optical media as well as USB media.
+
+part /boot --source isoimage-isohybrid --sourceparams="loader=grub-efi,image_name=HYBRID_ISO_IMG" --ondisk cd --label HYBRIDISO
+
+bootloader  --timeout=15  --append=""

sources/poky/scripts/lib/wic/canned-wks/qemuloongarch.wks

@@ -0,0 +1,3 @@
+# short-description: Create qcow2 image for LoongArch QEMU machines
+
+part / --source rootfs --fstype=ext4 --label root --align 4096 --size 5G

sources/poky/scripts/lib/wic/canned-wks/qemuriscv.wks

@@ -0,0 +1,3 @@
+# short-description: Create qcow2 image for RISC-V QEMU machines
+
+part / --source rootfs --fstype=ext4 --label root --align 4096 --size 5G

sources/poky/scripts/lib/wic/canned-wks/qemux86-directdisk.wks

@@ -0,0 +1,8 @@
+# short-description: Create a qemu machine 'pcbios' direct disk image
+# long-description: Creates a partitioned legacy BIOS disk image that the user
+# can directly use to boot a qemu machine.
+
+include common.wks.inc
+
+bootloader  --timeout=0  --append="rw oprofile.timer=1 rootfstype=ext4 console=tty console=ttyS0 "
+

sources/poky/scripts/lib/wic/canned-wks/sdimage-bootpart.wks

@@ -0,0 +1,6 @@
+# short-description: Create SD card image with a boot partition
+# long-description: Creates a partitioned SD card image. Boot files
+# are located in the first vfat partition.
+
+part /boot --source bootimg-partition --ondisk mmcblk0 --fstype=vfat --label boot --active --align 4 --size 16
+part / --source rootfs --ondisk mmcblk0 --fstype=ext4 --label root --align 4

sources/poky/scripts/lib/wic/canned-wks/systemd-bootdisk.wks

@@ -0,0 +1,11 @@
+# short-description: Create an EFI disk image with systemd-boot
+# long-description: Creates a partitioned EFI disk image that the user
+# can directly dd to boot media. The selected bootloader is systemd-boot.
+
+part /boot --source bootimg-efi --sourceparams="loader=systemd-boot" --ondisk sda --label msdos --active --align 1024 --use-uuid
+
+part / --source rootfs --ondisk sda --fstype=ext4 --label platform --align 1024 --use-uuid
+
+part swap --ondisk sda --size 44 --label swap1 --fstype=swap --use-uuid
+
+bootloader --ptable gpt --timeout=5 --append="rootwait rootfstype=ext4 console=ttyS0,115200 console=tty0"

sources/poky/scripts/lib/wic/engine.py

@@ -0,0 +1,628 @@
+#
+# Copyright (c) 2013, Intel Corporation.
+#
+# SPDX-License-Identifier: GPL-2.0-only
+#
+# DESCRIPTION
+
+# This module implements the image creation engine used by 'wic' to
+# create images.  The engine parses through the OpenEmbedded kickstart
+# (wks) file specified and generates images that can then be directly
+# written onto media.
+#
+# AUTHORS
+# Tom Zanussi <tom.zanussi (at] linux.intel.com>
+#
+
+import logging
+import os
+import tempfile
+import json
+import subprocess
+import shutil
+import re
+
+from collections import namedtuple, OrderedDict
+
+from wic import WicError
+from wic.filemap import sparse_copy
+from wic.pluginbase import PluginMgr
+from wic.misc import get_bitbake_var, exec_cmd
+
+logger = logging.getLogger('wic')
+
+def verify_build_env():
+    """
+    Verify that the build environment is sane.
+
+    Returns True if it is, false otherwise
+    """
+    if not os.environ.get("BUILDDIR"):
+        raise WicError("BUILDDIR not found, exiting. (Did you forget to source oe-init-build-env?)")
+
+    return True
+
+
+CANNED_IMAGE_DIR = "lib/wic/canned-wks" # relative to scripts
+SCRIPTS_CANNED_IMAGE_DIR = "scripts/" + CANNED_IMAGE_DIR
+WIC_DIR = "wic"
+
+def build_canned_image_list(path):
+    layers_path = get_bitbake_var("BBLAYERS")
+    canned_wks_layer_dirs = []
+
+    if layers_path is not None:
+        for layer_path in layers_path.split():
+            for wks_path in (WIC_DIR, SCRIPTS_CANNED_IMAGE_DIR):
+                cpath = os.path.join(layer_path, wks_path)
+                if os.path.isdir(cpath):
+                    canned_wks_layer_dirs.append(cpath)
+
+    cpath = os.path.join(path, CANNED_IMAGE_DIR)
+    canned_wks_layer_dirs.append(cpath)
+
+    return canned_wks_layer_dirs
+
+def find_canned_image(scripts_path, wks_file):
+    """
+    Find a .wks file with the given name in the canned files dir.
+
+    Return False if not found
+    """
+    layers_canned_wks_dir = build_canned_image_list(scripts_path)
+
+    for canned_wks_dir in layers_canned_wks_dir:
+        for root, dirs, files in os.walk(canned_wks_dir):
+            for fname in files:
+                if fname.endswith("~") or fname.endswith("#"):
+                    continue
+                if ((fname.endswith(".wks") and wks_file + ".wks" == fname) or \
+                   (fname.endswith(".wks.in") and wks_file + ".wks.in" == fname)):
+                    fullpath = os.path.join(canned_wks_dir, fname)
+                    return fullpath
+    return None
+
+
+def list_canned_images(scripts_path):
+    """
+    List the .wks files in the canned image dir, minus the extension.
+    """
+    layers_canned_wks_dir = build_canned_image_list(scripts_path)
+
+    for canned_wks_dir in layers_canned_wks_dir:
+        for root, dirs, files in os.walk(canned_wks_dir):
+            for fname in files:
+                if fname.endswith("~") or fname.endswith("#"):
+                    continue
+                if fname.endswith(".wks") or fname.endswith(".wks.in"):
+                    fullpath = os.path.join(canned_wks_dir, fname)
+                    with open(fullpath) as wks:
+                        for line in wks:
+                            desc = ""
+                            idx = line.find("short-description:")
+                            if idx != -1:
+                                desc = line[idx + len("short-description:"):].strip()
+                                break
+                    basename = fname.split('.')[0]
+                    print("  %s\t\t%s" % (basename.ljust(30), desc))
+
+
+def list_canned_image_help(scripts_path, fullpath):
+    """
+    List the help and params in the specified canned image.
+    """
+    found = False
+    with open(fullpath) as wks:
+        for line in wks:
+            if not found:
+                idx = line.find("long-description:")
+                if idx != -1:
+                    print()
+                    print(line[idx + len("long-description:"):].strip())
+                    found = True
+                continue
+            if not line.strip():
+                break
+            idx = line.find("#")
+            if idx != -1:
+                print(line[idx + len("#:"):].rstrip())
+            else:
+                break
+
+
+def list_source_plugins():
+    """
+    List the available source plugins i.e. plugins available for --source.
+    """
+    plugins = PluginMgr.get_plugins('source')
+
+    for plugin in plugins:
+        print("  %s" % plugin)
+
+
+def wic_create(wks_file, rootfs_dir, bootimg_dir, kernel_dir,
+               native_sysroot, options):
+    """
+    Create image
+
+    wks_file - user-defined OE kickstart file
+    rootfs_dir - absolute path to the build's /rootfs dir
+    bootimg_dir - absolute path to the build's boot artifacts directory
+    kernel_dir - absolute path to the build's kernel directory
+    native_sysroot - absolute path to the build's native sysroots dir
+    image_output_dir - dirname to create for image
+    options - wic command line options (debug, bmap, etc)
+
+    Normally, the values for the build artifacts values are determined
+    by 'wic -e' from the output of the 'bitbake -e' command given an
+    image name e.g. 'core-image-minimal' and a given machine set in
+    local.conf.  If that's the case, the variables get the following
+    values from the output of 'bitbake -e':
+
+    rootfs_dir:        IMAGE_ROOTFS
+    kernel_dir:        DEPLOY_DIR_IMAGE
+    native_sysroot:    STAGING_DIR_NATIVE
+
+    In the above case, bootimg_dir remains unset and the
+    plugin-specific image creation code is responsible for finding the
+    bootimg artifacts.
+
+    In the case where the values are passed in explicitly i.e 'wic -e'
+    is not used but rather the individual 'wic' options are used to
+    explicitly specify these values.
+    """
+    try:
+        oe_builddir = os.environ["BUILDDIR"]
+    except KeyError:
+        raise WicError("BUILDDIR not found, exiting. (Did you forget to source oe-init-build-env?)")
+
+    if not os.path.exists(options.outdir):
+        os.makedirs(options.outdir)
+
+    pname = options.imager
+    plugin_class = PluginMgr.get_plugins('imager').get(pname)
+    if not plugin_class:
+        raise WicError('Unknown plugin: %s' % pname)
+
+    plugin = plugin_class(wks_file, rootfs_dir, bootimg_dir, kernel_dir,
+                          native_sysroot, oe_builddir, options)
+
+    plugin.do_create()
+
+    logger.info("The image(s) were created using OE kickstart file:\n  %s", wks_file)
+
+
+def wic_list(args, scripts_path):
+    """
+    Print the list of images or source plugins.
+    """
+    if args.list_type is None:
+        return False
+
+    if args.list_type == "images":
+
+        list_canned_images(scripts_path)
+        return True
+    elif args.list_type == "source-plugins":
+        list_source_plugins()
+        return True
+    elif len(args.help_for) == 1 and args.help_for[0] == 'help':
+        wks_file = args.list_type
+        fullpath = find_canned_image(scripts_path, wks_file)
+        if not fullpath:
+            raise WicError("No image named %s found, exiting. "
+                           "(Use 'wic list images' to list available images, "
+                           "or specify a fully-qualified OE kickstart (.wks) "
+                           "filename)" % wks_file)
+
+        list_canned_image_help(scripts_path, fullpath)
+        return True
+
+    return False
+
+
+class Disk:
+    def __init__(self, imagepath, native_sysroot, fstypes=('fat', 'ext')):
+        self.imagepath = imagepath
+        self.native_sysroot = native_sysroot
+        self.fstypes = fstypes
+        self._partitions = None
+        self._partimages = {}
+        self._lsector_size = None
+        self._psector_size = None
+        self._ptable_format = None
+
+        # find parted
+        # read paths from $PATH environment variable
+        # if it fails, use hardcoded paths
+        pathlist = "/bin:/usr/bin:/usr/sbin:/sbin/"
+        try:
+            self.paths = os.environ['PATH'] + ":" + pathlist
+        except KeyError:
+            self.paths = pathlist
+
+        if native_sysroot:
+            for path in pathlist.split(':'):
+                self.paths = "%s%s:%s" % (native_sysroot, path, self.paths)
+
+        self.parted = shutil.which("parted", path=self.paths)
+        if not self.parted:
+            raise WicError("Can't find executable parted")
+
+        self.partitions = self.get_partitions()
+
+    def __del__(self):
+        for path in self._partimages.values():
+            os.unlink(path)
+
+    def get_partitions(self):
+        if self._partitions is None:
+            self._partitions = OrderedDict()
+            out = exec_cmd("%s -sm %s unit B print" % (self.parted, self.imagepath))
+            parttype = namedtuple("Part", "pnum start end size fstype")
+            splitted = out.splitlines()
+            # skip over possible errors in exec_cmd output
+            try:
+                idx =splitted.index("BYT;")
+            except ValueError:
+                raise WicError("Error getting partition information from %s" % (self.parted))
+            lsector_size, psector_size, self._ptable_format = splitted[idx + 1].split(":")[3:6]
+            self._lsector_size = int(lsector_size)
+            self._psector_size = int(psector_size)
+            for line in splitted[idx + 2:]:
+                pnum, start, end, size, fstype = line.split(':')[:5]
+                partition = parttype(int(pnum), int(start[:-1]), int(end[:-1]),
+                                     int(size[:-1]), fstype)
+                self._partitions[pnum] = partition
+
+        return self._partitions
+
+    def __getattr__(self, name):
+        """Get path to the executable in a lazy way."""
+        if name in ("mdir", "mcopy", "mdel", "mdeltree", "sfdisk", "e2fsck",
+                    "resize2fs", "mkswap", "mkdosfs", "debugfs","blkid"):
+            aname = "_%s" % name
+            if aname not in self.__dict__:
+                setattr(self, aname, shutil.which(name, path=self.paths))
+                if aname not in self.__dict__ or self.__dict__[aname] is None:
+                    raise WicError("Can't find executable '{}'".format(name))
+            return self.__dict__[aname]
+        return self.__dict__[name]
+
+    def _get_part_image(self, pnum):
+        if pnum not in self.partitions:
+            raise WicError("Partition %s is not in the image" % pnum)
+        part = self.partitions[pnum]
+        # check if fstype is supported
+        for fstype in self.fstypes:
+            if part.fstype.startswith(fstype):
+                break
+        else:
+            raise WicError("Not supported fstype: {}".format(part.fstype))
+        if pnum not in self._partimages:
+            tmpf = tempfile.NamedTemporaryFile(prefix="wic-part")
+            dst_fname = tmpf.name
+            tmpf.close()
+            sparse_copy(self.imagepath, dst_fname, skip=part.start, length=part.size)
+            self._partimages[pnum] = dst_fname
+
+        return self._partimages[pnum]
+
+    def _put_part_image(self, pnum):
+        """Put partition image into partitioned image."""
+        sparse_copy(self._partimages[pnum], self.imagepath,
+                    seek=self.partitions[pnum].start)
+
+    def dir(self, pnum, path):
+        if pnum not in self.partitions:
+            raise WicError("Partition %s is not in the image" % pnum)
+
+        if self.partitions[pnum].fstype.startswith('ext'):
+            return exec_cmd("{} {} -R 'ls -l {}'".format(self.debugfs,
+                                                         self._get_part_image(pnum),
+                                                         path), as_shell=True)
+        else: # fat
+            return exec_cmd("{} -i {} ::{}".format(self.mdir,
+                                                   self._get_part_image(pnum),
+                                                   path))
+
+    def copy(self, src, dest):
+        """Copy partition image into wic image."""
+        pnum =  dest.part if isinstance(src, str) else src.part
+
+        if self.partitions[pnum].fstype.startswith('ext'):
+            if isinstance(src, str):
+                cmd = "printf 'cd {}\nwrite {} {}\n' | {} -w {}".\
+                      format(os.path.dirname(dest.path), src, os.path.basename(src),
+                             self.debugfs, self._get_part_image(pnum))
+            else: # copy from wic
+                # run both dump and rdump to support both files and directory
+                cmd = "printf 'cd {}\ndump /{} {}\nrdump /{} {}\n' | {} {}".\
+                      format(os.path.dirname(src.path), src.path,
+                             dest, src.path, dest, self.debugfs,
+                             self._get_part_image(pnum))
+        else: # fat
+            if isinstance(src, str):
+                cmd = "{} -i {} -snop {} ::{}".format(self.mcopy,
+                                                  self._get_part_image(pnum),
+                                                  src, dest.path)
+            else:
+                cmd = "{} -i {} -snop ::{} {}".format(self.mcopy,
+                                                  self._get_part_image(pnum),
+                                                  src.path, dest)
+
+        exec_cmd(cmd, as_shell=True)
+        self._put_part_image(pnum)
+
+    def remove_ext(self, pnum, path, recursive):
+        """
+        Remove files/dirs and their contents from the partition.
+        This only applies to ext* partition.
+        """
+        abs_path = re.sub(r'\/\/+', '/', path)
+        cmd = "{} {} -wR 'rm \"{}\"'".format(self.debugfs,
+                                            self._get_part_image(pnum),
+                                            abs_path)
+        out = exec_cmd(cmd , as_shell=True)
+        for line in out.splitlines():
+            if line.startswith("rm:"):
+                if "file is a directory" in line:
+                    if recursive:
+                        # loop through content and delete them one by one if
+                        # flaged with -r
+                        subdirs = iter(self.dir(pnum, abs_path).splitlines())
+                        next(subdirs)
+                        for subdir in subdirs:
+                            dir = subdir.split(':')[1].split(" ", 1)[1]
+                            if not dir == "." and not dir == "..":
+                                self.remove_ext(pnum, "%s/%s" % (abs_path, dir), recursive)
+
+                    rmdir_out = exec_cmd("{} {} -wR 'rmdir \"{}\"'".format(self.debugfs,
+                                                    self._get_part_image(pnum),
+                                                    abs_path.rstrip('/'))
+                                                    , as_shell=True)
+
+                    for rmdir_line in rmdir_out.splitlines():
+                        if "directory not empty" in rmdir_line:
+                            raise WicError("Could not complete operation: \n%s \n"
+                                            "use -r to remove non-empty directory" % rmdir_line)
+                        if rmdir_line.startswith("rmdir:"):
+                            raise WicError("Could not complete operation: \n%s "
+                                            "\n%s" % (str(line), rmdir_line))
+
+                else:
+                    raise WicError("Could not complete operation: \n%s "
+                                    "\nUnable to remove %s" % (str(line), abs_path))
+
+    def remove(self, pnum, path, recursive):
+        """Remove files/dirs from the partition."""
+        partimg = self._get_part_image(pnum)
+        if self.partitions[pnum].fstype.startswith('ext'):
+            self.remove_ext(pnum, path, recursive)
+
+        else: # fat
+            cmd = "{} -i {} ::{}".format(self.mdel, partimg, path)
+            try:
+                exec_cmd(cmd)
+            except WicError as err:
+                if "not found" in str(err) or "non empty" in str(err):
+                    # mdel outputs 'File ... not found' or 'directory .. non empty"
+                    # try to use mdeltree as path could be a directory
+                    cmd = "{} -i {} ::{}".format(self.mdeltree,
+                                                 partimg, path)
+                    exec_cmd(cmd)
+                else:
+                    raise err
+        self._put_part_image(pnum)
+
+    def write(self, target, expand):
+        """Write disk image to the media or file."""
+        def write_sfdisk_script(outf, parts):
+            for key, val in parts['partitiontable'].items():
+                if key in ("partitions", "device", "firstlba", "lastlba"):
+                    continue
+                if key == "id":
+                    key = "label-id"
+                outf.write("{}: {}\n".format(key, val))
+            outf.write("\n")
+            for part in parts['partitiontable']['partitions']:
+                line = ''
+                for name in ('attrs', 'name', 'size', 'type', 'uuid'):
+                    if name == 'size' and part['type'] == 'f':
+                        # don't write size for extended partition
+                        continue
+                    val = part.get(name)
+                    if val:
+                        line += '{}={}, '.format(name, val)
+                if line:
+                    line = line[:-2] # strip ', '
+                if part.get('bootable'):
+                    line += ' ,bootable'
+                outf.write("{}\n".format(line))
+            outf.flush()
+
+        def read_ptable(path):
+            out = exec_cmd("{} -J {}".format(self.sfdisk, path))
+            return json.loads(out)
+
+        def write_ptable(parts, target):
+            with tempfile.NamedTemporaryFile(prefix="wic-sfdisk-", mode='w') as outf:
+                write_sfdisk_script(outf, parts)
+                cmd = "{} --no-reread {} < {} ".format(self.sfdisk, target, outf.name)
+                exec_cmd(cmd, as_shell=True)
+
+        if expand is None:
+            sparse_copy(self.imagepath, target)
+        else:
+            # copy first sectors that may contain bootloader
+            sparse_copy(self.imagepath, target, length=2048 * self._lsector_size)
+
+            # copy source partition table to the target
+            parts = read_ptable(self.imagepath)
+            write_ptable(parts, target)
+
+            # get size of unpartitioned space
+            free = None
+            for line in exec_cmd("{} -F {}".format(self.sfdisk, target)).splitlines():
+                if line.startswith("Unpartitioned space ") and line.endswith("sectors"):
+                    free = int(line.split()[-2])
+                    # Align free space to a 2048 sector boundary. YOCTO #12840.
+                    free = free - (free % 2048)
+            if free is None:
+                raise WicError("Can't get size of unpartitioned space")
+
+            # calculate expanded partitions sizes
+            sizes = {}
+            num_auto_resize = 0
+            for num, part in enumerate(parts['partitiontable']['partitions'], 1):
+                if num in expand:
+                    if expand[num] != 0: # don't resize partition if size is set to 0
+                        sectors = expand[num] // self._lsector_size
+                        free -= sectors - part['size']
+                        part['size'] = sectors
+                        sizes[num] = sectors
+                elif part['type'] != 'f':
+                    sizes[num] = -1
+                    num_auto_resize += 1
+
+            for num, part in enumerate(parts['partitiontable']['partitions'], 1):
+                if sizes.get(num) == -1:
+                    part['size'] += free // num_auto_resize
+
+            # write resized partition table to the target
+            write_ptable(parts, target)
+
+            # read resized partition table
+            parts = read_ptable(target)
+
+            # copy partitions content
+            for num, part in enumerate(parts['partitiontable']['partitions'], 1):
+                pnum = str(num)
+                fstype = self.partitions[pnum].fstype
+
+                # copy unchanged partition
+                if part['size'] == self.partitions[pnum].size // self._lsector_size:
+                    logger.info("copying unchanged partition {}".format(pnum))
+                    sparse_copy(self._get_part_image(pnum), target, seek=part['start'] * self._lsector_size)
+                    continue
+
+                # resize or re-create partitions
+                if fstype.startswith('ext') or fstype.startswith('fat') or \
+                   fstype.startswith('linux-swap'):
+
+                    partfname = None
+                    with tempfile.NamedTemporaryFile(prefix="wic-part{}-".format(pnum)) as partf:
+                        partfname = partf.name
+
+                    if fstype.startswith('ext'):
+                        logger.info("resizing ext partition {}".format(pnum))
+                        partimg = self._get_part_image(pnum)
+                        sparse_copy(partimg, partfname)
+                        exec_cmd("{} -pf {}".format(self.e2fsck, partfname))
+                        exec_cmd("{} {} {}s".format(\
+                                 self.resize2fs, partfname, part['size']))
+                    elif fstype.startswith('fat'):
+                        logger.info("copying content of the fat partition {}".format(pnum))
+                        with tempfile.TemporaryDirectory(prefix='wic-fatdir-') as tmpdir:
+                            # copy content to the temporary directory
+                            cmd = "{} -snompi {} :: {}".format(self.mcopy,
+                                                               self._get_part_image(pnum),
+                                                               tmpdir)
+                            exec_cmd(cmd)
+                            # create new msdos partition
+                            label = part.get("name")
+                            label_str = "-n {}".format(label) if label else ''
+
+                            cmd = "{} {} -C {} {}".format(self.mkdosfs, label_str, partfname,
+                                                          part['size'])
+                            exec_cmd(cmd)
+                            # copy content from the temporary directory to the new partition
+                            cmd = "{} -snompi {} {}/* ::".format(self.mcopy, partfname, tmpdir)
+                            exec_cmd(cmd, as_shell=True)
+                    elif fstype.startswith('linux-swap'):
+                        logger.info("creating swap partition {}".format(pnum))
+                        label = part.get("name")
+                        label_str = "-L {}".format(label) if label else ''
+                        out = exec_cmd("{} --probe {}".format(self.blkid, self._get_part_image(pnum)))
+                        uuid = out[out.index("UUID=\"")+6:out.index("UUID=\"")+42]
+                        uuid_str = "-U {}".format(uuid) if uuid else ''
+                        with open(partfname, 'w') as sparse:
+                            os.ftruncate(sparse.fileno(), part['size'] * self._lsector_size)
+                        exec_cmd("{} {} {} {}".format(self.mkswap, label_str, uuid_str, partfname))
+                    sparse_copy(partfname, target, seek=part['start'] * self._lsector_size)
+                    os.unlink(partfname)
+                elif part['type'] != 'f':
+                    logger.warning("skipping partition {}: unsupported fstype {}".format(pnum, fstype))
+
+def wic_ls(args, native_sysroot):
+    """List contents of partitioned image or vfat partition."""
+    disk = Disk(args.path.image, native_sysroot)
+    if not args.path.part:
+        if disk.partitions:
+            print('Num     Start        End          Size      Fstype')
+            for part in disk.partitions.values():
+                print("{:2d}  {:12d} {:12d} {:12d}  {}".format(\
+                          part.pnum, part.start, part.end,
+                          part.size, part.fstype))
+    else:
+        path = args.path.path or '/'
+        print(disk.dir(args.path.part, path))
+
+def wic_cp(args, native_sysroot):
+    """
+    Copy file or directory to/from the vfat/ext partition of
+    partitioned image.
+    """
+    if isinstance(args.dest, str):
+        disk = Disk(args.src.image, native_sysroot)
+    else:
+        disk = Disk(args.dest.image, native_sysroot)
+    disk.copy(args.src, args.dest)
+
+
+def wic_rm(args, native_sysroot):
+    """
+    Remove files or directories from the vfat partition of
+    partitioned image.
+    """
+    disk = Disk(args.path.image, native_sysroot)
+    disk.remove(args.path.part, args.path.path, args.recursive_delete)
+
+def wic_write(args, native_sysroot):
+    """
+    Write image to a target device.
+    """
+    disk = Disk(args.image, native_sysroot, ('fat', 'ext', 'linux-swap'))
+    disk.write(args.target, args.expand)
+
+def find_canned(scripts_path, file_name):
+    """
+    Find a file either by its path or by name in the canned files dir.
+
+    Return None if not found
+    """
+    if os.path.exists(file_name):
+        return file_name
+
+    layers_canned_wks_dir = build_canned_image_list(scripts_path)
+    for canned_wks_dir in layers_canned_wks_dir:
+        for root, dirs, files in os.walk(canned_wks_dir):
+            for fname in files:
+                if fname == file_name:
+                    fullpath = os.path.join(canned_wks_dir, fname)
+                    return fullpath
+
+def get_custom_config(boot_file):
+    """
+    Get the custom configuration to be used for the bootloader.
+
+    Return None if the file can't be found.
+    """
+    # Get the scripts path of poky
+    scripts_path = os.path.abspath("%s/../.." % os.path.dirname(__file__))
+
+    cfg_file = find_canned(scripts_path, boot_file)
+    if cfg_file:
+        with open(cfg_file, "r") as f:
+            config = f.read()
+        return config

sources/poky/scripts/lib/wic/filemap.py

@@ -0,0 +1,583 @@
+#
+# Copyright (c) 2012 Intel, Inc.
+#
+# SPDX-License-Identifier: GPL-2.0-only
+#
+
+"""
+This module implements python implements a way to get file block. Two methods
+are supported - the FIEMAP ioctl and the 'SEEK_HOLE / SEEK_DATA' features of
+the file seek syscall. The former is implemented by the 'FilemapFiemap' class,
+the latter is implemented by the 'FilemapSeek' class. Both classes provide the
+same API. The 'filemap' function automatically selects which class can be used
+and returns an instance of the class.
+"""
+
+# Disable the following pylint recommendations:
+#   * Too many instance attributes (R0902)
+# pylint: disable=R0902
+
+import errno
+import os
+import struct
+import array
+import fcntl
+import tempfile
+import logging
+
+def get_block_size(file_obj):
+    """
+    Returns block size for file object 'file_obj'. Errors are indicated by the
+    'IOError' exception.
+    """
+    # Get the block size of the host file-system for the image file by calling
+    # the FIGETBSZ ioctl (number 2).
+    try:
+        binary_data = fcntl.ioctl(file_obj, 2, struct.pack('I', 0))
+        bsize = struct.unpack('I', binary_data)[0]
+    except OSError:
+        bsize = None
+
+    # If ioctl causes OSError or give bsize to zero failback to os.fstat
+    if not bsize:
+        import os
+        stat = os.fstat(file_obj.fileno())
+        if hasattr(stat, 'st_blksize'):
+            bsize = stat.st_blksize
+        else:
+            raise IOError("Unable to determine block size")
+
+    # The logic in this script only supports a maximum of a 4KB
+    # block size
+    max_block_size = 4 * 1024
+    if bsize > max_block_size:
+        bsize = max_block_size
+
+    return bsize
+
+class ErrorNotSupp(Exception):
+    """
+    An exception of this type is raised when the 'FIEMAP' or 'SEEK_HOLE' feature
+    is not supported either by the kernel or the file-system.
+    """
+    pass
+
+class Error(Exception):
+    """A class for all the other exceptions raised by this module."""
+    pass
+
+
+class _FilemapBase(object):
+    """
+    This is a base class for a couple of other classes in this module. This
+    class simply performs the common parts of the initialization process: opens
+    the image file, gets its size, etc. The 'log' parameter is the logger object
+    to use for printing messages.
+    """
+
+    def __init__(self, image, log=None):
+        """
+        Initialize a class instance. The 'image' argument is full path to the
+        file or file object to operate on.
+        """
+
+        self._log = log
+        if self._log is None:
+            self._log = logging.getLogger(__name__)
+
+        self._f_image_needs_close = False
+
+        if hasattr(image, "fileno"):
+            self._f_image = image
+            self._image_path = image.name
+        else:
+            self._image_path = image
+            self._open_image_file()
+
+        try:
+            self.image_size = os.fstat(self._f_image.fileno()).st_size
+        except IOError as err:
+            raise Error("cannot get information about file '%s': %s"
+                        % (self._f_image.name, err))
+
+        try:
+            self.block_size = get_block_size(self._f_image)
+        except IOError as err:
+            raise Error("cannot get block size for '%s': %s"
+                        % (self._image_path, err))
+
+        self.blocks_cnt = self.image_size + self.block_size - 1
+        self.blocks_cnt //= self.block_size
+
+        try:
+            self._f_image.flush()
+        except IOError as err:
+            raise Error("cannot flush image file '%s': %s"
+                        % (self._image_path, err))
+
+        try:
+            os.fsync(self._f_image.fileno()),
+        except OSError as err:
+            raise Error("cannot synchronize image file '%s': %s "
+                        % (self._image_path, err.strerror))
+
+        self._log.debug("opened image \"%s\"" % self._image_path)
+        self._log.debug("block size %d, blocks count %d, image size %d"
+                        % (self.block_size, self.blocks_cnt, self.image_size))
+
+    def __del__(self):
+        """The class destructor which just closes the image file."""
+        if self._f_image_needs_close:
+            self._f_image.close()
+
+    def _open_image_file(self):
+        """Open the image file."""
+        try:
+            self._f_image = open(self._image_path, 'rb')
+        except IOError as err:
+            raise Error("cannot open image file '%s': %s"
+                        % (self._image_path, err))
+
+        self._f_image_needs_close = True
+
+    def block_is_mapped(self, block): # pylint: disable=W0613,R0201
+        """
+        This method has has to be implemented by child classes. It returns
+        'True' if block number 'block' of the image file is mapped and 'False'
+        otherwise.
+        """
+
+        raise Error("the method is not implemented")
+
+    def get_mapped_ranges(self, start, count): # pylint: disable=W0613,R0201
+        """
+        This method has has to be implemented by child classes. This is a
+        generator which yields ranges of mapped blocks in the file. The ranges
+        are tuples of 2 elements: [first, last], where 'first' is the first
+        mapped block and 'last' is the last mapped block.
+
+        The ranges are yielded for the area of the file of size 'count' blocks,
+        starting from block 'start'.
+        """
+
+        raise Error("the method is not implemented")
+
+
+# The 'SEEK_HOLE' and 'SEEK_DATA' options of the file seek system call
+_SEEK_DATA = 3
+_SEEK_HOLE = 4
+
+def _lseek(file_obj, offset, whence):
+    """This is a helper function which invokes 'os.lseek' for file object
+    'file_obj' and with specified 'offset' and 'whence'. The 'whence'
+    argument is supposed to be either '_SEEK_DATA' or '_SEEK_HOLE'. When
+    there is no more data or hole starting from 'offset', this function
+    returns '-1'.  Otherwise the data or hole position is returned."""
+
+    try:
+        return os.lseek(file_obj.fileno(), offset, whence)
+    except OSError as err:
+        # The 'lseek' system call returns the ENXIO if there is no data or
+        # hole starting from the specified offset.
+        if err.errno == errno.ENXIO:
+            return -1
+        elif err.errno == errno.EINVAL:
+            raise ErrorNotSupp("the kernel or file-system does not support "
+                               "\"SEEK_HOLE\" and \"SEEK_DATA\"")
+        else:
+            raise
+
+class FilemapSeek(_FilemapBase):
+    """
+    This class uses the 'SEEK_HOLE' and 'SEEK_DATA' to find file block mapping.
+    Unfortunately, the current implementation requires the caller to have write
+    access to the image file.
+    """
+
+    def __init__(self, image, log=None):
+        """Refer the '_FilemapBase' class for the documentation."""
+
+        # Call the base class constructor first
+        _FilemapBase.__init__(self, image, log)
+        self._log.debug("FilemapSeek: initializing")
+
+        self._probe_seek_hole()
+
+    def _probe_seek_hole(self):
+        """
+        Check whether the system implements 'SEEK_HOLE' and 'SEEK_DATA'.
+        Unfortunately, there seems to be no clean way for detecting this,
+        because often the system just fakes them by just assuming that all
+        files are fully mapped, so 'SEEK_HOLE' always returns EOF and
+        'SEEK_DATA' always returns the requested offset.
+
+        I could not invent a better way of detecting the fake 'SEEK_HOLE'
+        implementation than just to create a temporary file in the same
+        directory where the image file resides. It would be nice to change this
+        to something better.
+        """
+
+        directory = os.path.dirname(self._image_path)
+
+        try:
+            tmp_obj = tempfile.TemporaryFile("w+", dir=directory)
+        except IOError as err:
+            raise ErrorNotSupp("cannot create a temporary in \"%s\": %s" \
+                              % (directory, err))
+
+        try:
+            os.ftruncate(tmp_obj.fileno(), self.block_size)
+        except OSError as err:
+            raise ErrorNotSupp("cannot truncate temporary file in \"%s\": %s"
+                               % (directory, err))
+
+        offs = _lseek(tmp_obj, 0, _SEEK_HOLE)
+        if offs != 0:
+            # We are dealing with the stub 'SEEK_HOLE' implementation which
+            # always returns EOF.
+            self._log.debug("lseek(0, SEEK_HOLE) returned %d" % offs)
+            raise ErrorNotSupp("the file-system does not support "
+                               "\"SEEK_HOLE\" and \"SEEK_DATA\" but only "
+                               "provides a stub implementation")
+
+        tmp_obj.close()
+
+    def block_is_mapped(self, block):
+        """Refer the '_FilemapBase' class for the documentation."""
+        offs = _lseek(self._f_image, block * self.block_size, _SEEK_DATA)
+        if offs == -1:
+            result = False
+        else:
+            result = (offs // self.block_size == block)
+
+        self._log.debug("FilemapSeek: block_is_mapped(%d) returns %s"
+                        % (block, result))
+        return result
+
+    def _get_ranges(self, start, count, whence1, whence2):
+        """
+        This function implements 'get_mapped_ranges()' depending
+        on what is passed in the 'whence1' and 'whence2' arguments.
+        """
+
+        assert whence1 != whence2
+        end = start * self.block_size
+        limit = end + count * self.block_size
+
+        while True:
+            start = _lseek(self._f_image, end, whence1)
+            if start == -1 or start >= limit or start == self.image_size:
+                break
+
+            end = _lseek(self._f_image, start, whence2)
+            if end == -1 or end == self.image_size:
+                end = self.blocks_cnt * self.block_size
+            if end > limit:
+                end = limit
+
+            start_blk = start // self.block_size
+            end_blk = end // self.block_size - 1
+            self._log.debug("FilemapSeek: yielding range (%d, %d)"
+                            % (start_blk, end_blk))
+            yield (start_blk, end_blk)
+
+    def get_mapped_ranges(self, start, count):
+        """Refer the '_FilemapBase' class for the documentation."""
+        self._log.debug("FilemapSeek: get_mapped_ranges(%d,  %d(%d))"
+                        % (start, count, start + count - 1))
+        return self._get_ranges(start, count, _SEEK_DATA, _SEEK_HOLE)
+
+
+# Below goes the FIEMAP ioctl implementation, which is not very readable
+# because it deals with the rather complex FIEMAP ioctl. To understand the
+# code, you need to know the FIEMAP interface, which is documented in the
+# "Documentation/filesystems/fiemap.txt" file in the Linux kernel sources.
+
+# Format string for 'struct fiemap'
+_FIEMAP_FORMAT = "=QQLLLL"
+# sizeof(struct fiemap)
+_FIEMAP_SIZE = struct.calcsize(_FIEMAP_FORMAT)
+# Format string for 'struct fiemap_extent'
+_FIEMAP_EXTENT_FORMAT = "=QQQQQLLLL"
+# sizeof(struct fiemap_extent)
+_FIEMAP_EXTENT_SIZE = struct.calcsize(_FIEMAP_EXTENT_FORMAT)
+# The FIEMAP ioctl number
+_FIEMAP_IOCTL = 0xC020660B
+# This FIEMAP ioctl flag which instructs the kernel to sync the file before
+# reading the block map
+_FIEMAP_FLAG_SYNC = 0x00000001
+# Size of the buffer for 'struct fiemap_extent' elements which will be used
+# when invoking the FIEMAP ioctl. The larger is the buffer, the less times the
+# FIEMAP ioctl will be invoked.
+_FIEMAP_BUFFER_SIZE = 256 * 1024
+
+class FilemapFiemap(_FilemapBase):
+    """
+    This class provides API to the FIEMAP ioctl. Namely, it allows to iterate
+    over all mapped blocks and over all holes.
+
+    This class synchronizes the image file every time it invokes the FIEMAP
+    ioctl in order to work-around early FIEMAP implementation kernel bugs.
+    """
+
+    def __init__(self, image, log=None):
+        """
+        Initialize a class instance. The 'image' argument is full the file
+        object to operate on.
+        """
+
+        # Call the base class constructor first
+        _FilemapBase.__init__(self, image, log)
+        self._log.debug("FilemapFiemap: initializing")
+
+        self._buf_size = _FIEMAP_BUFFER_SIZE
+
+        # Calculate how many 'struct fiemap_extent' elements fit the buffer
+        self._buf_size -= _FIEMAP_SIZE
+        self._fiemap_extent_cnt = self._buf_size // _FIEMAP_EXTENT_SIZE
+        assert self._fiemap_extent_cnt > 0
+        self._buf_size = self._fiemap_extent_cnt * _FIEMAP_EXTENT_SIZE
+        self._buf_size += _FIEMAP_SIZE
+
+        # Allocate a mutable buffer for the FIEMAP ioctl
+        self._buf = array.array('B', [0] * self._buf_size)
+
+        # Check if the FIEMAP ioctl is supported
+        self.block_is_mapped(0)
+
+    def _invoke_fiemap(self, block, count):
+        """
+        Invoke the FIEMAP ioctl for 'count' blocks of the file starting from
+        block number 'block'.
+
+        The full result of the operation is stored in 'self._buf' on exit.
+        Returns the unpacked 'struct fiemap' data structure in form of a python
+        list (just like 'struct.upack()').
+        """
+
+        if self.blocks_cnt != 0 and (block < 0 or block >= self.blocks_cnt):
+            raise Error("bad block number %d, should be within [0, %d]"
+                        % (block, self.blocks_cnt))
+
+        # Initialize the 'struct fiemap' part of the buffer. We use the
+        # '_FIEMAP_FLAG_SYNC' flag in order to make sure the file is
+        # synchronized. The reason for this is that early FIEMAP
+        # implementations had many bugs related to cached dirty data, and
+        # synchronizing the file is a necessary work-around.
+        struct.pack_into(_FIEMAP_FORMAT, self._buf, 0, block * self.block_size,
+                         count * self.block_size, _FIEMAP_FLAG_SYNC, 0,
+                         self._fiemap_extent_cnt, 0)
+
+        try:
+            fcntl.ioctl(self._f_image, _FIEMAP_IOCTL, self._buf, 1)
+        except IOError as err:
+            # Note, the FIEMAP ioctl is supported by the Linux kernel starting
+            # from version 2.6.28 (year 2008).
+            if err.errno == errno.EOPNOTSUPP:
+                errstr = "FilemapFiemap: the FIEMAP ioctl is not supported " \
+                         "by the file-system"
+                self._log.debug(errstr)
+                raise ErrorNotSupp(errstr)
+            if err.errno == errno.ENOTTY:
+                errstr = "FilemapFiemap: the FIEMAP ioctl is not supported " \
+                         "by the kernel"
+                self._log.debug(errstr)
+                raise ErrorNotSupp(errstr)
+            raise Error("the FIEMAP ioctl failed for '%s': %s"
+                        % (self._image_path, err))
+
+        return struct.unpack(_FIEMAP_FORMAT, self._buf[:_FIEMAP_SIZE])
+
+    def block_is_mapped(self, block):
+        """Refer the '_FilemapBase' class for the documentation."""
+        struct_fiemap = self._invoke_fiemap(block, 1)
+
+        # The 3rd element of 'struct_fiemap' is the 'fm_mapped_extents' field.
+        # If it contains zero, the block is not mapped, otherwise it is
+        # mapped.
+        result = bool(struct_fiemap[3])
+        self._log.debug("FilemapFiemap: block_is_mapped(%d) returns %s"
+                        % (block, result))
+        return result
+
+    def _unpack_fiemap_extent(self, index):
+        """
+        Unpack a 'struct fiemap_extent' structure object number 'index' from
+        the internal 'self._buf' buffer.
+        """
+
+        offset = _FIEMAP_SIZE + _FIEMAP_EXTENT_SIZE * index
+        return struct.unpack(_FIEMAP_EXTENT_FORMAT,
+                             self._buf[offset : offset + _FIEMAP_EXTENT_SIZE])
+
+    def _do_get_mapped_ranges(self, start, count):
+        """
+        Implements most the functionality for the  'get_mapped_ranges()'
+        generator: invokes the FIEMAP ioctl, walks through the mapped extents
+        and yields mapped block ranges. However, the ranges may be consecutive
+        (e.g., (1, 100), (100, 200)) and 'get_mapped_ranges()' simply merges
+        them.
+        """
+
+        block = start
+        while block < start + count:
+            struct_fiemap = self._invoke_fiemap(block, count)
+
+            mapped_extents = struct_fiemap[3]
+            if mapped_extents == 0:
+                # No more mapped blocks
+                return
+
+            extent = 0
+            while extent < mapped_extents:
+                fiemap_extent = self._unpack_fiemap_extent(extent)
+
+                # Start of the extent
+                extent_start = fiemap_extent[0]
+                # Starting block number of the extent
+                extent_block = extent_start // self.block_size
+                # Length of the extent
+                extent_len = fiemap_extent[2]
+                # Count of blocks in the extent
+                extent_count = extent_len // self.block_size
+
+                # Extent length and offset have to be block-aligned
+                assert extent_start % self.block_size == 0
+                assert extent_len % self.block_size == 0
+
+                if extent_block > start + count - 1:
+                    return
+
+                first = max(extent_block, block)
+                last = min(extent_block + extent_count, start + count) - 1
+                yield (first, last)
+
+                extent += 1
+
+            block = extent_block + extent_count
+
+    def get_mapped_ranges(self, start, count):
+        """Refer the '_FilemapBase' class for the documentation."""
+        self._log.debug("FilemapFiemap: get_mapped_ranges(%d,  %d(%d))"
+                        % (start, count, start + count - 1))
+        iterator = self._do_get_mapped_ranges(start, count)
+        first_prev, last_prev = next(iterator)
+
+        for first, last in iterator:
+            if last_prev == first - 1:
+                last_prev = last
+            else:
+                self._log.debug("FilemapFiemap: yielding range (%d, %d)"
+                                % (first_prev, last_prev))
+                yield (first_prev, last_prev)
+                first_prev, last_prev = first, last
+
+        self._log.debug("FilemapFiemap: yielding range (%d, %d)"
+                        % (first_prev, last_prev))
+        yield (first_prev, last_prev)
+
+class FilemapNobmap(_FilemapBase):
+    """
+    This class is used when both the 'SEEK_DATA/HOLE' and FIEMAP are not
+    supported by the filesystem or kernel.
+    """
+
+    def __init__(self, image, log=None):
+        """Refer the '_FilemapBase' class for the documentation."""
+
+        # Call the base class constructor first
+        _FilemapBase.__init__(self, image, log)
+        self._log.debug("FilemapNobmap: initializing")
+
+    def block_is_mapped(self, block):
+        """Refer the '_FilemapBase' class for the documentation."""
+        return True
+
+    def get_mapped_ranges(self, start, count):
+        """Refer the '_FilemapBase' class for the documentation."""
+        self._log.debug("FilemapNobmap: get_mapped_ranges(%d,  %d(%d))"
+                        % (start, count, start + count - 1))
+        yield (start, start + count -1)
+
+def filemap(image, log=None):
+    """
+    Create and return an instance of a Filemap class - 'FilemapFiemap' or
+    'FilemapSeek', depending on what the system we run on supports. If the
+    FIEMAP ioctl is supported, an instance of the 'FilemapFiemap' class is
+    returned. Otherwise, if 'SEEK_HOLE' is supported an instance of the
+    'FilemapSeek' class is returned. If none of these are supported, the
+    function generates an 'Error' type exception.
+    """
+
+    try:
+        return FilemapFiemap(image, log)
+    except ErrorNotSupp:
+        try:
+            return FilemapSeek(image, log)
+        except ErrorNotSupp:
+            return FilemapNobmap(image, log)
+
+def sparse_copy(src_fname, dst_fname, skip=0, seek=0,
+                length=0, api=None):
+    """
+    Efficiently copy sparse file to or into another file.
+
+    src_fname: path to source file
+    dst_fname: path to destination file
+    skip: skip N bytes at thestart of src
+    seek: seek N bytes from the start of dst
+    length: read N bytes from src and write them to dst
+    api: FilemapFiemap or FilemapSeek object
+    """
+    if not api:
+        api = filemap
+    fmap = api(src_fname)
+    try:
+        dst_file = open(dst_fname, 'r+b')
+    except IOError:
+        dst_file = open(dst_fname, 'wb')
+        if length:
+            dst_size = length + seek
+        else:
+            dst_size = os.path.getsize(src_fname) + seek - skip
+        dst_file.truncate(dst_size)
+
+    written = 0
+    for first, last in fmap.get_mapped_ranges(0, fmap.blocks_cnt):
+        start = first * fmap.block_size
+        end = (last + 1) * fmap.block_size
+
+        if skip >= end:
+            continue
+
+        if start < skip < end:
+            start = skip
+
+        fmap._f_image.seek(start, os.SEEK_SET)
+
+        written += start - skip - written
+        if length and written >= length:
+            dst_file.seek(seek + length, os.SEEK_SET)
+            dst_file.close()
+            return
+
+        dst_file.seek(seek + start - skip, os.SEEK_SET)
+
+        chunk_size = 1024 * 1024
+        to_read = end - start
+        read = 0
+
+        while read < to_read:
+            if read + chunk_size > to_read:
+                chunk_size = to_read - read
+            size = chunk_size
+            if length and written + size > length:
+                size = length - written
+            chunk = fmap._f_image.read(size)
+            dst_file.write(chunk)
+            read += size
+            written += size
+            if written == length:
+                dst_file.close()
+                return
+    dst_file.close()

sources/poky/scripts/lib/wic/ksparser.py

@@ -0,0 +1,298 @@
+#!/usr/bin/env python3
+#
+# Copyright (c) 2016 Intel, Inc.
+#
+# SPDX-License-Identifier: GPL-2.0-only
+#
+# DESCRIPTION
+# This module provides parser for kickstart format
+#
+# AUTHORS
+# Ed Bartosh <ed.bartosh> (at] linux.intel.com>
+
+"""Kickstart parser module."""
+
+import os
+import shlex
+import logging
+import re
+
+from argparse import ArgumentParser, ArgumentError, ArgumentTypeError
+
+from wic.engine import find_canned
+from wic.partition import Partition
+from wic.misc import get_bitbake_var
+
+logger = logging.getLogger('wic')
+
+__expand_var_regexp__ = re.compile(r"\${[^{}@\n\t :]+}")
+
+def expand_line(line):
+    while True:
+        m = __expand_var_regexp__.search(line)
+        if not m:
+            return line
+        key = m.group()[2:-1]
+        val = get_bitbake_var(key)
+        if val is None:
+            logger.warning("cannot expand variable %s" % key)
+            return line
+        line = line[:m.start()] + val + line[m.end():]
+
+class KickStartError(Exception):
+    """Custom exception."""
+    pass
+
+class KickStartParser(ArgumentParser):
+    """
+    This class overwrites error method to throw exception
+    instead of producing usage message(default argparse behavior).
+    """
+    def error(self, message):
+        raise ArgumentError(None, message)
+
+def sizetype(default, size_in_bytes=False):
+    def f(arg):
+        """
+        Custom type for ArgumentParser
+        Converts size string in <num>[S|s|K|k|M|G] format into the integer value
+        """
+        try:
+            suffix = default
+            size = int(arg)
+        except ValueError:
+            try:
+                suffix = arg[-1:]
+                size = int(arg[:-1])
+            except ValueError:
+                raise ArgumentTypeError("Invalid size: %r" % arg)
+
+
+        if size_in_bytes:
+            if suffix == 's' or suffix == 'S':
+                return size * 512
+            mult = 1024
+        else:
+            mult = 1
+
+        if suffix == "k" or suffix == "K":
+            return size * mult
+        if suffix == "M":
+            return size * mult * 1024
+        if suffix == "G":
+            return size * mult * 1024 * 1024
+
+        raise ArgumentTypeError("Invalid size: %r" % arg)
+    return f
+
+def overheadtype(arg):
+    """
+    Custom type for ArgumentParser
+    Converts overhead string to float and checks if it's bigger than 1.0
+    """
+    try:
+        result = float(arg)
+    except ValueError:
+        raise ArgumentTypeError("Invalid value: %r" % arg)
+
+    if result < 1.0:
+        raise ArgumentTypeError("Overhead factor should be > 1.0" % arg)
+
+    return result
+
+def cannedpathtype(arg):
+    """
+    Custom type for ArgumentParser
+    Tries to find file in the list of canned wks paths
+    """
+    scripts_path = os.path.abspath(os.path.dirname(__file__) + '../../..')
+    result = find_canned(scripts_path, arg)
+    if not result:
+        raise ArgumentTypeError("file not found: %s" % arg)
+    return result
+
+def systemidtype(arg):
+    """
+    Custom type for ArgumentParser
+    Checks if the argument sutisfies system id requirements,
+    i.e. if it's one byte long integer > 0
+    """
+    error = "Invalid system type: %s. must be hex "\
+            "between 0x1 and 0xFF" % arg
+    try:
+        result = int(arg, 16)
+    except ValueError:
+        raise ArgumentTypeError(error)
+
+    if result <= 0 or result > 0xff:
+        raise ArgumentTypeError(error)
+
+    return arg
+
+class KickStart():
+    """Kickstart parser implementation."""
+
+    DEFAULT_EXTRA_SPACE = 10*1024
+    DEFAULT_OVERHEAD_FACTOR = 1.3
+
+    def __init__(self, confpath):
+
+        self.partitions = []
+        self.bootloader = None
+        self.lineno = 0
+        self.partnum = 0
+
+        parser = KickStartParser()
+        subparsers = parser.add_subparsers()
+
+        part = subparsers.add_parser('part')
+        part.add_argument('mountpoint', nargs='?')
+        part.add_argument('--active', action='store_true')
+        part.add_argument('--align', type=int)
+        part.add_argument('--offset', type=sizetype("K", True))
+        part.add_argument('--exclude-path', nargs='+')
+        part.add_argument('--include-path', nargs='+', action='append')
+        part.add_argument('--change-directory')
+        part.add_argument("--extra-space", type=sizetype("M"))
+        part.add_argument('--fsoptions', dest='fsopts')
+        part.add_argument('--fspassno', dest='fspassno')
+        part.add_argument('--fstype', default='vfat',
+                          choices=('ext2', 'ext3', 'ext4', 'btrfs',
+                                   'squashfs', 'vfat', 'msdos', 'erofs',
+                                   'swap', 'none'))
+        part.add_argument('--mkfs-extraopts', default='')
+        part.add_argument('--label')
+        part.add_argument('--use-label', action='store_true')
+        part.add_argument('--no-table', action='store_true')
+        part.add_argument('--ondisk', '--ondrive', dest='disk', default='sda')
+        part.add_argument("--overhead-factor", type=overheadtype)
+        part.add_argument('--part-name')
+        part.add_argument('--part-type')
+        part.add_argument('--rootfs-dir')
+        part.add_argument('--type', default='primary',
+                choices = ('primary', 'logical'))
+        part.add_argument('--hidden', action='store_true')
+
+        # --size and --fixed-size cannot be specified together; options
+        # ----extra-space and --overhead-factor should also raise a parser
+        # --error, but since nesting mutually exclusive groups does not work,
+        # ----extra-space/--overhead-factor are handled later
+        sizeexcl = part.add_mutually_exclusive_group()
+        sizeexcl.add_argument('--size', type=sizetype("M"), default=0)
+        sizeexcl.add_argument('--fixed-size', type=sizetype("M"), default=0)
+
+        part.add_argument('--source')
+        part.add_argument('--sourceparams')
+        part.add_argument('--system-id', type=systemidtype)
+        part.add_argument('--use-uuid', action='store_true')
+        part.add_argument('--uuid')
+        part.add_argument('--fsuuid')
+        part.add_argument('--no-fstab-update', action='store_true')
+        part.add_argument('--mbr', action='store_true')
+
+        bootloader = subparsers.add_parser('bootloader')
+        bootloader.add_argument('--append')
+        bootloader.add_argument('--configfile')
+        bootloader.add_argument('--ptable', choices=('msdos', 'gpt', 'gpt-hybrid'),
+                                default='msdos')
+        bootloader.add_argument('--timeout', type=int)
+        bootloader.add_argument('--source')
+
+        include = subparsers.add_parser('include')
+        include.add_argument('path', type=cannedpathtype)
+
+        self._parse(parser, confpath)
+        if not self.bootloader:
+            logger.warning('bootloader config not specified, using defaults\n')
+            self.bootloader = bootloader.parse_args([])
+
+    def _parse(self, parser, confpath):
+        """
+        Parse file in .wks format using provided parser.
+        """
+        with open(confpath) as conf:
+            lineno = 0
+            for line in conf:
+                line = line.strip()
+                lineno += 1
+                if line and line[0] != '#':
+                    line = expand_line(line)
+                    try:
+                        line_args = shlex.split(line)
+                        parsed = parser.parse_args(line_args)
+                    except ArgumentError as err:
+                        raise KickStartError('%s:%d: %s' % \
+                                             (confpath, lineno, err))
+                    if line.startswith('part'):
+                        # SquashFS does not support filesystem UUID
+                        if parsed.fstype == 'squashfs':
+                            if parsed.fsuuid:
+                                err = "%s:%d: SquashFS does not support UUID" \
+                                       % (confpath, lineno)
+                                raise KickStartError(err)
+                            if parsed.label:
+                                err = "%s:%d: SquashFS does not support LABEL" \
+                                       % (confpath, lineno)
+                                raise KickStartError(err)
+                        # erofs does not support filesystem labels
+                        if parsed.fstype == 'erofs' and parsed.label:
+                            err = "%s:%d: erofs does not support LABEL" % (confpath, lineno)
+                            raise KickStartError(err)
+                        if parsed.fstype == 'msdos' or parsed.fstype == 'vfat':
+                            if parsed.fsuuid:
+                                if parsed.fsuuid.upper().startswith('0X'):
+                                    if len(parsed.fsuuid) > 10:
+                                        err = "%s:%d: fsuuid %s given in wks kickstart file " \
+                                              "exceeds the length limit for %s filesystem. " \
+                                              "It should be in the form of a 32 bit hexadecimal" \
+                                              "number (for example, 0xABCD1234)." \
+                                              % (confpath, lineno, parsed.fsuuid, parsed.fstype)
+                                        raise KickStartError(err)
+                                elif len(parsed.fsuuid) > 8:
+                                    err = "%s:%d: fsuuid %s given in wks kickstart file " \
+                                          "exceeds the length limit for %s filesystem. " \
+                                          "It should be in the form of a 32 bit hexadecimal" \
+                                          "number (for example, 0xABCD1234)." \
+                                          % (confpath, lineno, parsed.fsuuid, parsed.fstype)
+                                    raise KickStartError(err)
+                        if parsed.use_label and not parsed.label:
+                            err = "%s:%d: Must set the label with --label" \
+                                  % (confpath, lineno)
+                            raise KickStartError(err)
+                        # using ArgumentParser one cannot easily tell if option
+                        # was passed as argument, if said option has a default
+                        # value; --overhead-factor/--extra-space cannot be used
+                        # with --fixed-size, so at least detect when these were
+                        # passed with non-0 values ...
+                        if parsed.fixed_size:
+                            if parsed.overhead_factor or parsed.extra_space:
+                                err = "%s:%d: arguments --overhead-factor and --extra-space not "\
+                                      "allowed with argument --fixed-size" \
+                                      % (confpath, lineno)
+                                raise KickStartError(err)
+                        else:
+                            # ... and provide defaults if not using
+                            # --fixed-size iff given option was not used
+                            # (again, one cannot tell if option was passed but
+                            # with value equal to 0)
+                            if '--overhead-factor' not in line_args:
+                                parsed.overhead_factor = self.DEFAULT_OVERHEAD_FACTOR
+                            if '--extra-space' not in line_args:
+                                parsed.extra_space = self.DEFAULT_EXTRA_SPACE
+
+                        self.partnum += 1
+                        self.partitions.append(Partition(parsed, self.partnum))
+                    elif line.startswith('include'):
+                        self._parse(parser, parsed.path)
+                    elif line.startswith('bootloader'):
+                        if not self.bootloader:
+                            self.bootloader = parsed
+                            # Concatenate the strings set in APPEND
+                            append_var = get_bitbake_var("APPEND")
+                            if append_var:
+                                self.bootloader.append = ' '.join(filter(None, \
+                                                         (self.bootloader.append, append_var)))
+                        else:
+                            err = "%s:%d: more than one bootloader specified" \
+                                      % (confpath, lineno)
+                            raise KickStartError(err)

sources/poky/scripts/lib/wic/misc.py

@@ -0,0 +1,266 @@
+#
+# Copyright (c) 2013, Intel Corporation.
+#
+# SPDX-License-Identifier: GPL-2.0-only
+#
+# DESCRIPTION
+# This module provides a place to collect various wic-related utils
+# for the OpenEmbedded Image Tools.
+#
+# AUTHORS
+# Tom Zanussi <tom.zanussi (at] linux.intel.com>
+#
+"""Miscellaneous functions."""
+
+import logging
+import os
+import re
+import subprocess
+import shutil
+
+from collections import defaultdict
+
+from wic import WicError
+
+logger = logging.getLogger('wic')
+
+# executable -> recipe pairs for exec_native_cmd
+NATIVE_RECIPES = {"bmaptool": "bmaptool",
+                  "dumpe2fs": "e2fsprogs",
+                  "grub-mkimage": "grub-efi",
+                  "isohybrid": "syslinux",
+                  "mcopy": "mtools",
+                  "mdel" : "mtools",
+                  "mdeltree" : "mtools",
+                  "mdir" : "mtools",
+                  "mkdosfs": "dosfstools",
+                  "mkisofs": "cdrtools",
+                  "mkfs.btrfs": "btrfs-tools",
+                  "mkfs.erofs": "erofs-utils",
+                  "mkfs.ext2": "e2fsprogs",
+                  "mkfs.ext3": "e2fsprogs",
+                  "mkfs.ext4": "e2fsprogs",
+                  "mkfs.vfat": "dosfstools",
+                  "mksquashfs": "squashfs-tools",
+                  "mkswap": "util-linux",
+                  "mmd": "mtools",
+                  "parted": "parted",
+                  "sfdisk": "util-linux",
+                  "sgdisk": "gptfdisk",
+                  "syslinux": "syslinux",
+                  "tar": "tar"
+                 }
+
+def runtool(cmdln_or_args):
+    """ wrapper for most of the subprocess calls
+    input:
+        cmdln_or_args: can be both args and cmdln str (shell=True)
+    return:
+        rc, output
+    """
+    if isinstance(cmdln_or_args, list):
+        cmd = cmdln_or_args[0]
+        shell = False
+    else:
+        import shlex
+        cmd = shlex.split(cmdln_or_args)[0]
+        shell = True
+
+    sout = subprocess.PIPE
+    serr = subprocess.STDOUT
+
+    try:
+        process = subprocess.Popen(cmdln_or_args, stdout=sout,
+                                   stderr=serr, shell=shell)
+        sout, serr = process.communicate()
+        # combine stdout and stderr, filter None out and decode
+        out = ''.join([out.decode('utf-8') for out in [sout, serr] if out])
+    except OSError as err:
+        if err.errno == 2:
+            # [Errno 2] No such file or directory
+            raise WicError('Cannot run command: %s, lost dependency?' % cmd)
+        else:
+            raise # relay
+
+    return process.returncode, out
+
+def _exec_cmd(cmd_and_args, as_shell=False):
+    """
+    Execute command, catching stderr, stdout
+
+    Need to execute as_shell if the command uses wildcards
+    """
+    logger.debug("_exec_cmd: %s", cmd_and_args)
+    args = cmd_and_args.split()
+    logger.debug(args)
+
+    if as_shell:
+        ret, out = runtool(cmd_and_args)
+    else:
+        ret, out = runtool(args)
+    out = out.strip()
+    if ret != 0:
+        raise WicError("_exec_cmd: %s returned '%s' instead of 0\noutput: %s" % \
+                       (cmd_and_args, ret, out))
+
+    logger.debug("_exec_cmd: output for %s (rc = %d): %s",
+                 cmd_and_args, ret, out)
+
+    return ret, out
+
+
+def exec_cmd(cmd_and_args, as_shell=False):
+    """
+    Execute command, return output
+    """
+    return _exec_cmd(cmd_and_args, as_shell)[1]
+
+def find_executable(cmd, paths):
+    recipe = cmd
+    if recipe in NATIVE_RECIPES:
+        recipe =  NATIVE_RECIPES[recipe]
+    provided = get_bitbake_var("ASSUME_PROVIDED")
+    if provided and "%s-native" % recipe in provided:
+        return True
+
+    return shutil.which(cmd, path=paths)
+
+def exec_native_cmd(cmd_and_args, native_sysroot, pseudo=""):
+    """
+    Execute native command, catching stderr, stdout
+
+    Need to execute as_shell if the command uses wildcards
+
+    Always need to execute native commands as_shell
+    """
+    # The reason -1 is used is because there may be "export" commands.
+    args = cmd_and_args.split(';')[-1].split()
+    logger.debug(args)
+
+    if pseudo:
+        cmd_and_args = pseudo + cmd_and_args
+
+    hosttools_dir = get_bitbake_var("HOSTTOOLS_DIR")
+    target_sys = get_bitbake_var("TARGET_SYS")
+
+    native_paths = "%s/sbin:%s/usr/sbin:%s/usr/bin:%s/usr/bin/%s:%s/bin:%s" % \
+                   (native_sysroot, native_sysroot,
+                    native_sysroot, native_sysroot, target_sys,
+                    native_sysroot, hosttools_dir)
+
+    native_cmd_and_args = "export PATH=%s:$PATH;%s" % \
+                   (native_paths, cmd_and_args)
+    logger.debug("exec_native_cmd: %s", native_cmd_and_args)
+
+    # If the command isn't in the native sysroot say we failed.
+    if find_executable(args[0], native_paths):
+        ret, out = _exec_cmd(native_cmd_and_args, True)
+    else:
+        ret = 127
+        out = "can't find native executable %s in %s" % (args[0], native_paths)
+
+    prog = args[0]
+    # shell command-not-found
+    if ret == 127 \
+       or (pseudo and ret == 1 and out == "Can't find '%s' in $PATH." % prog):
+        msg = "A native program %s required to build the image "\
+              "was not found (see details above).\n\n" % prog
+        recipe = NATIVE_RECIPES.get(prog)
+        if recipe:
+            msg += "Please make sure wic-tools have %s-native in its DEPENDS, "\
+                   "build it with 'bitbake wic-tools' and try again.\n" % recipe
+        else:
+            msg += "Wic failed to find a recipe to build native %s. Please "\
+                   "file a bug against wic.\n" % prog
+        raise WicError(msg)
+
+    return ret, out
+
+BOOTDD_EXTRA_SPACE = 16384
+
+class BitbakeVars(defaultdict):
+    """
+    Container for Bitbake variables.
+    """
+    def __init__(self):
+        defaultdict.__init__(self, dict)
+
+        # default_image and vars_dir attributes should be set from outside
+        self.default_image = None
+        self.vars_dir = None
+
+    def _parse_line(self, line, image, matcher=re.compile(r"^([a-zA-Z0-9\-_+./~]+)=(.*)")):
+        """
+        Parse one line from bitbake -e output or from .env file.
+        Put result key-value pair into the storage.
+        """
+        if "=" not in line:
+            return
+        match = matcher.match(line)
+        if not match:
+            return
+        key, val = match.groups()
+        self[image][key] = val.strip('"')
+
+    def get_var(self, var, image=None, cache=True):
+        """
+        Get bitbake variable from 'bitbake -e' output or from .env file.
+        This is a lazy method, i.e. it runs bitbake or parses file only when
+        only when variable is requested. It also caches results.
+        """
+        if not image:
+            image = self.default_image
+
+        if image not in self:
+            if image and self.vars_dir:
+                fname = os.path.join(self.vars_dir, image + '.env')
+                if os.path.isfile(fname):
+                    # parse .env file
+                    with open(fname) as varsfile:
+                        for line in varsfile:
+                            self._parse_line(line, image)
+                else:
+                    print("Couldn't get bitbake variable from %s." % fname)
+                    print("File %s doesn't exist." % fname)
+                    return
+            else:
+                # Get bitbake -e output
+                cmd = "bitbake -e"
+                if image:
+                    cmd += " %s" % image
+
+                log_level = logger.getEffectiveLevel()
+                logger.setLevel(logging.INFO)
+                ret, lines = _exec_cmd(cmd)
+                logger.setLevel(log_level)
+
+                if ret:
+                    logger.error("Couldn't get '%s' output.", cmd)
+                    logger.error("Bitbake failed with error:\n%s\n", lines)
+                    return
+
+                # Parse bitbake -e output
+                for line in lines.split('\n'):
+                    self._parse_line(line, image)
+
+            # Make first image a default set of variables
+            if cache:
+                images = [key for key in self if key]
+                if len(images) == 1:
+                    self[None] = self[image]
+
+        result = self[image].get(var)
+        if not cache:
+            self.pop(image, None)
+
+        return result
+
+# Create BB_VARS singleton
+BB_VARS = BitbakeVars()
+
+def get_bitbake_var(var, image=None, cache=True):
+    """
+    Provide old get_bitbake_var API by wrapping
+    get_var method of BB_VARS singleton.
+    """
+    return BB_VARS.get_var(var, image, cache)

sources/poky/scripts/lib/wic/partition.py

@@ -0,0 +1,551 @@
+#
+# Copyright (c) 2013-2016 Intel Corporation.
+#
+# SPDX-License-Identifier: GPL-2.0-only
+#
+# DESCRIPTION
+# This module provides the OpenEmbedded partition object definitions.
+#
+# AUTHORS
+# Tom Zanussi <tom.zanussi (at] linux.intel.com>
+# Ed Bartosh <ed.bartosh> (at] linux.intel.com>
+
+import logging
+import os
+import uuid
+
+from wic import WicError
+from wic.misc import exec_cmd, exec_native_cmd, get_bitbake_var
+from wic.pluginbase import PluginMgr
+
+logger = logging.getLogger('wic')
+
+class Partition():
+
+    def __init__(self, args, lineno):
+        self.args = args
+        self.active = args.active
+        self.align = args.align
+        self.disk = args.disk
+        self.device = None
+        self.extra_space = args.extra_space
+        self.exclude_path = args.exclude_path
+        self.include_path = args.include_path
+        self.change_directory = args.change_directory
+        self.fsopts = args.fsopts
+        self.fspassno = args.fspassno
+        self.fstype = args.fstype
+        self.label = args.label
+        self.use_label = args.use_label
+        self.mkfs_extraopts = args.mkfs_extraopts
+        self.mountpoint = args.mountpoint
+        self.no_table = args.no_table
+        self.num = None
+        self.offset = args.offset
+        self.overhead_factor = args.overhead_factor
+        self.part_name = args.part_name
+        self.part_type = args.part_type
+        self.rootfs_dir = args.rootfs_dir
+        self.size = args.size
+        self.fixed_size = args.fixed_size
+        self.source = args.source
+        self.sourceparams = args.sourceparams
+        self.system_id = args.system_id
+        self.use_uuid = args.use_uuid
+        self.uuid = args.uuid
+        self.fsuuid = args.fsuuid
+        self.type = args.type
+        self.no_fstab_update = args.no_fstab_update
+        self.updated_fstab_path = None
+        self.has_fstab = False
+        self.update_fstab_in_rootfs = False
+        self.hidden = args.hidden
+        self.mbr = args.mbr
+
+        self.lineno = lineno
+        self.source_file = ""
+
+    def get_extra_block_count(self, current_blocks):
+        """
+        The --size param is reflected in self.size (in kB), and we already
+        have current_blocks (1k) blocks, calculate and return the
+        number of (1k) blocks we need to add to get to --size, 0 if
+        we're already there or beyond.
+        """
+        logger.debug("Requested partition size for %s: %d",
+                     self.mountpoint, self.size)
+
+        if not self.size:
+            return 0
+
+        requested_blocks = self.size
+
+        logger.debug("Requested blocks %d, current_blocks %d",
+                     requested_blocks, current_blocks)
+
+        if requested_blocks > current_blocks:
+            return requested_blocks - current_blocks
+        else:
+            return 0
+
+    def get_rootfs_size(self, actual_rootfs_size=0):
+        """
+        Calculate the required size of rootfs taking into consideration
+        --size/--fixed-size flags as well as overhead and extra space, as
+        specified in kickstart file. Raises an error if the
+        `actual_rootfs_size` is larger than fixed-size rootfs.
+
+        """
+        if self.fixed_size:
+            rootfs_size = self.fixed_size
+            if actual_rootfs_size > rootfs_size:
+                raise WicError("Actual rootfs size (%d kB) is larger than "
+                               "allowed size %d kB" %
+                               (actual_rootfs_size, rootfs_size))
+        else:
+            extra_blocks = self.get_extra_block_count(actual_rootfs_size)
+            if extra_blocks < self.extra_space:
+                extra_blocks = self.extra_space
+
+            rootfs_size = actual_rootfs_size + extra_blocks
+            rootfs_size = int(rootfs_size * self.overhead_factor)
+
+            logger.debug("Added %d extra blocks to %s to get to %d total blocks",
+                         extra_blocks, self.mountpoint, rootfs_size)
+
+        return rootfs_size
+
+    @property
+    def disk_size(self):
+        """
+        Obtain on-disk size of partition taking into consideration
+        --size/--fixed-size options.
+
+        """
+        return self.fixed_size if self.fixed_size else self.size
+
+    def prepare(self, creator, cr_workdir, oe_builddir, rootfs_dir,
+                bootimg_dir, kernel_dir, native_sysroot, updated_fstab_path):
+        """
+        Prepare content for individual partitions, depending on
+        partition command parameters.
+        """
+        self.updated_fstab_path = updated_fstab_path
+        if self.updated_fstab_path and not (self.fstype.startswith("ext") or self.fstype == "msdos"):
+            self.update_fstab_in_rootfs = True
+
+        if not self.source:
+            if self.fstype == "none" or self.no_table:
+                return
+            if not self.size and not self.fixed_size:
+                raise WicError("The %s partition has a size of zero. Please "
+                               "specify a non-zero --size/--fixed-size for that "
+                               "partition." % self.mountpoint)
+
+            if self.fstype == "swap":
+                self.prepare_swap_partition(cr_workdir, oe_builddir,
+                                            native_sysroot)
+                self.source_file = "%s/fs.%s" % (cr_workdir, self.fstype)
+            else:
+                if self.fstype in ('squashfs', 'erofs'):
+                    raise WicError("It's not possible to create empty %s "
+                                   "partition '%s'" % (self.fstype, self.mountpoint))
+
+                rootfs = "%s/fs_%s.%s.%s" % (cr_workdir, self.label,
+                                             self.lineno, self.fstype)
+                if os.path.isfile(rootfs):
+                    os.remove(rootfs)
+
+                prefix = "ext" if self.fstype.startswith("ext") else self.fstype
+                method = getattr(self, "prepare_empty_partition_" + prefix)
+                method(rootfs, oe_builddir, native_sysroot)
+                self.source_file = rootfs
+            return
+
+        plugins = PluginMgr.get_plugins('source')
+
+        if self.source not in plugins:
+            raise WicError("The '%s' --source specified for %s doesn't exist.\n\t"
+                           "See 'wic list source-plugins' for a list of available"
+                           " --sources.\n\tSee 'wic help source-plugins' for "
+                           "details on adding a new source plugin." %
+                           (self.source, self.mountpoint))
+
+        srcparams_dict = {}
+        if self.sourceparams:
+            # Split sourceparams string of the form key1=val1[,key2=val2,...]
+            # into a dict.  Also accepts valueless keys i.e. without =
+            splitted = self.sourceparams.split(',')
+            srcparams_dict = dict((par.split('=', 1) + [None])[:2] for par in splitted if par)
+
+        plugin = PluginMgr.get_plugins('source')[self.source]
+        plugin.do_configure_partition(self, srcparams_dict, creator,
+                                      cr_workdir, oe_builddir, bootimg_dir,
+                                      kernel_dir, native_sysroot)
+        plugin.do_stage_partition(self, srcparams_dict, creator,
+                                  cr_workdir, oe_builddir, bootimg_dir,
+                                  kernel_dir, native_sysroot)
+        plugin.do_prepare_partition(self, srcparams_dict, creator,
+                                    cr_workdir, oe_builddir, bootimg_dir,
+                                    kernel_dir, rootfs_dir, native_sysroot)
+        plugin.do_post_partition(self, srcparams_dict, creator,
+                                    cr_workdir, oe_builddir, bootimg_dir,
+                                    kernel_dir, rootfs_dir, native_sysroot)
+
+        # further processing required Partition.size to be an integer, make
+        # sure that it is one
+        if not isinstance(self.size, int):
+            raise WicError("Partition %s internal size is not an integer. "
+                           "This a bug in source plugin %s and needs to be fixed." %
+                           (self.mountpoint, self.source))
+
+        if self.fixed_size and self.size > self.fixed_size:
+            raise WicError("File system image of partition %s is "
+                           "larger (%d kB) than its allowed size %d kB" %
+                           (self.mountpoint, self.size, self.fixed_size))
+
+    def prepare_rootfs(self, cr_workdir, oe_builddir, rootfs_dir,
+                       native_sysroot, real_rootfs = True, pseudo_dir = None):
+        """
+        Prepare content for a rootfs partition i.e. create a partition
+        and fill it from a /rootfs dir.
+
+        Currently handles ext2/3/4, btrfs, vfat and squashfs.
+        """
+
+        rootfs = "%s/rootfs_%s.%s.%s" % (cr_workdir, self.label,
+                                         self.lineno, self.fstype)
+        if os.path.isfile(rootfs):
+            os.remove(rootfs)
+
+        p_prefix = os.environ.get("PSEUDO_PREFIX", "%s/usr" % native_sysroot)
+        if (pseudo_dir):
+            # Canonicalize the ignore paths. This corresponds to
+            # calling oe.path.canonicalize(), which is used in bitbake.conf.
+            ignore_paths = [rootfs] + (get_bitbake_var("PSEUDO_IGNORE_PATHS") or "").split(",")
+            canonical_paths = []
+            for path in ignore_paths:
+                if "$" not in path:
+                    trailing_slash = path.endswith("/") and "/" or ""
+                    canonical_paths.append(os.path.realpath(path) + trailing_slash)
+            ignore_paths = ",".join(canonical_paths)
+
+            pseudo = "export PSEUDO_PREFIX=%s;" % p_prefix
+            pseudo += "export PSEUDO_LOCALSTATEDIR=%s;" % pseudo_dir
+            pseudo += "export PSEUDO_PASSWD=%s;" % rootfs_dir
+            pseudo += "export PSEUDO_NOSYMLINKEXP=1;"
+            pseudo += "export PSEUDO_IGNORE_PATHS=%s;" % ignore_paths
+            pseudo += "%s " % get_bitbake_var("FAKEROOTCMD")
+        else:
+            pseudo = None
+
+        if not self.size and real_rootfs:
+            # The rootfs size is not set in .ks file so try to get it
+            # from bitbake variable
+            rsize_bb = get_bitbake_var('ROOTFS_SIZE')
+            rdir = get_bitbake_var('IMAGE_ROOTFS')
+            if rsize_bb and rdir == rootfs_dir:
+                # Bitbake variable ROOTFS_SIZE is calculated in
+                # Image._get_rootfs_size method from meta/lib/oe/image.py
+                # using IMAGE_ROOTFS_SIZE, IMAGE_ROOTFS_ALIGNMENT,
+                # IMAGE_OVERHEAD_FACTOR and IMAGE_ROOTFS_EXTRA_SPACE
+                self.size = int(round(float(rsize_bb)))
+            else:
+                # Bitbake variable ROOTFS_SIZE is not defined so compute it
+                # from the rootfs_dir size using the same logic found in
+                # get_rootfs_size() from meta/classes/image.bbclass
+                du_cmd = "du -ks %s" % rootfs_dir
+                out = exec_cmd(du_cmd)
+                self.size = int(out.split()[0])
+
+        prefix = "ext" if self.fstype.startswith("ext") else self.fstype
+        method = getattr(self, "prepare_rootfs_" + prefix)
+        method(rootfs, cr_workdir, oe_builddir, rootfs_dir, native_sysroot, pseudo)
+        self.source_file = rootfs
+
+        # get the rootfs size in the right units for kickstart (kB)
+        du_cmd = "du -Lbks %s" % rootfs
+        out = exec_cmd(du_cmd)
+        self.size = int(out.split()[0])
+
+    def prepare_rootfs_ext(self, rootfs, cr_workdir, oe_builddir, rootfs_dir,
+                           native_sysroot, pseudo):
+        """
+        Prepare content for an ext2/3/4 rootfs partition.
+        """
+        du_cmd = "du -ks %s" % rootfs_dir
+        out = exec_cmd(du_cmd)
+        actual_rootfs_size = int(out.split()[0])
+
+        rootfs_size = self.get_rootfs_size(actual_rootfs_size)
+
+        with open(rootfs, 'w') as sparse:
+            os.ftruncate(sparse.fileno(), rootfs_size * 1024)
+
+        extraopts = self.mkfs_extraopts or "-F -i 8192"
+
+        # use hash_seed to generate reproducible ext4 images
+        (extraopts, pseudo) = self.get_hash_seed_ext4(extraopts, pseudo)
+
+        label_str = ""
+        if self.label:
+            label_str = "-L %s" % self.label
+
+        mkfs_cmd = "mkfs.%s %s %s %s -U %s -d %s" % \
+            (self.fstype, extraopts, rootfs, label_str, self.fsuuid, rootfs_dir)
+        exec_native_cmd(mkfs_cmd, native_sysroot, pseudo=pseudo)
+
+        if self.updated_fstab_path and self.has_fstab and not self.no_fstab_update:
+            debugfs_script_path = os.path.join(cr_workdir, "debugfs_script")
+            with open(debugfs_script_path, "w") as f:
+                f.write("cd etc\n")
+                f.write("rm fstab\n")
+                f.write("write %s fstab\n" % (self.updated_fstab_path))
+            debugfs_cmd = "debugfs -w -f %s %s" % (debugfs_script_path, rootfs)
+            exec_native_cmd(debugfs_cmd, native_sysroot)
+
+        mkfs_cmd = "fsck.%s -pvfD %s" % (self.fstype, rootfs)
+        exec_native_cmd(mkfs_cmd, native_sysroot, pseudo=pseudo)
+
+        if os.getenv('SOURCE_DATE_EPOCH'):
+            sde_time = hex(int(os.getenv('SOURCE_DATE_EPOCH')))
+            debugfs_script_path = os.path.join(cr_workdir, "debugfs_script")
+            files = []
+            for root, dirs, others in os.walk(rootfs_dir):
+                base = root.replace(rootfs_dir, "").rstrip(os.sep)
+                files += [ "/" if base == "" else base ]
+                files += [ base + "/" + n for n in dirs + others ]
+            with open(debugfs_script_path, "w") as f:
+                f.write("set_current_time %s\n" % (sde_time))
+                if self.updated_fstab_path and self.has_fstab and not self.no_fstab_update:
+                    f.write("set_inode_field /etc/fstab mtime %s\n" % (sde_time))
+                    f.write("set_inode_field /etc/fstab mtime_extra 0\n")
+                for file in set(files):
+                    for time in ["atime", "ctime", "crtime"]:
+                        f.write("set_inode_field \"%s\" %s %s\n" % (file, time, sde_time))
+                        f.write("set_inode_field \"%s\" %s_extra 0\n" % (file, time))
+                for time in ["wtime", "mkfs_time", "lastcheck"]:
+                    f.write("set_super_value %s %s\n" % (time, sde_time))
+                for time in ["mtime", "first_error_time", "last_error_time"]:
+                    f.write("set_super_value %s 0\n" % (time))
+            debugfs_cmd = "debugfs -w -f %s %s" % (debugfs_script_path, rootfs)
+            exec_native_cmd(debugfs_cmd, native_sysroot)
+
+        self.check_for_Y2038_problem(rootfs, native_sysroot)
+
+    def get_hash_seed_ext4(self, extraopts, pseudo):
+        if os.getenv('SOURCE_DATE_EPOCH'):
+            sde_time = int(os.getenv('SOURCE_DATE_EPOCH'))
+            if pseudo:
+                pseudo = "export E2FSPROGS_FAKE_TIME=%s;%s " % (sde_time, pseudo)
+            else:
+                pseudo = "export E2FSPROGS_FAKE_TIME=%s; " % sde_time
+
+            # Set hash_seed to generate deterministic directory indexes
+            namespace = uuid.UUID("e7429877-e7b3-4a68-a5c9-2f2fdf33d460")
+            if self.fsuuid:
+                namespace = uuid.UUID(self.fsuuid)
+            hash_seed = str(uuid.uuid5(namespace, str(sde_time)))
+            extraopts += " -E hash_seed=%s" % hash_seed
+
+        return (extraopts, pseudo)
+
+    def prepare_rootfs_btrfs(self, rootfs, cr_workdir, oe_builddir, rootfs_dir,
+                             native_sysroot, pseudo):
+        """
+        Prepare content for a btrfs rootfs partition.
+        """
+        du_cmd = "du -ks %s" % rootfs_dir
+        out = exec_cmd(du_cmd)
+        actual_rootfs_size = int(out.split()[0])
+
+        rootfs_size = self.get_rootfs_size(actual_rootfs_size)
+
+        with open(rootfs, 'w') as sparse:
+            os.ftruncate(sparse.fileno(), rootfs_size * 1024)
+
+        label_str = ""
+        if self.label:
+            label_str = "-L %s" % self.label
+
+        mkfs_cmd = "mkfs.%s -b %d -r %s %s %s -U %s %s" % \
+            (self.fstype, rootfs_size * 1024, rootfs_dir, label_str,
+             self.mkfs_extraopts, self.fsuuid, rootfs)
+        exec_native_cmd(mkfs_cmd, native_sysroot, pseudo=pseudo)
+
+    def prepare_rootfs_msdos(self, rootfs, cr_workdir, oe_builddir, rootfs_dir,
+                             native_sysroot, pseudo):
+        """
+        Prepare content for a msdos/vfat rootfs partition.
+        """
+        du_cmd = "du -bks %s" % rootfs_dir
+        out = exec_cmd(du_cmd)
+        blocks = int(out.split()[0])
+
+        rootfs_size = self.get_rootfs_size(blocks)
+
+        label_str = "-n boot"
+        if self.label:
+            label_str = "-n %s" % self.label
+
+        size_str = ""
+
+        extraopts = self.mkfs_extraopts or '-S 512'
+
+        dosfs_cmd = "mkdosfs %s -i %s %s %s -C %s %d" % \
+                    (label_str, self.fsuuid, size_str, extraopts, rootfs,
+                     rootfs_size)
+        exec_native_cmd(dosfs_cmd, native_sysroot)
+
+        mcopy_cmd = "mcopy -i %s -s %s/* ::/" % (rootfs, rootfs_dir)
+        exec_native_cmd(mcopy_cmd, native_sysroot)
+
+        if self.updated_fstab_path and self.has_fstab and not self.no_fstab_update:
+            mcopy_cmd = "mcopy -m -i %s %s ::/etc/fstab" % (rootfs, self.updated_fstab_path)
+            exec_native_cmd(mcopy_cmd, native_sysroot)
+
+        chmod_cmd = "chmod 644 %s" % rootfs
+        exec_cmd(chmod_cmd)
+
+    prepare_rootfs_vfat = prepare_rootfs_msdos
+
+    def prepare_rootfs_squashfs(self, rootfs, cr_workdir, oe_builddir, rootfs_dir,
+                                native_sysroot, pseudo):
+        """
+        Prepare content for a squashfs rootfs partition.
+        """
+        extraopts = self.mkfs_extraopts or '-noappend'
+        squashfs_cmd = "mksquashfs %s %s %s" % \
+                       (rootfs_dir, rootfs, extraopts)
+        exec_native_cmd(squashfs_cmd, native_sysroot, pseudo=pseudo)
+
+    def prepare_rootfs_erofs(self, rootfs, cr_workdir, oe_builddir, rootfs_dir,
+                             native_sysroot, pseudo):
+        """
+        Prepare content for a erofs rootfs partition.
+        """
+        extraopts = self.mkfs_extraopts or ''
+        erofs_cmd = "mkfs.erofs %s -U %s %s %s" % \
+                       (extraopts, self.fsuuid, rootfs, rootfs_dir)
+        exec_native_cmd(erofs_cmd, native_sysroot, pseudo=pseudo)
+
+    def prepare_empty_partition_none(self, rootfs, oe_builddir, native_sysroot):
+        pass
+
+    def prepare_empty_partition_ext(self, rootfs, oe_builddir,
+                                    native_sysroot):
+        """
+        Prepare an empty ext2/3/4 partition.
+        """
+        size = self.disk_size
+        with open(rootfs, 'w') as sparse:
+            os.ftruncate(sparse.fileno(), size * 1024)
+
+        extraopts = self.mkfs_extraopts or "-i 8192"
+
+        # use hash_seed to generate reproducible ext4 images
+        (extraopts, pseudo) = self.get_hash_seed_ext4(extraopts, None)
+
+        label_str = ""
+        if self.label:
+            label_str = "-L %s" % self.label
+
+        mkfs_cmd = "mkfs.%s -F %s %s -U %s %s" % \
+            (self.fstype, extraopts, label_str, self.fsuuid, rootfs)
+        exec_native_cmd(mkfs_cmd, native_sysroot, pseudo=pseudo)
+
+        self.check_for_Y2038_problem(rootfs, native_sysroot)
+
+    def prepare_empty_partition_btrfs(self, rootfs, oe_builddir,
+                                      native_sysroot):
+        """
+        Prepare an empty btrfs partition.
+        """
+        size = self.disk_size
+        with open(rootfs, 'w') as sparse:
+            os.ftruncate(sparse.fileno(), size * 1024)
+
+        label_str = ""
+        if self.label:
+            label_str = "-L %s" % self.label
+
+        mkfs_cmd = "mkfs.%s -b %d %s -U %s %s %s" % \
+                   (self.fstype, self.size * 1024, label_str, self.fsuuid,
+                    self.mkfs_extraopts, rootfs)
+        exec_native_cmd(mkfs_cmd, native_sysroot)
+
+    def prepare_empty_partition_msdos(self, rootfs, oe_builddir,
+                                      native_sysroot):
+        """
+        Prepare an empty vfat partition.
+        """
+        blocks = self.disk_size
+
+        label_str = "-n boot"
+        if self.label:
+            label_str = "-n %s" % self.label
+
+        size_str = ""
+
+        extraopts = self.mkfs_extraopts or '-S 512'
+
+        dosfs_cmd = "mkdosfs %s -i %s %s %s -C %s %d" % \
+                    (label_str, self.fsuuid, extraopts, size_str, rootfs,
+                     blocks)
+
+        exec_native_cmd(dosfs_cmd, native_sysroot)
+
+        chmod_cmd = "chmod 644 %s" % rootfs
+        exec_cmd(chmod_cmd)
+
+    prepare_empty_partition_vfat = prepare_empty_partition_msdos
+
+    def prepare_swap_partition(self, cr_workdir, oe_builddir, native_sysroot):
+        """
+        Prepare a swap partition.
+        """
+        path = "%s/fs.%s" % (cr_workdir, self.fstype)
+
+        with open(path, 'w') as sparse:
+            os.ftruncate(sparse.fileno(), self.size * 1024)
+
+        label_str = ""
+        if self.label:
+            label_str = "-L %s" % self.label
+
+        mkswap_cmd = "mkswap %s -U %s %s" % (label_str, self.fsuuid, path)
+        exec_native_cmd(mkswap_cmd, native_sysroot)
+
+    def check_for_Y2038_problem(self, rootfs, native_sysroot):
+        """
+        Check if the filesystem is affected by the Y2038 problem
+        (Y2038 problem = 32 bit time_t overflow in January 2038)
+        """
+        def get_err_str(part):
+            err = "The {} filesystem {} has no Y2038 support."
+            if part.mountpoint:
+                args = [part.fstype, "mounted at %s" % part.mountpoint]
+            elif part.label:
+                args = [part.fstype, "labeled '%s'" % part.label]
+            elif part.part_name:
+                args = [part.fstype, "in partition '%s'" % part.part_name]
+            else:
+                args = [part.fstype, "in partition %s" % part.num]
+            return err.format(*args)
+
+        # ext2 and ext3 are always affected by the Y2038 problem
+        if self.fstype in ["ext2", "ext3"]:
+            logger.warn(get_err_str(self))
+            return
+
+        ret, out = exec_native_cmd("dumpe2fs %s" % rootfs, native_sysroot)
+
+        # if ext4 is affected by the Y2038 problem depends on the inode size
+        for line in out.splitlines():
+            if line.startswith("Inode size:"):
+                size = int(line.split(":")[1].strip())
+                if size < 256:
+                    logger.warn("%s Inodes (of size %d) are too small." %
+                                (get_err_str(self), size))
+                break
+

sources/poky/scripts/lib/wic/pluginbase.py

@@ -0,0 +1,144 @@
+#!/usr/bin/env python3
+#
+# Copyright (c) 2011 Intel, Inc.
+#
+# SPDX-License-Identifier: GPL-2.0-only
+#
+
+__all__ = ['ImagerPlugin', 'SourcePlugin']
+
+import os
+import logging
+import types
+
+from collections import defaultdict
+import importlib
+import importlib.util
+
+from wic import WicError
+from wic.misc import get_bitbake_var
+
+PLUGIN_TYPES = ["imager", "source"]
+
+SCRIPTS_PLUGIN_DIR = ["scripts/lib/wic/plugins", "lib/wic/plugins"]
+
+logger = logging.getLogger('wic')
+
+PLUGINS = defaultdict(dict)
+
+class PluginMgr:
+    _plugin_dirs = []
+
+    @classmethod
+    def get_plugins(cls, ptype):
+        """Get dictionary of <plugin_name>:<class> pairs."""
+        if ptype not in PLUGIN_TYPES:
+            raise WicError('%s is not valid plugin type' % ptype)
+
+        # collect plugin directories
+        if not cls._plugin_dirs:
+            cls._plugin_dirs = [os.path.join(os.path.dirname(__file__), 'plugins')]
+            layers = get_bitbake_var("BBLAYERS") or ''
+            for layer_path in layers.split():
+                for script_plugin_dir in SCRIPTS_PLUGIN_DIR:
+                    path = os.path.join(layer_path, script_plugin_dir)
+                    path = os.path.abspath(os.path.expanduser(path))
+                    if path not in cls._plugin_dirs and os.path.isdir(path):
+                        cls._plugin_dirs.insert(0, path)
+
+        if ptype not in PLUGINS:
+            # load all ptype plugins
+            for pdir in cls._plugin_dirs:
+                ppath = os.path.join(pdir, ptype)
+                if os.path.isdir(ppath):
+                    for fname in os.listdir(ppath):
+                        if fname.endswith('.py'):
+                            mname = fname[:-3]
+                            mpath = os.path.join(ppath, fname)
+                            logger.debug("loading plugin module %s", mpath)
+                            spec = importlib.util.spec_from_file_location(mname, mpath)
+                            module = importlib.util.module_from_spec(spec)
+                            spec.loader.exec_module(module)
+
+        return PLUGINS.get(ptype)
+
+class PluginMeta(type):
+    def __new__(cls, name, bases, attrs):
+        class_type = type.__new__(cls, name, bases, attrs)
+        if 'name' in attrs:
+            PLUGINS[class_type.wic_plugin_type][attrs['name']] = class_type
+
+        return class_type
+
+class ImagerPlugin(metaclass=PluginMeta):
+    wic_plugin_type = "imager"
+
+    def do_create(self):
+        raise WicError("Method %s.do_create is not implemented" %
+                       self.__class__.__name__)
+
+class SourcePlugin(metaclass=PluginMeta):
+    wic_plugin_type = "source"
+    """
+    The methods that can be implemented by --source plugins.
+
+    Any methods not implemented in a subclass inherit these.
+    """
+
+    @classmethod
+    def do_install_disk(cls, disk, disk_name, creator, workdir, oe_builddir,
+                        bootimg_dir, kernel_dir, native_sysroot):
+        """
+        Called after all partitions have been prepared and assembled into a
+        disk image.  This provides a hook to allow finalization of a
+        disk image e.g. to write an MBR to it.
+        """
+        logger.debug("SourcePlugin: do_install_disk: disk: %s", disk_name)
+
+    @classmethod
+    def do_stage_partition(cls, part, source_params, creator, cr_workdir,
+                           oe_builddir, bootimg_dir, kernel_dir,
+                           native_sysroot):
+        """
+        Special content staging hook called before do_prepare_partition(),
+        normally empty.
+
+        Typically, a partition will just use the passed-in parame e.g
+        straight bootimg_dir, etc, but in some cases, things need to
+        be more tailored e.g. to use a deploy dir + /boot, etc.  This
+        hook allows those files to be staged in a customized fashion.
+        Not that get_bitbake_var() allows you to acces non-standard
+        variables that you might want to use for this.
+        """
+        logger.debug("SourcePlugin: do_stage_partition: part: %s", part)
+
+    @classmethod
+    def do_configure_partition(cls, part, source_params, creator, cr_workdir,
+                               oe_builddir, bootimg_dir, kernel_dir,
+                               native_sysroot):
+        """
+        Called before do_prepare_partition(), typically used to create
+        custom configuration files for a partition, for example
+        syslinux or grub config files.
+        """
+        logger.debug("SourcePlugin: do_configure_partition: part: %s", part)
+
+    @classmethod
+    def do_prepare_partition(cls, part, source_params, creator, cr_workdir,
+                             oe_builddir, bootimg_dir, kernel_dir, rootfs_dir,
+                             native_sysroot):
+        """
+        Called to do the actual content population for a partition i.e. it
+        'prepares' the partition to be incorporated into the image.
+        """
+        logger.debug("SourcePlugin: do_prepare_partition: part: %s", part)
+
+    @classmethod
+    def do_post_partition(cls, part, source_params, creator, cr_workdir,
+                             oe_builddir, bootimg_dir, kernel_dir, rootfs_dir,
+                             native_sysroot):
+        """
+        Called after the partition is created. It is useful to add post
+        operations e.g. security signing the partition.
+        """
+        logger.debug("SourcePlugin: do_post_partition: part: %s", part)

sources/poky/scripts/lib/wic/plugins/imager/direct.py

@@ -0,0 +1,694 @@
+#
+# Copyright (c) 2013, Intel Corporation.
+#
+# SPDX-License-Identifier: GPL-2.0-only
+#
+# DESCRIPTION
+# This implements the 'direct' imager plugin class for 'wic'
+#
+# AUTHORS
+# Tom Zanussi <tom.zanussi (at] linux.intel.com>
+#
+
+import logging
+import os
+import random
+import shutil
+import tempfile
+import uuid
+
+from time import strftime
+
+from oe.path import copyhardlinktree
+
+from wic import WicError
+from wic.filemap import sparse_copy
+from wic.ksparser import KickStart, KickStartError
+from wic.pluginbase import PluginMgr, ImagerPlugin
+from wic.misc import get_bitbake_var, exec_cmd, exec_native_cmd
+
+logger = logging.getLogger('wic')
+
+class DirectPlugin(ImagerPlugin):
+    """
+    Install a system into a file containing a partitioned disk image.
+
+    An image file is formatted with a partition table, each partition
+    created from a rootfs or other OpenEmbedded build artifact and dd'ed
+    into the virtual disk. The disk image can subsequently be dd'ed onto
+    media and used on actual hardware.
+    """
+    name = 'direct'
+
+    def __init__(self, wks_file, rootfs_dir, bootimg_dir, kernel_dir,
+                 native_sysroot, oe_builddir, options):
+        try:
+            self.ks = KickStart(wks_file)
+        except KickStartError as err:
+            raise WicError(str(err))
+
+        # parse possible 'rootfs=name' items
+        self.rootfs_dir = dict(rdir.split('=') for rdir in rootfs_dir.split(' '))
+        self.bootimg_dir = bootimg_dir
+        self.kernel_dir = kernel_dir
+        self.native_sysroot = native_sysroot
+        self.oe_builddir = oe_builddir
+
+        self.debug = options.debug
+        self.outdir = options.outdir
+        self.compressor = options.compressor
+        self.bmap = options.bmap
+        self.no_fstab_update = options.no_fstab_update
+        self.updated_fstab_path = None
+
+        self.name = "%s-%s" % (os.path.splitext(os.path.basename(wks_file))[0],
+                               strftime("%Y%m%d%H%M"))
+        self.workdir = self.setup_workdir(options.workdir)
+        self._image = None
+        self.ptable_format = self.ks.bootloader.ptable
+        self.parts = self.ks.partitions
+
+        # as a convenience, set source to the boot partition source
+        # instead of forcing it to be set via bootloader --source
+        for part in self.parts:
+            if not self.ks.bootloader.source and part.mountpoint == "/boot":
+                self.ks.bootloader.source = part.source
+                break
+
+        image_path = self._full_path(self.workdir, self.parts[0].disk, "direct")
+        self._image = PartitionedImage(image_path, self.ptable_format,
+                                       self.parts, self.native_sysroot,
+                                       options.extra_space)
+
+    def setup_workdir(self, workdir):
+        if workdir:
+            if os.path.exists(workdir):
+                raise WicError("Internal workdir '%s' specified in wic arguments already exists!" % (workdir))
+
+            os.makedirs(workdir)
+            return workdir
+        else:
+            return tempfile.mkdtemp(dir=self.outdir, prefix='tmp.wic.')
+
+    def do_create(self):
+        """
+        Plugin entry point.
+        """
+        try:
+            self.create()
+            self.assemble()
+            self.finalize()
+            self.print_info()
+        finally:
+            self.cleanup()
+
+    def update_fstab(self, image_rootfs):
+        """Assume partition order same as in wks"""
+        if not image_rootfs:
+            return
+
+        fstab_path = image_rootfs + "/etc/fstab"
+        if not os.path.isfile(fstab_path):
+            return
+
+        with open(fstab_path) as fstab:
+            fstab_lines = fstab.readlines()
+
+        updated = False
+        for part in self.parts:
+            if not part.realnum or not part.mountpoint \
+               or part.mountpoint == "/" or not (part.mountpoint.startswith('/') or part.mountpoint == "swap"):
+                continue
+
+            if part.use_uuid:
+                if part.fsuuid:
+                    # FAT UUID is different from others
+                    if len(part.fsuuid) == 10:
+                        device_name = "UUID=%s-%s" % \
+                                       (part.fsuuid[2:6], part.fsuuid[6:])
+                    else:
+                        device_name = "UUID=%s" % part.fsuuid
+                else:
+                    device_name = "PARTUUID=%s" % part.uuid
+            elif part.use_label:
+                device_name = "LABEL=%s" % part.label
+            else:
+                # mmc device partitions are named mmcblk0p1, mmcblk0p2..
+                prefix = 'p' if  part.disk.startswith('mmcblk') else ''
+                device_name = "/dev/%s%s%d" % (part.disk, prefix, part.realnum)
+
+            opts = part.fsopts if part.fsopts else "defaults"
+            passno = part.fspassno if part.fspassno else "0"
+            line = "\t".join([device_name, part.mountpoint, part.fstype,
+                              opts, "0", passno]) + "\n"
+
+            fstab_lines.append(line)
+            updated = True
+
+        if updated:
+            self.updated_fstab_path = os.path.join(self.workdir, "fstab")
+            with open(self.updated_fstab_path, "w") as f:
+                f.writelines(fstab_lines)
+            if os.getenv('SOURCE_DATE_EPOCH'):
+                fstab_time = int(os.getenv('SOURCE_DATE_EPOCH'))
+                os.utime(self.updated_fstab_path, (fstab_time, fstab_time))
+
+    def _full_path(self, path, name, extention):
+        """ Construct full file path to a file we generate. """
+        return os.path.join(path, "%s-%s.%s" % (self.name, name, extention))
+
+    #
+    # Actual implemention
+    #
+    def create(self):
+        """
+        For 'wic', we already have our build artifacts - we just create
+        filesystems from the artifacts directly and combine them into
+        a partitioned image.
+        """
+        if not self.no_fstab_update:
+            self.update_fstab(self.rootfs_dir.get("ROOTFS_DIR"))
+
+        for part in self.parts:
+            # get rootfs size from bitbake variable if it's not set in .ks file
+            if not part.size:
+                # and if rootfs name is specified for the partition
+                image_name = self.rootfs_dir.get(part.rootfs_dir)
+                if image_name and os.path.sep not in image_name:
+                    # Bitbake variable ROOTFS_SIZE is calculated in
+                    # Image._get_rootfs_size method from meta/lib/oe/image.py
+                    # using IMAGE_ROOTFS_SIZE, IMAGE_ROOTFS_ALIGNMENT,
+                    # IMAGE_OVERHEAD_FACTOR and IMAGE_ROOTFS_EXTRA_SPACE
+                    rsize_bb = get_bitbake_var('ROOTFS_SIZE', image_name)
+                    if rsize_bb:
+                        part.size = int(round(float(rsize_bb)))
+
+        self._image.prepare(self)
+        self._image.layout_partitions()
+        self._image.create()
+
+    def assemble(self):
+        """
+        Assemble partitions into disk image
+        """
+        self._image.assemble()
+
+    def finalize(self):
+        """
+        Finalize the disk image.
+
+        For example, prepare the image to be bootable by e.g.
+        creating and installing a bootloader configuration.
+        """
+        source_plugin = self.ks.bootloader.source
+        disk_name = self.parts[0].disk
+        if source_plugin:
+            plugin = PluginMgr.get_plugins('source')[source_plugin]
+            plugin.do_install_disk(self._image, disk_name, self, self.workdir,
+                                   self.oe_builddir, self.bootimg_dir,
+                                   self.kernel_dir, self.native_sysroot)
+
+        full_path = self._image.path
+        # Generate .bmap
+        if self.bmap:
+            logger.debug("Generating bmap file for %s", disk_name)
+            python = os.path.join(self.native_sysroot, 'usr/bin/python3-native/python3')
+            bmaptool = os.path.join(self.native_sysroot, 'usr/bin/bmaptool')
+            exec_native_cmd("%s %s create %s -o %s.bmap" % \
+                            (python, bmaptool, full_path, full_path), self.native_sysroot)
+        # Compress the image
+        if self.compressor:
+            logger.debug("Compressing disk %s with %s", disk_name, self.compressor)
+            exec_cmd("%s %s" % (self.compressor, full_path))
+
+    def print_info(self):
+        """
+        Print the image(s) and artifacts used, for the user.
+        """
+        msg = "The new image(s) can be found here:\n"
+
+        extension = "direct" + {"gzip": ".gz",
+                                "bzip2": ".bz2",
+                                "xz": ".xz",
+                                None: ""}.get(self.compressor)
+        full_path = self._full_path(self.outdir, self.parts[0].disk, extension)
+        msg += '  %s\n\n' % full_path
+
+        msg += 'The following build artifacts were used to create the image(s):\n'
+        for part in self.parts:
+            if part.rootfs_dir is None:
+                continue
+            if part.mountpoint == '/':
+                suffix = ':'
+            else:
+                suffix = '["%s"]:' % (part.mountpoint or part.label)
+            rootdir = part.rootfs_dir
+            msg += '  ROOTFS_DIR%s%s\n' % (suffix.ljust(20), rootdir)
+
+        msg += '  BOOTIMG_DIR:                  %s\n' % self.bootimg_dir
+        msg += '  KERNEL_DIR:                   %s\n' % self.kernel_dir
+        msg += '  NATIVE_SYSROOT:               %s\n' % self.native_sysroot
+
+        logger.info(msg)
+
+    @property
+    def rootdev(self):
+        """
+        Get root device name to use as a 'root' parameter
+        in kernel command line.
+
+        Assume partition order same as in wks
+        """
+        for part in self.parts:
+            if part.mountpoint == "/":
+                if part.uuid:
+                    return "PARTUUID=%s" % part.uuid
+                elif part.label and self.ptable_format != 'msdos':
+                    return "PARTLABEL=%s" % part.label
+                else:
+                    suffix = 'p' if part.disk.startswith('mmcblk') else ''
+                    return "/dev/%s%s%-d" % (part.disk, suffix, part.realnum)
+
+    def cleanup(self):
+        if self._image:
+            self._image.cleanup()
+
+        # Move results to the output dir
+        if not os.path.exists(self.outdir):
+            os.makedirs(self.outdir)
+
+        for fname in os.listdir(self.workdir):
+            path = os.path.join(self.workdir, fname)
+            if os.path.isfile(path):
+                shutil.move(path, os.path.join(self.outdir, fname))
+
+        # remove work directory when it is not in debugging mode
+        if not self.debug:
+            shutil.rmtree(self.workdir, ignore_errors=True)
+
+# Overhead of the MBR partitioning scheme (just one sector)
+MBR_OVERHEAD = 1
+
+# Overhead of the GPT partitioning scheme
+GPT_OVERHEAD = 34
+
+# Size of a sector in bytes
+SECTOR_SIZE = 512
+
+class PartitionedImage():
+    """
+    Partitioned image in a file.
+    """
+
+    def __init__(self, path, ptable_format, partitions, native_sysroot=None, extra_space=0):
+        self.path = path  # Path to the image file
+        self.numpart = 0  # Number of allocated partitions
+        self.realpart = 0 # Number of partitions in the partition table
+        self.primary_part_num = 0  # Number of primary partitions (msdos)
+        self.extendedpart = 0      # Create extended partition before this logical partition (msdos)
+        self.extended_size_sec = 0 # Size of exteded partition (msdos)
+        self.logical_part_cnt = 0  # Number of total logical paritions (msdos)
+        self.offset = 0   # Offset of next partition (in sectors)
+        self.min_size = 0 # Minimum required disk size to fit
+                          # all partitions (in bytes)
+        self.ptable_format = ptable_format  # Partition table format
+        # Disk system identifier
+        if os.getenv('SOURCE_DATE_EPOCH'):
+            self.identifier = random.Random(int(os.getenv('SOURCE_DATE_EPOCH'))).randint(1, 0xffffffff)
+        else:
+            self.identifier = random.SystemRandom().randint(1, 0xffffffff)
+
+        self.partitions = partitions
+        self.partimages = []
+        # Size of a sector used in calculations
+        self.sector_size = SECTOR_SIZE
+        self.native_sysroot = native_sysroot
+        num_real_partitions = len([p for p in self.partitions if not p.no_table])
+        self.extra_space = extra_space
+
+        # calculate the real partition number, accounting for partitions not
+        # in the partition table and logical partitions
+        realnum = 0
+        for part in self.partitions:
+            if part.no_table:
+                part.realnum = 0
+            else:
+                realnum += 1
+                if self.ptable_format == 'msdos' and realnum > 3 and num_real_partitions > 4:
+                    part.realnum = realnum + 1
+                    continue
+                part.realnum = realnum
+
+        # generate parition and filesystem UUIDs
+        for part in self.partitions:
+            if not part.uuid and part.use_uuid:
+                if self.ptable_format in ('gpt', 'gpt-hybrid'):
+                    part.uuid = str(uuid.uuid4())
+                else: # msdos partition table
+                    part.uuid = '%08x-%02d' % (self.identifier, part.realnum)
+            if not part.fsuuid:
+                if part.fstype == 'vfat' or part.fstype == 'msdos':
+                    part.fsuuid = '0x' + str(uuid.uuid4())[:8].upper()
+                else:
+                    part.fsuuid = str(uuid.uuid4())
+            else:
+                #make sure the fsuuid for vfat/msdos align with format 0xYYYYYYYY
+                if part.fstype == 'vfat' or part.fstype == 'msdos':
+                    if part.fsuuid.upper().startswith("0X"):
+                        part.fsuuid = '0x' + part.fsuuid.upper()[2:].rjust(8,"0")
+                    else:
+                        part.fsuuid = '0x' + part.fsuuid.upper().rjust(8,"0")
+
+    def prepare(self, imager):
+        """Prepare an image. Call prepare method of all image partitions."""
+        for part in self.partitions:
+            # need to create the filesystems in order to get their
+            # sizes before we can add them and do the layout.
+            part.prepare(imager, imager.workdir, imager.oe_builddir,
+                         imager.rootfs_dir, imager.bootimg_dir,
+                         imager.kernel_dir, imager.native_sysroot,
+                         imager.updated_fstab_path)
+
+            # Converting kB to sectors for parted
+            part.size_sec = part.disk_size * 1024 // self.sector_size
+
+    def layout_partitions(self):
+        """ Layout the partitions, meaning calculate the position of every
+        partition on the disk. The 'ptable_format' parameter defines the
+        partition table format and may be "msdos". """
+
+        logger.debug("Assigning %s partitions to disks", self.ptable_format)
+
+        # The number of primary and logical partitions. Extended partition and
+        # partitions not listed in the table are not included.
+        num_real_partitions = len([p for p in self.partitions if not p.no_table])
+
+        # Go through partitions in the order they are added in .ks file
+        for num in range(len(self.partitions)):
+            part = self.partitions[num]
+
+            if self.ptable_format == 'msdos' and part.part_name:
+                raise WicError("setting custom partition name is not " \
+                               "implemented for msdos partitions")
+
+            if self.ptable_format == 'msdos' and part.part_type:
+                # The --part-type can also be implemented for MBR partitions,
+                # in which case it would map to the 1-byte "partition type"
+                # filed at offset 3 of the partition entry.
+                raise WicError("setting custom partition type is not " \
+                               "implemented for msdos partitions")
+
+            if part.mbr and self.ptable_format != 'gpt-hybrid':
+                raise WicError("Partition may only be included in MBR with " \
+                               "a gpt-hybrid partition table")
+
+            # Get the disk where the partition is located
+            self.numpart += 1
+            if not part.no_table:
+                self.realpart += 1
+
+            if self.numpart == 1:
+                if self.ptable_format == "msdos":
+                    overhead = MBR_OVERHEAD
+                elif self.ptable_format in ("gpt", "gpt-hybrid"):
+                    overhead = GPT_OVERHEAD
+
+                # Skip one sector required for the partitioning scheme overhead
+                self.offset += overhead
+
+            if self.ptable_format == "msdos":
+                if self.primary_part_num > 3 or \
+                   (self.extendedpart == 0 and self.primary_part_num >= 3 and num_real_partitions > 4):
+                    part.type = 'logical'
+                # Reserve a sector for EBR for every logical partition
+                # before alignment is performed.
+                if part.type == 'logical':
+                    self.offset += 2
+
+            align_sectors = 0
+            if part.align:
+                # If not first partition and we do have alignment set we need
+                # to align the partition.
+                # FIXME: This leaves a empty spaces to the disk. To fill the
+                # gaps we could enlargea the previous partition?
+
+                # Calc how much the alignment is off.
+                align_sectors = self.offset % (part.align * 1024 // self.sector_size)
+
+                if align_sectors:
+                    # If partition is not aligned as required, we need
+                    # to move forward to the next alignment point
+                    align_sectors = (part.align * 1024 // self.sector_size) - align_sectors
+
+                    logger.debug("Realignment for %s%s with %s sectors, original"
+                                 " offset %s, target alignment is %sK.",
+                                 part.disk, self.numpart, align_sectors,
+                                 self.offset, part.align)
+
+                    # increase the offset so we actually start the partition on right alignment
+                    self.offset += align_sectors
+
+            if part.offset is not None:
+                offset = part.offset // self.sector_size
+
+                if offset * self.sector_size != part.offset:
+                    raise WicError("Could not place %s%s at offset %d with sector size %d" % (part.disk, self.numpart, part.offset, self.sector_size))
+
+                delta = offset - self.offset
+                if delta < 0:
+                    raise WicError("Could not place %s%s at offset %d: next free sector is %d (delta: %d)" % (part.disk, self.numpart, part.offset, self.offset, delta))
+
+                logger.debug("Skipping %d sectors to place %s%s at offset %dK",
+                             delta, part.disk, self.numpart, part.offset)
+
+                self.offset = offset
+
+            part.start = self.offset
+            self.offset += part.size_sec
+
+            if not part.no_table:
+                part.num = self.realpart
+            else:
+                part.num = 0
+
+            if self.ptable_format == "msdos" and not part.no_table:
+                if part.type == 'logical':
+                    self.logical_part_cnt += 1
+                    part.num = self.logical_part_cnt + 4
+                    if self.extendedpart == 0:
+                        # Create extended partition as a primary partition
+                        self.primary_part_num += 1
+                        self.extendedpart = part.num
+                    else:
+                        self.extended_size_sec += align_sectors
+                    self.extended_size_sec += part.size_sec + 2
+                else:
+                    self.primary_part_num += 1
+                    part.num = self.primary_part_num
+
+            logger.debug("Assigned %s to %s%d, sectors range %d-%d size %d "
+                         "sectors (%d bytes).", part.mountpoint, part.disk,
+                         part.num, part.start, self.offset - 1, part.size_sec,
+                         part.size_sec * self.sector_size)
+
+        # Once all the partitions have been layed out, we can calculate the
+        # minumim disk size
+        self.min_size = self.offset
+        if self.ptable_format in ("gpt", "gpt-hybrid"):
+            self.min_size += GPT_OVERHEAD
+
+        self.min_size *= self.sector_size
+        self.min_size += self.extra_space
+
+    def _create_partition(self, device, parttype, fstype, start, size):
+        """ Create a partition on an image described by the 'device' object. """
+
+        # Start is included to the size so we need to substract one from the end.
+        end = start + size - 1
+        logger.debug("Added '%s' partition, sectors %d-%d, size %d sectors",
+                     parttype, start, end, size)
+
+        cmd = "parted -s %s unit s mkpart %s" % (device, parttype)
+        if fstype:
+            cmd += " %s" % fstype
+        cmd += " %d %d" % (start, end)
+
+        return exec_native_cmd(cmd, self.native_sysroot)
+
+    def _write_identifier(self, device, identifier):
+        logger.debug("Set disk identifier %x", identifier)
+        with open(device, 'r+b') as img:
+            img.seek(0x1B8)
+            img.write(identifier.to_bytes(4, 'little'))
+
+    def _make_disk(self, device, ptable_format, min_size):
+        logger.debug("Creating sparse file %s", device)
+        with open(device, 'w') as sparse:
+            os.ftruncate(sparse.fileno(), min_size)
+
+        logger.debug("Initializing partition table for %s", device)
+        exec_native_cmd("parted -s %s mklabel %s" % (device, ptable_format),
+                        self.native_sysroot)
+
+    def _write_disk_guid(self):
+        if self.ptable_format in ('gpt', 'gpt-hybrid'):
+            if os.getenv('SOURCE_DATE_EPOCH'):
+                self.disk_guid = uuid.UUID(int=int(os.getenv('SOURCE_DATE_EPOCH')))
+            else:
+                self.disk_guid = uuid.uuid4()
+
+            logger.debug("Set disk guid %s", self.disk_guid)
+            sfdisk_cmd = "sfdisk --disk-id %s %s" % (self.path, self.disk_guid)
+            exec_native_cmd(sfdisk_cmd, self.native_sysroot)
+
+    def create(self):
+        self._make_disk(self.path,
+                        "gpt" if self.ptable_format == "gpt-hybrid" else self.ptable_format,
+                        self.min_size)
+
+        self._write_identifier(self.path, self.identifier)
+        self._write_disk_guid()
+
+        if self.ptable_format == "gpt-hybrid":
+            mbr_path = self.path + ".mbr"
+            self._make_disk(mbr_path, "msdos", self.min_size)
+            self._write_identifier(mbr_path, self.identifier)
+
+        logger.debug("Creating partitions")
+
+        hybrid_mbr_part_num = 0
+
+        for part in self.partitions:
+            if part.num == 0:
+                continue
+
+            if self.ptable_format == "msdos" and part.num == self.extendedpart:
+                # Create an extended partition (note: extended
+                # partition is described in MBR and contains all
+                # logical partitions). The logical partitions save a
+                # sector for an EBR just before the start of a
+                # partition. The extended partition must start one
+                # sector before the start of the first logical
+                # partition. This way the first EBR is inside of the
+                # extended partition. Since the extended partitions
+                # starts a sector before the first logical partition,
+                # add a sector at the back, so that there is enough
+                # room for all logical partitions.
+                self._create_partition(self.path, "extended",
+                                       None, part.start - 2,
+                                       self.extended_size_sec)
+
+            if part.fstype == "swap":
+                parted_fs_type = "linux-swap"
+            elif part.fstype == "vfat":
+                parted_fs_type = "fat32"
+            elif part.fstype == "msdos":
+                parted_fs_type = "fat16"
+                if not part.system_id:
+                    part.system_id = '0x6' # FAT16
+            else:
+                # Type for ext2/ext3/ext4/btrfs
+                parted_fs_type = "ext2"
+
+            # Boot ROM of OMAP boards require vfat boot partition to have an
+            # even number of sectors.
+            if part.mountpoint == "/boot" and part.fstype in ["vfat", "msdos"] \
+               and part.size_sec % 2:
+                logger.debug("Subtracting one sector from '%s' partition to "
+                             "get even number of sectors for the partition",
+                             part.mountpoint)
+                part.size_sec -= 1
+
+            self._create_partition(self.path, part.type,
+                                   parted_fs_type, part.start, part.size_sec)
+
+            if self.ptable_format == "gpt-hybrid" and part.mbr:
+                hybrid_mbr_part_num += 1
+                if hybrid_mbr_part_num > 4:
+                    raise WicError("Extended MBR partitions are not supported in hybrid MBR")
+                self._create_partition(mbr_path, "primary",
+                                       parted_fs_type, part.start, part.size_sec)
+
+            if self.ptable_format in ("gpt", "gpt-hybrid") and (part.part_name or part.label):
+                partition_label = part.part_name if part.part_name else part.label
+                logger.debug("partition %d: set name to %s",
+                             part.num, partition_label)
+                exec_native_cmd("sgdisk --change-name=%d:%s %s" % \
+                                         (part.num, partition_label,
+                                          self.path), self.native_sysroot)
+
+            if part.part_type:
+                logger.debug("partition %d: set type UID to %s",
+                             part.num, part.part_type)
+                exec_native_cmd("sgdisk --typecode=%d:%s %s" % \
+                                         (part.num, part.part_type,
+                                          self.path), self.native_sysroot)
+
+            if part.uuid and self.ptable_format in ("gpt", "gpt-hybrid"):
+                logger.debug("partition %d: set UUID to %s",
+                             part.num, part.uuid)
+                exec_native_cmd("sgdisk --partition-guid=%d:%s %s" % \
+                                (part.num, part.uuid, self.path),
+                                self.native_sysroot)
+
+            if part.active:
+                flag_name = "legacy_boot" if self.ptable_format in ('gpt', 'gpt-hybrid') else "boot"
+                logger.debug("Set '%s' flag for partition '%s' on disk '%s'",
+                             flag_name, part.num, self.path)
+                exec_native_cmd("parted -s %s set %d %s on" % \
+                                (self.path, part.num, flag_name),
+                                self.native_sysroot)
+                if self.ptable_format == 'gpt-hybrid' and part.mbr:
+                    exec_native_cmd("parted -s %s set %d %s on" % \
+                                    (mbr_path, hybrid_mbr_part_num, "boot"),
+                                    self.native_sysroot)
+            if part.system_id:
+                exec_native_cmd("sfdisk --part-type %s %s %s" % \
+                                (self.path, part.num, part.system_id),
+                                self.native_sysroot)
+
+            if part.hidden and self.ptable_format == "gpt":
+                logger.debug("Set hidden attribute for partition '%s' on disk '%s'",
+                             part.num, self.path)
+                exec_native_cmd("sfdisk --part-attrs %s %s RequiredPartition" % \
+                                (self.path, part.num),
+                                self.native_sysroot)
+
+        if self.ptable_format == "gpt-hybrid":
+            # Write a protective GPT partition
+            hybrid_mbr_part_num += 1
+            if hybrid_mbr_part_num > 4:
+                raise WicError("Extended MBR partitions are not supported in hybrid MBR")
+
+            # parted cannot directly create a protective GPT partition, so
+            # create with an arbitrary type, then change it to the correct type
+            # with sfdisk
+            self._create_partition(mbr_path, "primary", "fat32", 1, GPT_OVERHEAD)
+            exec_native_cmd("sfdisk --part-type %s %d 0xee" % (mbr_path, hybrid_mbr_part_num),
+                            self.native_sysroot)
+
+            # Copy hybrid MBR
+            with open(mbr_path, "rb") as mbr_file:
+                with open(self.path, "r+b") as image_file:
+                    mbr = mbr_file.read(512)
+                    image_file.write(mbr)
+
+    def cleanup(self):
+        pass
+
+    def assemble(self):
+        logger.debug("Installing partitions")
+
+        for part in self.partitions:
+            source = part.source_file
+            if source:
+                # install source_file contents into a partition
+                sparse_copy(source, self.path, seek=part.start * self.sector_size)
+
+                logger.debug("Installed %s in partition %d, sectors %d-%d, "
+                             "size %d sectors", source, part.num, part.start,
+                             part.start + part.size_sec - 1, part.size_sec)
+
+                partimage = self.path + '.p%d' % part.num
+                os.rename(source, partimage)
+                self.partimages.append(partimage)

sources/poky/scripts/lib/wic/plugins/source/bootimg-biosplusefi.py

@@ -0,0 +1,213 @@
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# DESCRIPTION
+# This implements the 'bootimg-biosplusefi' source plugin class for 'wic'
+#
+# AUTHORS
+# William Bourque <wbourque [at) gmail.com>
+
+import types
+
+from wic.pluginbase import SourcePlugin
+from importlib.machinery import SourceFileLoader
+
+class BootimgBiosPlusEFIPlugin(SourcePlugin):
+    """
+    Create MBR + EFI boot partition
+
+    This plugin creates a boot partition that contains both
+    legacy BIOS and EFI content. It will be able to boot from both.
+    This is useful when managing PC fleet with some older machines
+    without EFI support.
+
+    Note it is possible to create an image that can boot from both
+    legacy BIOS and EFI by defining two partitions : one with arg
+    --source bootimg-efi  and another one with --source bootimg-pcbios.
+    However, this method has the obvious downside that it requires TWO
+    partitions to be created on the storage device.
+    Both partitions will also be marked as "bootable" which does not work on
+    most BIOS, has BIOS often uses the "bootable" flag to determine
+    what to boot. If you have such a BIOS, you need to manually remove the
+    "bootable" flag from the EFI partition for the drive to be bootable.
+    Having two partitions also seems to confuse wic : the content of
+    the first partition will be duplicated into the second, even though it
+    will not be used at all.
+
+    Also, unlike "isoimage-isohybrid" that also does BIOS and EFI, this plugin
+    allows you to have more than only a single rootfs partitions and does
+    not turn the rootfs into an initramfs RAM image.
+
+    This plugin is made to put everything into a single /boot partition so it
+    does not have the limitations listed above.
+
+    The plugin is made so it does tries not to reimplement what's already
+    been done in other plugins; as such it imports "bootimg-pcbios"
+    and "bootimg-efi".
+    Plugin "bootimg-pcbios" is used to generate legacy BIOS boot.
+    Plugin "bootimg-efi" is used to generate the UEFI boot. Note that it
+    requires a --sourceparams argument to know which loader to use; refer
+    to "bootimg-efi" code/documentation for the list of loader.
+
+    Imports are handled with "SourceFileLoader" from importlib as it is
+    otherwise very difficult to import module that has hyphen "-" in their
+    filename.
+    The SourcePlugin() methods used in the plugins (do_install_disk,
+    do_configure_partition, do_prepare_partition) are then called on both,
+    beginning by "bootimg-efi".
+
+    Plugin options, such as "--sourceparams" can still be passed to a
+    plugin, as long they does not cause issue in the other plugin.
+
+    Example wic configuration:
+    part /boot --source bootimg-biosplusefi --sourceparams="loader=grub-efi"\\
+               --ondisk sda --label os_boot --active --align 1024 --use-uuid
+    """
+
+    name = 'bootimg-biosplusefi'
+
+    __PCBIOS_MODULE_NAME = "bootimg-pcbios"
+    __EFI_MODULE_NAME = "bootimg-efi"
+
+    __imgEFIObj = None
+    __imgBiosObj = None
+
+    @classmethod
+    def __init__(cls):
+        """
+        Constructor (init)
+        """
+
+        # XXX
+        # For some reasons, __init__ constructor is never called.
+        # Something to do with how pluginbase works?
+        cls.__instanciateSubClasses()
+
+    @classmethod
+    def __instanciateSubClasses(cls):
+        """
+
+        """
+
+        # Import bootimg-pcbios (class name "BootimgPcbiosPlugin")
+        modulePath = os.path.join(os.path.dirname(os.path.realpath(__file__)),
+                                  cls.__PCBIOS_MODULE_NAME + ".py")
+        loader = SourceFileLoader(cls.__PCBIOS_MODULE_NAME, modulePath)
+        mod = types.ModuleType(loader.name)
+        loader.exec_module(mod)
+        cls.__imgBiosObj = mod.BootimgPcbiosPlugin()
+
+        # Import bootimg-efi (class name "BootimgEFIPlugin")
+        modulePath = os.path.join(os.path.dirname(os.path.realpath(__file__)),
+                                  cls.__EFI_MODULE_NAME + ".py")
+        loader = SourceFileLoader(cls.__EFI_MODULE_NAME, modulePath)
+        mod = types.ModuleType(loader.name)
+        loader.exec_module(mod)
+        cls.__imgEFIObj = mod.BootimgEFIPlugin()
+
+    @classmethod
+    def do_install_disk(cls, disk, disk_name, creator, workdir, oe_builddir,
+                        bootimg_dir, kernel_dir, native_sysroot):
+        """
+        Called after all partitions have been prepared and assembled into a
+        disk image.
+        """
+
+        if ( (not cls.__imgEFIObj) or (not cls.__imgBiosObj) ):
+            cls.__instanciateSubClasses()
+
+        cls.__imgEFIObj.do_install_disk(
+            disk,
+            disk_name,
+            creator,
+            workdir,
+            oe_builddir,
+            bootimg_dir,
+            kernel_dir,
+            native_sysroot)
+
+        cls.__imgBiosObj.do_install_disk(
+            disk,
+            disk_name,
+            creator,
+            workdir,
+            oe_builddir,
+            bootimg_dir,
+            kernel_dir,
+            native_sysroot)
+
+    @classmethod
+    def do_configure_partition(cls, part, source_params, creator, cr_workdir,
+                               oe_builddir, bootimg_dir, kernel_dir,
+                               native_sysroot):
+        """
+        Called before do_prepare_partition()
+        """
+
+        if ( (not cls.__imgEFIObj) or (not cls.__imgBiosObj) ):
+            cls.__instanciateSubClasses()
+
+        cls.__imgEFIObj.do_configure_partition(
+            part,
+            source_params,
+            creator,
+            cr_workdir,
+            oe_builddir,
+            bootimg_dir,
+            kernel_dir,
+            native_sysroot)
+
+        cls.__imgBiosObj.do_configure_partition(
+            part,
+            source_params,
+            creator,
+            cr_workdir,
+            oe_builddir,
+            bootimg_dir,
+            kernel_dir,
+            native_sysroot)
+
+    @classmethod
+    def do_prepare_partition(cls, part, source_params, creator, cr_workdir,
+                             oe_builddir, bootimg_dir, kernel_dir,
+                             rootfs_dir, native_sysroot):
+        """
+        Called to do the actual content population for a partition i.e. it
+        'prepares' the partition to be incorporated into the image.
+        """
+
+        if ( (not cls.__imgEFIObj) or (not cls.__imgBiosObj) ):
+            cls.__instanciateSubClasses()
+
+        cls.__imgEFIObj.do_prepare_partition(
+            part,
+            source_params,
+            creator,
+            cr_workdir,
+            oe_builddir,
+            bootimg_dir,
+            kernel_dir,
+            rootfs_dir,
+            native_sysroot)
+
+        cls.__imgBiosObj.do_prepare_partition(
+            part,
+            source_params,
+            creator,
+            cr_workdir,
+            oe_builddir,
+            bootimg_dir,
+            kernel_dir,
+            rootfs_dir,
+            native_sysroot)

sources/poky/scripts/lib/wic/plugins/source/bootimg-efi.py

@@ -0,0 +1,507 @@
+#
+# Copyright (c) 2014, Intel Corporation.
+#
+# SPDX-License-Identifier: GPL-2.0-only
+#
+# DESCRIPTION
+# This implements the 'bootimg-efi' source plugin class for 'wic'
+#
+# AUTHORS
+# Tom Zanussi <tom.zanussi (at] linux.intel.com>
+#
+
+import logging
+import os
+import tempfile
+import shutil
+import re
+
+from glob import glob
+
+from wic import WicError
+from wic.engine import get_custom_config
+from wic.pluginbase import SourcePlugin
+from wic.misc import (exec_cmd, exec_native_cmd,
+                      get_bitbake_var, BOOTDD_EXTRA_SPACE)
+
+logger = logging.getLogger('wic')
+
+class BootimgEFIPlugin(SourcePlugin):
+    """
+    Create EFI boot partition.
+    This plugin supports GRUB 2 and systemd-boot bootloaders.
+    """
+
+    name = 'bootimg-efi'
+
+    @classmethod
+    def _copy_additional_files(cls, hdddir, initrd, dtb):
+        bootimg_dir = get_bitbake_var("DEPLOY_DIR_IMAGE")
+        if not bootimg_dir:
+            raise WicError("Couldn't find DEPLOY_DIR_IMAGE, exiting")
+
+        if initrd:
+            initrds = initrd.split(';')
+            for rd in initrds:
+                cp_cmd = "cp %s/%s %s" % (bootimg_dir, rd, hdddir)
+                exec_cmd(cp_cmd, True)
+        else:
+            logger.debug("Ignoring missing initrd")
+
+        if dtb:
+            if ';' in dtb:
+                raise WicError("Only one DTB supported, exiting")
+            cp_cmd = "cp %s/%s %s" % (bootimg_dir, dtb, hdddir)
+            exec_cmd(cp_cmd, True)
+
+    @classmethod
+    def do_configure_grubefi(cls, hdddir, creator, cr_workdir, source_params):
+        """
+        Create loader-specific (grub-efi) config
+        """
+        configfile = creator.ks.bootloader.configfile
+        custom_cfg = None
+        if configfile:
+            custom_cfg = get_custom_config(configfile)
+            if custom_cfg:
+                # Use a custom configuration for grub
+                grubefi_conf = custom_cfg
+                logger.debug("Using custom configuration file "
+                             "%s for grub.cfg", configfile)
+            else:
+                raise WicError("configfile is specified but failed to "
+                               "get it from %s." % configfile)
+
+        initrd = source_params.get('initrd')
+        dtb = source_params.get('dtb')
+
+        cls._copy_additional_files(hdddir, initrd, dtb)
+
+        if not custom_cfg:
+            # Create grub configuration using parameters from wks file
+            bootloader = creator.ks.bootloader
+            title = source_params.get('title')
+
+            grubefi_conf = ""
+            grubefi_conf += "serial --unit=0 --speed=115200 --word=8 --parity=no --stop=1\n"
+            grubefi_conf += "default=boot\n"
+            grubefi_conf += "timeout=%s\n" % bootloader.timeout
+            grubefi_conf += "menuentry '%s'{\n" % (title if title else "boot")
+
+            kernel = get_bitbake_var("KERNEL_IMAGETYPE")
+            if get_bitbake_var("INITRAMFS_IMAGE_BUNDLE") == "1":
+                if get_bitbake_var("INITRAMFS_IMAGE"):
+                    kernel = "%s-%s.bin" % \
+                        (get_bitbake_var("KERNEL_IMAGETYPE"), get_bitbake_var("INITRAMFS_LINK_NAME"))
+
+            label = source_params.get('label')
+            label_conf = "root=%s" % creator.rootdev
+            if label:
+                label_conf = "LABEL=%s" % label
+
+            grubefi_conf += "linux /%s %s rootwait %s\n" \
+                % (kernel, label_conf, bootloader.append)
+
+            if initrd:
+                initrds = initrd.split(';')
+                grubefi_conf += "initrd"
+                for rd in initrds:
+                    grubefi_conf += " /%s" % rd
+                grubefi_conf += "\n"
+
+            if dtb:
+                grubefi_conf += "devicetree /%s\n" % dtb
+
+            grubefi_conf += "}\n"
+
+        logger.debug("Writing grubefi config %s/hdd/boot/EFI/BOOT/grub.cfg",
+                     cr_workdir)
+        cfg = open("%s/hdd/boot/EFI/BOOT/grub.cfg" % cr_workdir, "w")
+        cfg.write(grubefi_conf)
+        cfg.close()
+
+    @classmethod
+    def do_configure_systemdboot(cls, hdddir, creator, cr_workdir, source_params):
+        """
+        Create loader-specific systemd-boot/gummiboot config
+        """
+        install_cmd = "install -d %s/loader" % hdddir
+        exec_cmd(install_cmd)
+
+        install_cmd = "install -d %s/loader/entries" % hdddir
+        exec_cmd(install_cmd)
+
+        bootloader = creator.ks.bootloader
+
+        unified_image = source_params.get('create-unified-kernel-image') == "true"
+
+        loader_conf = ""
+        if not unified_image:
+            loader_conf += "default boot\n"
+        loader_conf += "timeout %d\n" % bootloader.timeout
+
+        initrd = source_params.get('initrd')
+        dtb = source_params.get('dtb')
+
+        if not unified_image:
+            cls._copy_additional_files(hdddir, initrd, dtb)
+
+        logger.debug("Writing systemd-boot config "
+                     "%s/hdd/boot/loader/loader.conf", cr_workdir)
+        cfg = open("%s/hdd/boot/loader/loader.conf" % cr_workdir, "w")
+        cfg.write(loader_conf)
+        cfg.close()
+
+        configfile = creator.ks.bootloader.configfile
+        custom_cfg = None
+        if configfile:
+            custom_cfg = get_custom_config(configfile)
+            if custom_cfg:
+                # Use a custom configuration for systemd-boot
+                boot_conf = custom_cfg
+                logger.debug("Using custom configuration file "
+                             "%s for systemd-boots's boot.conf", configfile)
+            else:
+                raise WicError("configfile is specified but failed to "
+                               "get it from %s.", configfile)
+
+        if not custom_cfg:
+            # Create systemd-boot configuration using parameters from wks file
+            kernel = get_bitbake_var("KERNEL_IMAGETYPE")
+            if get_bitbake_var("INITRAMFS_IMAGE_BUNDLE") == "1":
+                if get_bitbake_var("INITRAMFS_IMAGE"):
+                    kernel = "%s-%s.bin" % \
+                        (get_bitbake_var("KERNEL_IMAGETYPE"), get_bitbake_var("INITRAMFS_LINK_NAME"))
+
+            title = source_params.get('title')
+
+            boot_conf = ""
+            boot_conf += "title %s\n" % (title if title else "boot")
+            boot_conf += "linux /%s\n" % kernel
+
+            label = source_params.get('label')
+            label_conf = "LABEL=Boot root=%s" % creator.rootdev
+            if label:
+                label_conf = "LABEL=%s" % label
+
+            boot_conf += "options %s %s\n" % \
+                             (label_conf, bootloader.append)
+
+            if initrd:
+                initrds = initrd.split(';')
+                for rd in initrds:
+                    boot_conf += "initrd /%s\n" % rd
+
+            if dtb:
+                boot_conf += "devicetree /%s\n" % dtb
+
+        if not unified_image:
+            logger.debug("Writing systemd-boot config "
+                         "%s/hdd/boot/loader/entries/boot.conf", cr_workdir)
+            cfg = open("%s/hdd/boot/loader/entries/boot.conf" % cr_workdir, "w")
+            cfg.write(boot_conf)
+            cfg.close()
+
+
+    @classmethod
+    def do_configure_partition(cls, part, source_params, creator, cr_workdir,
+                               oe_builddir, bootimg_dir, kernel_dir,
+                               native_sysroot):
+        """
+        Called before do_prepare_partition(), creates loader-specific config
+        """
+        hdddir = "%s/hdd/boot" % cr_workdir
+
+        install_cmd = "install -d %s/EFI/BOOT" % hdddir
+        exec_cmd(install_cmd)
+
+        try:
+            if source_params['loader'] == 'grub-efi':
+                cls.do_configure_grubefi(hdddir, creator, cr_workdir, source_params)
+            elif source_params['loader'] == 'systemd-boot':
+                cls.do_configure_systemdboot(hdddir, creator, cr_workdir, source_params)
+            elif source_params['loader'] == 'uefi-kernel':
+                pass
+            else:
+                raise WicError("unrecognized bootimg-efi loader: %s" % source_params['loader'])
+        except KeyError:
+            raise WicError("bootimg-efi requires a loader, none specified")
+
+        if get_bitbake_var("IMAGE_EFI_BOOT_FILES") is None:
+            logger.debug('No boot files defined in IMAGE_EFI_BOOT_FILES')
+        else:
+            boot_files = None
+            for (fmt, id) in (("_uuid-%s", part.uuid), ("_label-%s", part.label), (None, None)):
+                if fmt:
+                    var = fmt % id
+                else:
+                    var = ""
+
+                boot_files = get_bitbake_var("IMAGE_EFI_BOOT_FILES" + var)
+                if boot_files:
+                    break
+
+            logger.debug('Boot files: %s', boot_files)
+
+            # list of tuples (src_name, dst_name)
+            deploy_files = []
+            for src_entry in re.findall(r'[\w;\-\.\+/\*]+', boot_files):
+                if ';' in src_entry:
+                    dst_entry = tuple(src_entry.split(';'))
+                    if not dst_entry[0] or not dst_entry[1]:
+                        raise WicError('Malformed boot file entry: %s' % src_entry)
+                else:
+                    dst_entry = (src_entry, src_entry)
+
+                logger.debug('Destination entry: %r', dst_entry)
+                deploy_files.append(dst_entry)
+
+            cls.install_task = [];
+            for deploy_entry in deploy_files:
+                src, dst = deploy_entry
+                if '*' in src:
+                    # by default install files under their basename
+                    entry_name_fn = os.path.basename
+                    if dst != src:
+                        # unless a target name was given, then treat name
+                        # as a directory and append a basename
+                        entry_name_fn = lambda name: \
+                                        os.path.join(dst,
+                                                     os.path.basename(name))
+
+                    srcs = glob(os.path.join(kernel_dir, src))
+
+                    logger.debug('Globbed sources: %s', ', '.join(srcs))
+                    for entry in srcs:
+                        src = os.path.relpath(entry, kernel_dir)
+                        entry_dst_name = entry_name_fn(entry)
+                        cls.install_task.append((src, entry_dst_name))
+                else:
+                    cls.install_task.append((src, dst))
+
+    @classmethod
+    def do_prepare_partition(cls, part, source_params, creator, cr_workdir,
+                             oe_builddir, bootimg_dir, kernel_dir,
+                             rootfs_dir, native_sysroot):
+        """
+        Called to do the actual content population for a partition i.e. it
+        'prepares' the partition to be incorporated into the image.
+        In this case, prepare content for an EFI (grub) boot partition.
+        """
+        if not kernel_dir:
+            kernel_dir = get_bitbake_var("DEPLOY_DIR_IMAGE")
+            if not kernel_dir:
+                raise WicError("Couldn't find DEPLOY_DIR_IMAGE, exiting")
+
+        staging_kernel_dir = kernel_dir
+
+        hdddir = "%s/hdd/boot" % cr_workdir
+
+        kernel = get_bitbake_var("KERNEL_IMAGETYPE")
+        if get_bitbake_var("INITRAMFS_IMAGE_BUNDLE") == "1":
+            if get_bitbake_var("INITRAMFS_IMAGE"):
+                kernel = "%s-%s.bin" % \
+                    (get_bitbake_var("KERNEL_IMAGETYPE"), get_bitbake_var("INITRAMFS_LINK_NAME"))
+
+        if source_params.get('create-unified-kernel-image') == "true":
+            initrd = source_params.get('initrd')
+            if not initrd:
+                raise WicError("initrd= must be specified when create-unified-kernel-image=true, exiting")
+
+            deploy_dir = get_bitbake_var("DEPLOY_DIR_IMAGE")
+            efi_stub = glob("%s/%s" % (deploy_dir, "linux*.efi.stub"))
+            if len(efi_stub) == 0:
+                raise WicError("Unified Kernel Image EFI stub not found, exiting")
+            efi_stub = efi_stub[0]
+
+            with tempfile.TemporaryDirectory() as tmp_dir:
+                label = source_params.get('label')
+                label_conf = "root=%s" % creator.rootdev
+                if label:
+                    label_conf = "LABEL=%s" % label
+
+                bootloader = creator.ks.bootloader
+                cmdline = open("%s/cmdline" % tmp_dir, "w")
+                cmdline.write("%s %s" % (label_conf, bootloader.append))
+                cmdline.close()
+
+                initrds = initrd.split(';')
+                initrd = open("%s/initrd" % tmp_dir, "wb")
+                for f in initrds:
+                    with open("%s/%s" % (deploy_dir, f), 'rb') as in_file:
+                        shutil.copyfileobj(in_file, initrd)
+                initrd.close()
+
+                # Searched by systemd-boot:
+                # https://systemd.io/BOOT_LOADER_SPECIFICATION/#type-2-efi-unified-kernel-images
+                install_cmd = "install -d %s/EFI/Linux" % hdddir
+                exec_cmd(install_cmd)
+
+                staging_dir_host = get_bitbake_var("STAGING_DIR_HOST")
+                target_sys = get_bitbake_var("TARGET_SYS")
+
+                objdump_cmd = "%s-objdump" % target_sys
+                objdump_cmd += " -p %s" % efi_stub
+                objdump_cmd += " | awk '{ if ($1 == \"SectionAlignment\"){print $2} }'"
+
+                ret, align_str = exec_native_cmd(objdump_cmd, native_sysroot)
+                align = int(align_str, 16)
+
+                objdump_cmd = "%s-objdump" % target_sys
+                objdump_cmd += " -h %s | tail -2" % efi_stub
+                ret, output = exec_native_cmd(objdump_cmd, native_sysroot)
+
+                offset = int(output.split()[2], 16) + int(output.split()[3], 16)
+
+                osrel_off = offset + align - offset % align
+                osrel_path = "%s/usr/lib/os-release" % staging_dir_host
+                osrel_sz = os.stat(osrel_path).st_size
+
+                cmdline_off = osrel_off + osrel_sz
+                cmdline_off = cmdline_off + align - cmdline_off % align
+                cmdline_sz = os.stat(cmdline.name).st_size
+
+                dtb_off = cmdline_off + cmdline_sz
+                dtb_off = dtb_off + align - dtb_off % align
+
+                dtb = source_params.get('dtb')
+                if dtb:
+                    if ';' in dtb:
+                        raise WicError("Only one DTB supported, exiting")
+                    dtb_path = "%s/%s" % (deploy_dir, dtb)
+                    dtb_params = '--add-section .dtb=%s --change-section-vma .dtb=0x%x' % \
+                            (dtb_path, dtb_off)
+                    linux_off = dtb_off + os.stat(dtb_path).st_size
+                    linux_off = linux_off + align - linux_off % align
+                else:
+                    dtb_params = ''
+                    linux_off = dtb_off
+
+                linux_path = "%s/%s" % (staging_kernel_dir, kernel)
+                linux_sz = os.stat(linux_path).st_size
+
+                initrd_off = linux_off + linux_sz
+                initrd_off = initrd_off + align - initrd_off % align
+
+                # https://www.freedesktop.org/software/systemd/man/systemd-stub.html
+                objcopy_cmd = "%s-objcopy" % target_sys
+                objcopy_cmd += " --enable-deterministic-archives"
+                objcopy_cmd += " --preserve-dates"
+                objcopy_cmd += " --add-section .osrel=%s" % osrel_path
+                objcopy_cmd += " --change-section-vma .osrel=0x%x" % osrel_off
+                objcopy_cmd += " --add-section .cmdline=%s" % cmdline.name
+                objcopy_cmd += " --change-section-vma .cmdline=0x%x" % cmdline_off
+                objcopy_cmd += dtb_params
+                objcopy_cmd += " --add-section .linux=%s" % linux_path
+                objcopy_cmd += " --change-section-vma .linux=0x%x" % linux_off
+                objcopy_cmd += " --add-section .initrd=%s" % initrd.name
+                objcopy_cmd += " --change-section-vma .initrd=0x%x" % initrd_off
+                objcopy_cmd += " %s %s/EFI/Linux/linux.efi" % (efi_stub, hdddir)
+
+                exec_native_cmd(objcopy_cmd, native_sysroot)
+        else:
+            if source_params.get('install-kernel-into-boot-dir') != 'false':
+                install_cmd = "install -m 0644 %s/%s %s/%s" % \
+                    (staging_kernel_dir, kernel, hdddir, kernel)
+                exec_cmd(install_cmd)
+
+        if get_bitbake_var("IMAGE_EFI_BOOT_FILES"):
+            for src_path, dst_path in cls.install_task:
+                install_cmd = "install -m 0644 -D %s %s" \
+                              % (os.path.join(kernel_dir, src_path),
+                                 os.path.join(hdddir, dst_path))
+                exec_cmd(install_cmd)
+
+        try:
+            if source_params['loader'] == 'grub-efi':
+                shutil.copyfile("%s/hdd/boot/EFI/BOOT/grub.cfg" % cr_workdir,
+                                "%s/grub.cfg" % cr_workdir)
+                for mod in [x for x in os.listdir(kernel_dir) if x.startswith("grub-efi-")]:
+                    cp_cmd = "cp %s/%s %s/EFI/BOOT/%s" % (kernel_dir, mod, hdddir, mod[9:])
+                    exec_cmd(cp_cmd, True)
+                shutil.move("%s/grub.cfg" % cr_workdir,
+                            "%s/hdd/boot/EFI/BOOT/grub.cfg" % cr_workdir)
+            elif source_params['loader'] == 'systemd-boot':
+                for mod in [x for x in os.listdir(kernel_dir) if x.startswith("systemd-")]:
+                    cp_cmd = "cp %s/%s %s/EFI/BOOT/%s" % (kernel_dir, mod, hdddir, mod[8:])
+                    exec_cmd(cp_cmd, True)
+            elif source_params['loader'] == 'uefi-kernel':
+                kernel = get_bitbake_var("KERNEL_IMAGETYPE")
+                if not kernel:
+                    raise WicError("Empty KERNEL_IMAGETYPE")
+                target = get_bitbake_var("TARGET_SYS")
+                if not target:
+                    raise WicError("Empty TARGET_SYS")
+
+                if re.match("x86_64", target):
+                    kernel_efi_image = "bootx64.efi"
+                elif re.match('i.86', target):
+                    kernel_efi_image = "bootia32.efi"
+                elif re.match('aarch64', target):
+                    kernel_efi_image = "bootaa64.efi"
+                elif re.match('arm', target):
+                    kernel_efi_image = "bootarm.efi"
+                else:
+                    raise WicError("UEFI stub kernel is incompatible with target %s" % target)
+
+                for mod in [x for x in os.listdir(kernel_dir) if x.startswith(kernel)]:
+                    cp_cmd = "cp %s/%s %s/EFI/BOOT/%s" % (kernel_dir, mod, hdddir, kernel_efi_image)
+                    exec_cmd(cp_cmd, True)
+            else:
+                raise WicError("unrecognized bootimg-efi loader: %s" %
+                               source_params['loader'])
+        except KeyError:
+            raise WicError("bootimg-efi requires a loader, none specified")
+
+        startup = os.path.join(kernel_dir, "startup.nsh")
+        if os.path.exists(startup):
+            cp_cmd = "cp %s %s/" % (startup, hdddir)
+            exec_cmd(cp_cmd, True)
+
+        for paths in part.include_path or []:
+            for path in paths:
+                cp_cmd = "cp -r %s %s/" % (path, hdddir)
+                exec_cmd(cp_cmd, True)
+
+        du_cmd = "du -bks %s" % hdddir
+        out = exec_cmd(du_cmd)
+        blocks = int(out.split()[0])
+
+        extra_blocks = part.get_extra_block_count(blocks)
+
+        if extra_blocks < BOOTDD_EXTRA_SPACE:
+            extra_blocks = BOOTDD_EXTRA_SPACE
+
+        blocks += extra_blocks
+
+        logger.debug("Added %d extra blocks to %s to get to %d total blocks",
+                     extra_blocks, part.mountpoint, blocks)
+
+        # required for compatibility with certain devices expecting file system
+        # block count to be equal to partition block count
+        if blocks < part.fixed_size:
+            blocks = part.fixed_size
+            logger.debug("Overriding %s to %d total blocks for compatibility",
+                     part.mountpoint, blocks)
+
+        # dosfs image, created by mkdosfs
+        bootimg = "%s/boot.img" % cr_workdir
+
+        label = part.label if part.label else "ESP"
+
+        dosfs_cmd = "mkdosfs -n %s -i %s -C %s %d" % \
+                    (label, part.fsuuid, bootimg, blocks)
+        exec_native_cmd(dosfs_cmd, native_sysroot)
+
+        mcopy_cmd = "mcopy -i %s -s %s/* ::/" % (bootimg, hdddir)
+        exec_native_cmd(mcopy_cmd, native_sysroot)
+
+        chmod_cmd = "chmod 644 %s" % bootimg
+        exec_cmd(chmod_cmd)
+
+        du_cmd = "du -Lbks %s" % bootimg
+        out = exec_cmd(du_cmd)
+        bootimg_size = out.split()[0]
+
+        part.size = int(bootimg_size)
+        part.source_file = bootimg

sources/poky/scripts/lib/wic/plugins/source/bootimg-partition.py

@@ -0,0 +1,197 @@
+#
+# Copyright OpenEmbedded Contributors
+#
+# SPDX-License-Identifier: GPL-2.0-only
+#
+# DESCRIPTION
+# This implements the 'bootimg-partition' source plugin class for
+# 'wic'. The plugin creates an image of boot partition, copying over
+# files listed in IMAGE_BOOT_FILES bitbake variable.
+#
+# AUTHORS
+# Maciej Borzecki <maciej.borzecki (at] open-rnd.pl>
+#
+
+import logging
+import os
+import re
+
+from glob import glob
+
+from wic import WicError
+from wic.engine import get_custom_config
+from wic.pluginbase import SourcePlugin
+from wic.misc import exec_cmd, get_bitbake_var
+
+logger = logging.getLogger('wic')
+
+class BootimgPartitionPlugin(SourcePlugin):
+    """
+    Create an image of boot partition, copying over files
+    listed in IMAGE_BOOT_FILES bitbake variable.
+    """
+
+    name = 'bootimg-partition'
+    image_boot_files_var_name = 'IMAGE_BOOT_FILES'
+
+    @classmethod
+    def do_configure_partition(cls, part, source_params, cr, cr_workdir,
+                             oe_builddir, bootimg_dir, kernel_dir,
+                             native_sysroot):
+        """
+        Called before do_prepare_partition(), create u-boot specific boot config
+        """
+        hdddir = "%s/boot.%d" % (cr_workdir, part.lineno)
+        install_cmd = "install -d %s" % hdddir
+        exec_cmd(install_cmd)
+
+        if not kernel_dir:
+            kernel_dir = get_bitbake_var("DEPLOY_DIR_IMAGE")
+            if not kernel_dir:
+                raise WicError("Couldn't find DEPLOY_DIR_IMAGE, exiting")
+
+        boot_files = None
+        for (fmt, id) in (("_uuid-%s", part.uuid), ("_label-%s", part.label), (None, None)):
+            if fmt:
+                var = fmt % id
+            else:
+                var = ""
+
+            boot_files = get_bitbake_var(cls.image_boot_files_var_name + var)
+            if boot_files is not None:
+                break
+
+        if boot_files is None:
+            raise WicError('No boot files defined, %s unset for entry #%d' % (cls.image_boot_files_var_name, part.lineno))
+
+        logger.debug('Boot files: %s', boot_files)
+
+        # list of tuples (src_name, dst_name)
+        deploy_files = []
+        for src_entry in re.findall(r'[\w;\-\./\*]+', boot_files):
+            if ';' in src_entry:
+                dst_entry = tuple(src_entry.split(';'))
+                if not dst_entry[0] or not dst_entry[1]:
+                    raise WicError('Malformed boot file entry: %s' % src_entry)
+            else:
+                dst_entry = (src_entry, src_entry)
+
+            logger.debug('Destination entry: %r', dst_entry)
+            deploy_files.append(dst_entry)
+
+        cls.install_task = [];
+        for deploy_entry in deploy_files:
+            src, dst = deploy_entry
+            if '*' in src:
+                # by default install files under their basename
+                entry_name_fn = os.path.basename
+                if dst != src:
+                    # unless a target name was given, then treat name
+                    # as a directory and append a basename
+                    entry_name_fn = lambda name: \
+                                    os.path.join(dst,
+                                                 os.path.basename(name))
+
+                srcs = glob(os.path.join(kernel_dir, src))
+
+                logger.debug('Globbed sources: %s', ', '.join(srcs))
+                for entry in srcs:
+                    src = os.path.relpath(entry, kernel_dir)
+                    entry_dst_name = entry_name_fn(entry)
+                    cls.install_task.append((src, entry_dst_name))
+            else:
+                cls.install_task.append((src, dst))
+
+        if source_params.get('loader') != "u-boot":
+            return
+
+        configfile = cr.ks.bootloader.configfile
+        custom_cfg = None
+        if configfile:
+            custom_cfg = get_custom_config(configfile)
+            if custom_cfg:
+                # Use a custom configuration for extlinux.conf
+                extlinux_conf = custom_cfg
+                logger.debug("Using custom configuration file "
+                             "%s for extlinux.conf", configfile)
+            else:
+                raise WicError("configfile is specified but failed to "
+                               "get it from %s." % configfile)
+
+        if not custom_cfg:
+            # The kernel types supported by the sysboot of u-boot
+            kernel_types = ["zImage", "Image", "fitImage", "uImage", "vmlinux"]
+            has_dtb = False
+            fdt_dir = '/'
+            kernel_name = None
+
+            # Find the kernel image name, from the highest precedence to lowest
+            for image in kernel_types:
+                for task in cls.install_task:
+                    src, dst = task
+                    if re.match(image, src):
+                        kernel_name = os.path.join('/', dst)
+                        break
+                if kernel_name:
+                    break
+
+            for task in cls.install_task:
+                src, dst = task
+                # We suppose that all the dtb are in the same directory
+                if re.search(r'\.dtb', src) and fdt_dir == '/':
+                    has_dtb = True
+                    fdt_dir = os.path.join(fdt_dir, os.path.dirname(dst))
+                    break
+
+            if not kernel_name:
+                raise WicError('No kernel file found')
+
+            # Compose the extlinux.conf
+            extlinux_conf = "default Yocto\n"
+            extlinux_conf += "label Yocto\n"
+            extlinux_conf += "   kernel %s\n" % kernel_name
+            if has_dtb:
+                extlinux_conf += "   fdtdir %s\n" % fdt_dir
+            bootloader = cr.ks.bootloader
+            extlinux_conf += "append root=%s rootwait %s\n" \
+                             % (cr.rootdev, bootloader.append if bootloader.append else '')
+
+        install_cmd = "install -d %s/extlinux/" % hdddir
+        exec_cmd(install_cmd)
+        cfg = open("%s/extlinux/extlinux.conf" % hdddir, "w")
+        cfg.write(extlinux_conf)
+        cfg.close()
+
+
+    @classmethod
+    def do_prepare_partition(cls, part, source_params, cr, cr_workdir,
+                             oe_builddir, bootimg_dir, kernel_dir,
+                             rootfs_dir, native_sysroot):
+        """
+        Called to do the actual content population for a partition i.e. it
+        'prepares' the partition to be incorporated into the image.
+        In this case, does the following:
+        - sets up a vfat partition
+        - copies all files listed in IMAGE_BOOT_FILES variable
+        """
+        hdddir = "%s/boot.%d" % (cr_workdir, part.lineno)
+
+        if not kernel_dir:
+            kernel_dir = get_bitbake_var("DEPLOY_DIR_IMAGE")
+            if not kernel_dir:
+                raise WicError("Couldn't find DEPLOY_DIR_IMAGE, exiting")
+
+        logger.debug('Kernel dir: %s', bootimg_dir)
+
+
+        for task in cls.install_task:
+            src_path, dst_path = task
+            logger.debug('Install %s as %s', src_path, dst_path)
+            install_cmd = "install -m 0644 -D %s %s" \
+                          % (os.path.join(kernel_dir, src_path),
+                             os.path.join(hdddir, dst_path))
+            exec_cmd(install_cmd)
+
+        logger.debug('Prepare boot partition using rootfs in %s', hdddir)
+        part.prepare_rootfs(cr_workdir, oe_builddir, hdddir,
+                            native_sysroot, False)

sources/poky/scripts/lib/wic/plugins/source/bootimg-pcbios.py

@@ -0,0 +1,209 @@
+#
+# Copyright (c) 2014, Intel Corporation.
+#
+# SPDX-License-Identifier: GPL-2.0-only
+#
+# DESCRIPTION
+# This implements the 'bootimg-pcbios' source plugin class for 'wic'
+#
+# AUTHORS
+# Tom Zanussi <tom.zanussi (at] linux.intel.com>
+#
+
+import logging
+import os
+import re
+
+from wic import WicError
+from wic.engine import get_custom_config
+from wic.pluginbase import SourcePlugin
+from wic.misc import (exec_cmd, exec_native_cmd,
+                      get_bitbake_var, BOOTDD_EXTRA_SPACE)
+
+logger = logging.getLogger('wic')
+
+class BootimgPcbiosPlugin(SourcePlugin):
+    """
+    Create MBR boot partition and install syslinux on it.
+    """
+
+    name = 'bootimg-pcbios'
+
+    @classmethod
+    def _get_bootimg_dir(cls, bootimg_dir, dirname):
+        """
+        Check if dirname exists in default bootimg_dir or in STAGING_DIR.
+        """
+        staging_datadir = get_bitbake_var("STAGING_DATADIR")
+        for result in (bootimg_dir, staging_datadir):
+            if os.path.exists("%s/%s" % (result, dirname)):
+                return result
+
+        # STAGING_DATADIR is expanded with MLPREFIX if multilib is enabled
+        # but dependency syslinux is still populated to original STAGING_DATADIR
+        nonarch_datadir = re.sub('/[^/]*recipe-sysroot', '/recipe-sysroot', staging_datadir)
+        if os.path.exists(os.path.join(nonarch_datadir, dirname)):
+            return nonarch_datadir
+
+        raise WicError("Couldn't find correct bootimg_dir, exiting")
+
+    @classmethod
+    def do_install_disk(cls, disk, disk_name, creator, workdir, oe_builddir,
+                        bootimg_dir, kernel_dir, native_sysroot):
+        """
+        Called after all partitions have been prepared and assembled into a
+        disk image.  In this case, we install the MBR.
+        """
+        bootimg_dir = cls._get_bootimg_dir(bootimg_dir, 'syslinux')
+        mbrfile = "%s/syslinux/" % bootimg_dir
+        if creator.ptable_format == 'msdos':
+            mbrfile += "mbr.bin"
+        elif creator.ptable_format == 'gpt':
+            mbrfile += "gptmbr.bin"
+        else:
+            raise WicError("Unsupported partition table: %s" %
+                           creator.ptable_format)
+
+        if not os.path.exists(mbrfile):
+            raise WicError("Couldn't find %s.  If using the -e option, do you "
+                           "have the right MACHINE set in local.conf?  If not, "
+                           "is the bootimg_dir path correct?" % mbrfile)
+
+        full_path = creator._full_path(workdir, disk_name, "direct")
+        logger.debug("Installing MBR on disk %s as %s with size %s bytes",
+                     disk_name, full_path, disk.min_size)
+
+        dd_cmd = "dd if=%s of=%s conv=notrunc" % (mbrfile, full_path)
+        exec_cmd(dd_cmd, native_sysroot)
+
+    @classmethod
+    def do_configure_partition(cls, part, source_params, creator, cr_workdir,
+                               oe_builddir, bootimg_dir, kernel_dir,
+                               native_sysroot):
+        """
+        Called before do_prepare_partition(), creates syslinux config
+        """
+        hdddir = "%s/hdd/boot" % cr_workdir
+
+        install_cmd = "install -d %s" % hdddir
+        exec_cmd(install_cmd)
+
+        bootloader = creator.ks.bootloader
+
+        custom_cfg = None
+        if bootloader.configfile:
+            custom_cfg = get_custom_config(bootloader.configfile)
+            if custom_cfg:
+                # Use a custom configuration for grub
+                syslinux_conf = custom_cfg
+                logger.debug("Using custom configuration file %s "
+                             "for syslinux.cfg", bootloader.configfile)
+            else:
+                raise WicError("configfile is specified but failed to "
+                               "get it from %s." % bootloader.configfile)
+
+        if not custom_cfg:
+            # Create syslinux configuration using parameters from wks file
+            splash = os.path.join(cr_workdir, "/hdd/boot/splash.jpg")
+            if os.path.exists(splash):
+                splashline = "menu background splash.jpg"
+            else:
+                splashline = ""
+
+            syslinux_conf = ""
+            syslinux_conf += "PROMPT 0\n"
+            syslinux_conf += "TIMEOUT " + str(bootloader.timeout) + "\n"
+            syslinux_conf += "\n"
+            syslinux_conf += "ALLOWOPTIONS 1\n"
+            syslinux_conf += "SERIAL 0 115200\n"
+            syslinux_conf += "\n"
+            if splashline:
+                syslinux_conf += "%s\n" % splashline
+            syslinux_conf += "DEFAULT boot\n"
+            syslinux_conf += "LABEL boot\n"
+
+            kernel = "/" + get_bitbake_var("KERNEL_IMAGETYPE")
+            syslinux_conf += "KERNEL " + kernel + "\n"
+
+            syslinux_conf += "APPEND label=boot root=%s %s\n" % \
+                             (creator.rootdev, bootloader.append)
+
+        logger.debug("Writing syslinux config %s/hdd/boot/syslinux.cfg",
+                     cr_workdir)
+        cfg = open("%s/hdd/boot/syslinux.cfg" % cr_workdir, "w")
+        cfg.write(syslinux_conf)
+        cfg.close()
+
+    @classmethod
+    def do_prepare_partition(cls, part, source_params, creator, cr_workdir,
+                             oe_builddir, bootimg_dir, kernel_dir,
+                             rootfs_dir, native_sysroot):
+        """
+        Called to do the actual content population for a partition i.e. it
+        'prepares' the partition to be incorporated into the image.
+        In this case, prepare content for legacy bios boot partition.
+        """
+        bootimg_dir = cls._get_bootimg_dir(bootimg_dir, 'syslinux')
+
+        staging_kernel_dir = kernel_dir
+
+        hdddir = "%s/hdd/boot" % cr_workdir
+
+        kernel = get_bitbake_var("KERNEL_IMAGETYPE")
+        if get_bitbake_var("INITRAMFS_IMAGE_BUNDLE") == "1":
+            if get_bitbake_var("INITRAMFS_IMAGE"):
+                kernel = "%s-%s.bin" % \
+                    (get_bitbake_var("KERNEL_IMAGETYPE"), get_bitbake_var("INITRAMFS_LINK_NAME"))
+
+        cmds = ("install -m 0644 %s/%s %s/%s" %
+                (staging_kernel_dir, kernel, hdddir, get_bitbake_var("KERNEL_IMAGETYPE")),
+                "install -m 444 %s/syslinux/ldlinux.sys %s/ldlinux.sys" %
+                (bootimg_dir, hdddir),
+                "install -m 0644 %s/syslinux/vesamenu.c32 %s/vesamenu.c32" %
+                (bootimg_dir, hdddir),
+                "install -m 444 %s/syslinux/libcom32.c32 %s/libcom32.c32" %
+                (bootimg_dir, hdddir),
+                "install -m 444 %s/syslinux/libutil.c32 %s/libutil.c32" %
+                (bootimg_dir, hdddir))
+
+        for install_cmd in cmds:
+            exec_cmd(install_cmd)
+
+        du_cmd = "du -bks %s" % hdddir
+        out = exec_cmd(du_cmd)
+        blocks = int(out.split()[0])
+
+        extra_blocks = part.get_extra_block_count(blocks)
+
+        if extra_blocks < BOOTDD_EXTRA_SPACE:
+            extra_blocks = BOOTDD_EXTRA_SPACE
+
+        blocks += extra_blocks
+
+        logger.debug("Added %d extra blocks to %s to get to %d total blocks",
+                     extra_blocks, part.mountpoint, blocks)
+
+        # dosfs image, created by mkdosfs
+        bootimg = "%s/boot%s.img" % (cr_workdir, part.lineno)
+
+        label = part.label if part.label else "boot"
+
+        dosfs_cmd = "mkdosfs -n %s -i %s -S 512 -C %s %d" % \
+                    (label, part.fsuuid, bootimg, blocks)
+        exec_native_cmd(dosfs_cmd, native_sysroot)
+
+        mcopy_cmd = "mcopy -i %s -s %s/* ::/" % (bootimg, hdddir)
+        exec_native_cmd(mcopy_cmd, native_sysroot)
+
+        syslinux_cmd = "syslinux %s" % bootimg
+        exec_native_cmd(syslinux_cmd, native_sysroot)
+
+        chmod_cmd = "chmod 644 %s" % bootimg
+        exec_cmd(chmod_cmd)
+
+        du_cmd = "du -Lbks %s" % bootimg
+        out = exec_cmd(du_cmd)
+        bootimg_size = out.split()[0]
+
+        part.size = int(bootimg_size)
+        part.source_file = bootimg

sources/poky/scripts/lib/wic/plugins/source/empty.py

@@ -0,0 +1,89 @@
+#
+# Copyright OpenEmbedded Contributors
+#
+# SPDX-License-Identifier: MIT
+#
+
+# The empty wic plugin is used to create unformatted empty partitions for wic
+# images.
+# To use it you must pass "empty" as argument for the "--source" parameter in
+# the wks file. For example:
+# part foo --source empty --ondisk sda --size="1024" --align 1024
+#
+# The plugin supports writing zeros to the start of the
+# partition. This is useful to overwrite old content like
+# filesystem signatures which may be re-recognized otherwise.
+# This feature can be enabled with
+# '--sourceparams="[fill|size=<N>[S|s|K|k|M|G]][,][bs=<N>[S|s|K|k|M|G]]"'
+# Conflicting or missing options throw errors.
+
+import logging
+import os
+
+from wic import WicError
+from wic.ksparser import sizetype
+from wic.pluginbase import SourcePlugin
+
+logger = logging.getLogger('wic')
+
+class EmptyPartitionPlugin(SourcePlugin):
+    """
+    Populate unformatted empty partition.
+
+    The following sourceparams are supported:
+    - fill
+      Fill the entire partition with zeros. Requires '--fixed-size' option
+      to be set.
+    - size=<N>[S|s|K|k|M|G]
+      Set the first N bytes of the partition to zero. Default unit is 'K'.
+    - bs=<N>[S|s|K|k|M|G]
+      Write at most N bytes at a time during source file creation.
+      Defaults to '1M'. Default unit is 'K'.
+    """
+
+    name = 'empty'
+
+    @classmethod
+    def do_prepare_partition(cls, part, source_params, cr, cr_workdir,
+                             oe_builddir, bootimg_dir, kernel_dir,
+                             rootfs_dir, native_sysroot):
+        """
+        Called to do the actual content population for a partition i.e. it
+        'prepares' the partition to be incorporated into the image.
+        """
+        get_byte_count = sizetype('K', True)
+        size = 0
+
+        if 'fill' in source_params and 'size' in source_params:
+            raise WicError("Conflicting source parameters 'fill' and 'size' specified, exiting.")
+
+        # Set the size of the zeros to be written to the partition
+        if 'fill' in source_params:
+            if part.fixed_size == 0:
+                raise WicError("Source parameter 'fill' only works with the '--fixed-size' option, exiting.")
+            size = get_byte_count(part.fixed_size)
+        elif 'size' in source_params:
+            size = get_byte_count(source_params['size'])
+
+        if size == 0:
+            # Nothing to do, create empty partition
+            return
+
+        if 'bs' in source_params:
+            bs = get_byte_count(source_params['bs'])
+        else:
+            bs = get_byte_count('1M')
+
+        # Create a binary file of the requested size filled with zeros
+        source_file = os.path.join(cr_workdir, 'empty-plugin-zeros%s.bin' % part.lineno)
+        if not os.path.exists(os.path.dirname(source_file)):
+            os.makedirs(os.path.dirname(source_file))
+
+        quotient, remainder = divmod(size, bs)
+        with open(source_file, 'wb') as file:
+            for _ in range(quotient):
+                file.write(bytearray(bs))
+            file.write(bytearray(remainder))
+
+        part.size = (size + 1024 - 1) // 1024  # size in KB rounded up
+        part.source_file = source_file

sources/poky/scripts/lib/wic/plugins/source/isoimage-isohybrid.py

@@ -0,0 +1,463 @@
+#
+# Copyright OpenEmbedded Contributors
+#
+# SPDX-License-Identifier: GPL-2.0-only
+#
+# DESCRIPTION
+# This implements the 'isoimage-isohybrid' source plugin class for 'wic'
+#
+# AUTHORS
+# Mihaly Varga <mihaly.varga (at] ni.com>
+
+import glob
+import logging
+import os
+import re
+import shutil
+
+from wic import WicError
+from wic.engine import get_custom_config
+from wic.pluginbase import SourcePlugin
+from wic.misc import exec_cmd, exec_native_cmd, get_bitbake_var
+
+logger = logging.getLogger('wic')
+
+class IsoImagePlugin(SourcePlugin):
+    """
+    Create a bootable ISO image
+
+    This plugin creates a hybrid, legacy and EFI bootable ISO image. The
+    generated image can be used on optical media as well as USB media.
+
+    Legacy boot uses syslinux and EFI boot uses grub or gummiboot (not
+    implemented yet) as bootloader. The plugin creates the directories required
+    by bootloaders and populates them by creating and configuring the
+    bootloader files.
+
+    Example kickstart file:
+    part /boot --source isoimage-isohybrid --sourceparams="loader=grub-efi, \\
+    image_name= IsoImage" --ondisk cd --label LIVECD
+    bootloader  --timeout=10  --append=" "
+
+    In --sourceparams "loader" specifies the bootloader used for booting in EFI
+    mode, while "image_name" specifies the name of the generated image. In the
+    example above, wic creates an ISO image named IsoImage-cd.direct (default
+    extension added by direct imeger plugin) and a file named IsoImage-cd.iso
+    """
+
+    name = 'isoimage-isohybrid'
+
+    @classmethod
+    def do_configure_syslinux(cls, creator, cr_workdir):
+        """
+        Create loader-specific (syslinux) config
+        """
+        splash = os.path.join(cr_workdir, "ISO/boot/splash.jpg")
+        if os.path.exists(splash):
+            splashline = "menu background splash.jpg"
+        else:
+            splashline = ""
+
+        bootloader = creator.ks.bootloader
+
+        syslinux_conf = ""
+        syslinux_conf += "PROMPT 0\n"
+        syslinux_conf += "TIMEOUT %s \n" % (bootloader.timeout or 10)
+        syslinux_conf += "\n"
+        syslinux_conf += "ALLOWOPTIONS 1\n"
+        syslinux_conf += "SERIAL 0 115200\n"
+        syslinux_conf += "\n"
+        if splashline:
+            syslinux_conf += "%s\n" % splashline
+        syslinux_conf += "DEFAULT boot\n"
+        syslinux_conf += "LABEL boot\n"
+
+        kernel = get_bitbake_var("KERNEL_IMAGETYPE")
+        if get_bitbake_var("INITRAMFS_IMAGE_BUNDLE") == "1":
+            if get_bitbake_var("INITRAMFS_IMAGE"):
+                kernel = "%s-%s.bin" % \
+                    (get_bitbake_var("KERNEL_IMAGETYPE"), get_bitbake_var("INITRAMFS_LINK_NAME"))
+
+        syslinux_conf += "KERNEL /" + kernel + "\n"
+        syslinux_conf += "APPEND initrd=/initrd LABEL=boot %s\n" \
+                             % bootloader.append
+
+        logger.debug("Writing syslinux config %s/ISO/isolinux/isolinux.cfg",
+                     cr_workdir)
+
+        with open("%s/ISO/isolinux/isolinux.cfg" % cr_workdir, "w") as cfg:
+            cfg.write(syslinux_conf)
+
+    @classmethod
+    def do_configure_grubefi(cls, part, creator, target_dir):
+        """
+        Create loader-specific (grub-efi) config
+        """
+        configfile = creator.ks.bootloader.configfile
+        if configfile:
+            grubefi_conf = get_custom_config(configfile)
+            if grubefi_conf:
+                logger.debug("Using custom configuration file %s for grub.cfg",
+                             configfile)
+            else:
+                raise WicError("configfile is specified "
+                               "but failed to get it from %s", configfile)
+        else:
+            splash = os.path.join(target_dir, "splash.jpg")
+            if os.path.exists(splash):
+                splashline = "menu background splash.jpg"
+            else:
+                splashline = ""
+
+            bootloader = creator.ks.bootloader
+
+            grubefi_conf = ""
+            grubefi_conf += "serial --unit=0 --speed=115200 --word=8 "
+            grubefi_conf += "--parity=no --stop=1\n"
+            grubefi_conf += "default=boot\n"
+            grubefi_conf += "timeout=%s\n" % (bootloader.timeout or 10)
+            grubefi_conf += "\n"
+            grubefi_conf += "search --set=root --label %s " % part.label
+            grubefi_conf += "\n"
+            grubefi_conf += "menuentry 'boot'{\n"
+
+            kernel = get_bitbake_var("KERNEL_IMAGETYPE")
+            if get_bitbake_var("INITRAMFS_IMAGE_BUNDLE") == "1":
+                if get_bitbake_var("INITRAMFS_IMAGE"):
+                    kernel = "%s-%s.bin" % \
+                        (get_bitbake_var("KERNEL_IMAGETYPE"), get_bitbake_var("INITRAMFS_LINK_NAME"))
+
+            grubefi_conf += "linux /%s rootwait %s\n" \
+                            % (kernel, bootloader.append)
+            grubefi_conf += "initrd /initrd \n"
+            grubefi_conf += "}\n"
+
+            if splashline:
+                grubefi_conf += "%s\n" % splashline
+
+        cfg_path = os.path.join(target_dir, "grub.cfg")
+        logger.debug("Writing grubefi config %s", cfg_path)
+
+        with open(cfg_path, "w") as cfg:
+            cfg.write(grubefi_conf)
+
+    @staticmethod
+    def _build_initramfs_path(rootfs_dir, cr_workdir):
+        """
+        Create path for initramfs image
+        """
+
+        initrd = get_bitbake_var("INITRD_LIVE") or get_bitbake_var("INITRD")
+        if not initrd:
+            initrd_dir = get_bitbake_var("DEPLOY_DIR_IMAGE")
+            if not initrd_dir:
+                raise WicError("Couldn't find DEPLOY_DIR_IMAGE, exiting.")
+
+            image_name = get_bitbake_var("IMAGE_BASENAME")
+            if not image_name:
+                raise WicError("Couldn't find IMAGE_BASENAME, exiting.")
+
+            image_type = get_bitbake_var("INITRAMFS_FSTYPES")
+            if not image_type:
+                raise WicError("Couldn't find INITRAMFS_FSTYPES, exiting.")
+
+            machine = os.path.basename(initrd_dir)
+
+            pattern = '%s/%s*%s.%s' % (initrd_dir, image_name, machine, image_type)
+            files = glob.glob(pattern)
+            if files:
+                initrd = files[0]
+
+        if not initrd or not os.path.exists(initrd):
+            # Create initrd from rootfs directory
+            initrd = "%s/initrd.cpio.gz" % cr_workdir
+            initrd_dir = "%s/INITRD" % cr_workdir
+            shutil.copytree("%s" % rootfs_dir, \
+                            "%s" % initrd_dir, symlinks=True)
+
+            if os.path.isfile("%s/init" % rootfs_dir):
+                shutil.copy2("%s/init" % rootfs_dir, "%s/init" % initrd_dir)
+            elif os.path.lexists("%s/init" % rootfs_dir):
+                os.symlink(os.readlink("%s/init" % rootfs_dir), \
+                            "%s/init" % initrd_dir)
+            elif os.path.isfile("%s/sbin/init" % rootfs_dir):
+                shutil.copy2("%s/sbin/init" % rootfs_dir, \
+                            "%s" % initrd_dir)
+            elif os.path.lexists("%s/sbin/init" % rootfs_dir):
+                os.symlink(os.readlink("%s/sbin/init" % rootfs_dir), \
+                            "%s/init" % initrd_dir)
+            else:
+                raise WicError("Couldn't find or build initrd, exiting.")
+
+            exec_cmd("cd %s && find . | cpio -o -H newc -R root:root >%s/initrd.cpio " \
+                     % (initrd_dir, cr_workdir), as_shell=True)
+            exec_cmd("gzip -f -9 %s/initrd.cpio" % cr_workdir, as_shell=True)
+            shutil.rmtree(initrd_dir)
+
+        return initrd
+
+    @classmethod
+    def do_configure_partition(cls, part, source_params, creator, cr_workdir,
+                               oe_builddir, bootimg_dir, kernel_dir,
+                               native_sysroot):
+        """
+        Called before do_prepare_partition(), creates loader-specific config
+        """
+        isodir = "%s/ISO/" % cr_workdir
+
+        if os.path.exists(isodir):
+            shutil.rmtree(isodir)
+
+        install_cmd = "install -d %s " % isodir
+        exec_cmd(install_cmd)
+
+        # Overwrite the name of the created image
+        logger.debug(source_params)
+        if 'image_name' in source_params and \
+                    source_params['image_name'].strip():
+            creator.name = source_params['image_name'].strip()
+            logger.debug("The name of the image is: %s", creator.name)
+
+    @staticmethod
+    def _install_payload(source_params, iso_dir):
+        """
+        Copies contents of payload directory (as specified in 'payload_dir' param) into iso_dir
+        """
+
+        if source_params.get('payload_dir'):
+            payload_dir = source_params['payload_dir']
+
+            logger.debug("Payload directory: %s", payload_dir)
+            shutil.copytree(payload_dir, iso_dir, symlinks=True, dirs_exist_ok=True)
+
+    @classmethod
+    def do_prepare_partition(cls, part, source_params, creator, cr_workdir,
+                             oe_builddir, bootimg_dir, kernel_dir,
+                             rootfs_dir, native_sysroot):
+        """
+        Called to do the actual content population for a partition i.e. it
+        'prepares' the partition to be incorporated into the image.
+        In this case, prepare content for a bootable ISO image.
+        """
+
+        isodir = "%s/ISO" % cr_workdir
+
+        cls._install_payload(source_params, isodir)
+
+        if part.rootfs_dir is None:
+            if not 'ROOTFS_DIR' in rootfs_dir:
+                raise WicError("Couldn't find --rootfs-dir, exiting.")
+            rootfs_dir = rootfs_dir['ROOTFS_DIR']
+        else:
+            if part.rootfs_dir in rootfs_dir:
+                rootfs_dir = rootfs_dir[part.rootfs_dir]
+            elif part.rootfs_dir:
+                rootfs_dir = part.rootfs_dir
+            else:
+                raise WicError("Couldn't find --rootfs-dir=%s connection "
+                               "or it is not a valid path, exiting." %
+                               part.rootfs_dir)
+
+        if not os.path.isdir(rootfs_dir):
+            rootfs_dir = get_bitbake_var("IMAGE_ROOTFS")
+        if not os.path.isdir(rootfs_dir):
+            raise WicError("Couldn't find IMAGE_ROOTFS, exiting.")
+
+        part.rootfs_dir = rootfs_dir
+        deploy_dir = get_bitbake_var("DEPLOY_DIR_IMAGE")
+        img_iso_dir = get_bitbake_var("ISODIR")
+
+        # Remove the temporary file created by part.prepare_rootfs()
+        if os.path.isfile(part.source_file):
+            os.remove(part.source_file)
+
+        # Support using a different initrd other than default
+        if source_params.get('initrd'):
+            initrd = source_params['initrd']
+            if not deploy_dir:
+                raise WicError("Couldn't find DEPLOY_DIR_IMAGE, exiting")
+            cp_cmd = "cp %s/%s %s" % (deploy_dir, initrd, cr_workdir)
+            exec_cmd(cp_cmd)
+        else:
+            # Prepare initial ramdisk
+            initrd = "%s/initrd" % deploy_dir
+            if not os.path.isfile(initrd):
+                initrd = "%s/initrd" % img_iso_dir
+            if not os.path.isfile(initrd):
+                initrd = cls._build_initramfs_path(rootfs_dir, cr_workdir)
+
+        install_cmd = "install -m 0644 %s %s/initrd" % (initrd, isodir)
+        exec_cmd(install_cmd)
+
+        # Remove the temporary file created by _build_initramfs_path function
+        if os.path.isfile("%s/initrd.cpio.gz" % cr_workdir):
+            os.remove("%s/initrd.cpio.gz" % cr_workdir)
+
+        kernel = get_bitbake_var("KERNEL_IMAGETYPE")
+        if get_bitbake_var("INITRAMFS_IMAGE_BUNDLE") == "1":
+            if get_bitbake_var("INITRAMFS_IMAGE"):
+                kernel = "%s-%s.bin" % \
+                    (get_bitbake_var("KERNEL_IMAGETYPE"), get_bitbake_var("INITRAMFS_LINK_NAME"))
+
+        install_cmd = "install -m 0644 %s/%s %s/%s" % \
+                      (kernel_dir, kernel, isodir, kernel)
+        exec_cmd(install_cmd)
+
+        #Create bootloader for efi boot
+        try:
+            target_dir = "%s/EFI/BOOT" % isodir
+            if os.path.exists(target_dir):
+                shutil.rmtree(target_dir)
+
+            os.makedirs(target_dir)
+
+            if source_params['loader'] == 'grub-efi':
+                # Builds bootx64.efi/bootia32.efi if ISODIR didn't exist or
+                # didn't contains it
+                target_arch = get_bitbake_var("TARGET_SYS")
+                if not target_arch:
+                    raise WicError("Coludn't find target architecture")
+
+                if re.match("x86_64", target_arch):
+                    grub_src_image = "grub-efi-bootx64.efi"
+                    grub_dest_image = "bootx64.efi"
+                elif re.match('i.86', target_arch):
+                    grub_src_image = "grub-efi-bootia32.efi"
+                    grub_dest_image = "bootia32.efi"
+                else:
+                    raise WicError("grub-efi is incompatible with target %s" %
+                                   target_arch)
+
+                grub_target = os.path.join(target_dir, grub_dest_image)
+                if not os.path.isfile(grub_target):
+                    grub_src = os.path.join(deploy_dir, grub_src_image)
+                    if not os.path.exists(grub_src):
+                        raise WicError("Grub loader %s is not found in %s. "
+                                       "Please build grub-efi first" % (grub_src_image, deploy_dir))
+                    shutil.copy(grub_src, grub_target)
+
+                if not os.path.isfile(os.path.join(target_dir, "boot.cfg")):
+                    cls.do_configure_grubefi(part, creator, target_dir)
+
+            else:
+                raise WicError("unrecognized bootimg-efi loader: %s" %
+                               source_params['loader'])
+        except KeyError:
+            raise WicError("bootimg-efi requires a loader, none specified")
+
+        # Create efi.img that contains bootloader files for EFI booting
+        # if ISODIR didn't exist or didn't contains it
+        if os.path.isfile("%s/efi.img" % img_iso_dir):
+            install_cmd = "install -m 0644 %s/efi.img %s/efi.img" % \
+                (img_iso_dir, isodir)
+            exec_cmd(install_cmd)
+        else:
+            # Default to 100 blocks of extra space for file system overhead
+            esp_extra_blocks = int(source_params.get('esp_extra_blocks', '100'))
+
+            du_cmd = "du -bks %s/EFI" % isodir
+            out = exec_cmd(du_cmd)
+            blocks = int(out.split()[0])
+            blocks += esp_extra_blocks
+            logger.debug("Added 100 extra blocks to %s to get to %d "
+                         "total blocks", part.mountpoint, blocks)
+
+            # dosfs image for EFI boot
+            bootimg = "%s/efi.img" % isodir
+
+            esp_label = source_params.get('esp_label', 'EFIimg')
+
+            dosfs_cmd = 'mkfs.vfat -n \'%s\' -S 512 -C %s %d' \
+                        % (esp_label, bootimg, blocks)
+            exec_native_cmd(dosfs_cmd, native_sysroot)
+
+            mmd_cmd = "mmd -i %s ::/EFI" % bootimg
+            exec_native_cmd(mmd_cmd, native_sysroot)
+
+            mcopy_cmd = "mcopy -i %s -s %s/EFI/* ::/EFI/" \
+                        % (bootimg, isodir)
+            exec_native_cmd(mcopy_cmd, native_sysroot)
+
+            chmod_cmd = "chmod 644 %s" % bootimg
+            exec_cmd(chmod_cmd)
+
+        # Prepare files for legacy boot
+        syslinux_dir = get_bitbake_var("STAGING_DATADIR")
+        if not syslinux_dir:
+            raise WicError("Couldn't find STAGING_DATADIR, exiting.")
+
+        if os.path.exists("%s/isolinux" % isodir):
+            shutil.rmtree("%s/isolinux" % isodir)
+
+        install_cmd = "install -d %s/isolinux" % isodir
+        exec_cmd(install_cmd)
+
+        cls.do_configure_syslinux(creator, cr_workdir)
+
+        install_cmd = "install -m 444 %s/syslinux/ldlinux.sys " % syslinux_dir
+        install_cmd += "%s/isolinux/ldlinux.sys" % isodir
+        exec_cmd(install_cmd)
+
+        install_cmd = "install -m 444 %s/syslinux/isohdpfx.bin " % syslinux_dir
+        install_cmd += "%s/isolinux/isohdpfx.bin" % isodir
+        exec_cmd(install_cmd)
+
+        install_cmd = "install -m 644 %s/syslinux/isolinux.bin " % syslinux_dir
+        install_cmd += "%s/isolinux/isolinux.bin" % isodir
+        exec_cmd(install_cmd)
+
+        install_cmd = "install -m 644 %s/syslinux/ldlinux.c32 " % syslinux_dir
+        install_cmd += "%s/isolinux/ldlinux.c32" % isodir
+        exec_cmd(install_cmd)
+
+        #create ISO image
+        iso_img = "%s/tempiso_img.iso" % cr_workdir
+        iso_bootimg = "isolinux/isolinux.bin"
+        iso_bootcat = "isolinux/boot.cat"
+        efi_img = "efi.img"
+
+        mkisofs_cmd = "mkisofs -V %s " % part.label
+        mkisofs_cmd += "-o %s -U " % iso_img
+        mkisofs_cmd += "-J -joliet-long -r -iso-level 2 -b %s " % iso_bootimg
+        mkisofs_cmd += "-c %s -no-emul-boot -boot-load-size 4 " % iso_bootcat
+        mkisofs_cmd += "-boot-info-table -eltorito-alt-boot "
+        mkisofs_cmd += "-eltorito-platform 0xEF -eltorito-boot %s " % efi_img
+        mkisofs_cmd += "-no-emul-boot %s " % isodir
+
+        logger.debug("running command: %s", mkisofs_cmd)
+        exec_native_cmd(mkisofs_cmd, native_sysroot)
+
+        shutil.rmtree(isodir)
+
+        du_cmd = "du -Lbks %s" % iso_img
+        out = exec_cmd(du_cmd)
+        isoimg_size = int(out.split()[0])
+
+        part.size = isoimg_size
+        part.source_file = iso_img
+
+    @classmethod
+    def do_install_disk(cls, disk, disk_name, creator, workdir, oe_builddir,
+                        bootimg_dir, kernel_dir, native_sysroot):
+        """
+        Called after all partitions have been prepared and assembled into a
+        disk image.  In this case, we insert/modify the MBR using isohybrid
+        utility for booting via BIOS from disk storage devices.
+        """
+
+        iso_img = "%s.p1" % disk.path
+        full_path = creator._full_path(workdir, disk_name, "direct")
+        full_path_iso = creator._full_path(workdir, disk_name, "iso")
+
+        isohybrid_cmd = "isohybrid -u %s" % iso_img
+        logger.debug("running command: %s", isohybrid_cmd)
+        exec_native_cmd(isohybrid_cmd, native_sysroot)
+
+        # Replace the image created by direct plugin with the one created by
+        # mkisofs command. This is necessary because the iso image created by
+        # mkisofs has a very specific MBR is system area of the ISO image, and
+        # direct plugin adds and configures an another MBR.
+        logger.debug("Replaceing the image created by direct plugin\n")
+        os.remove(disk.path)
+        shutil.copy2(iso_img, full_path_iso)
+        shutil.copy2(full_path_iso, full_path)

sources/poky/scripts/lib/wic/plugins/source/rawcopy.py

@@ -0,0 +1,115 @@
+#
+# Copyright OpenEmbedded Contributors
+#
+# SPDX-License-Identifier: GPL-2.0-only
+#
+
+import logging
+import os
+import signal
+import subprocess
+
+from wic import WicError
+from wic.pluginbase import SourcePlugin
+from wic.misc import exec_cmd, get_bitbake_var
+from wic.filemap import sparse_copy
+
+logger = logging.getLogger('wic')
+
+class RawCopyPlugin(SourcePlugin):
+    """
+    Populate partition content from raw image file.
+    """
+
+    name = 'rawcopy'
+
+    @staticmethod
+    def do_image_label(fstype, dst, label):
+        # don't create label when fstype is none
+        if fstype == 'none':
+            return
+
+        if fstype.startswith('ext'):
+            cmd = 'tune2fs -L %s %s' % (label, dst)
+        elif fstype in ('msdos', 'vfat'):
+            cmd = 'dosfslabel %s %s' % (dst, label)
+        elif fstype == 'btrfs':
+            cmd = 'btrfs filesystem label %s %s' % (dst, label)
+        elif fstype == 'swap':
+            cmd = 'mkswap -L %s %s' % (label, dst)
+        elif fstype in ('squashfs', 'erofs'):
+            raise WicError("It's not possible to update a %s "
+                           "filesystem label '%s'" % (fstype, label))
+        else:
+            raise WicError("Cannot update filesystem label: "
+                           "Unknown fstype: '%s'" % (fstype))
+
+        exec_cmd(cmd)
+
+    @staticmethod
+    def do_image_uncompression(src, dst, workdir):
+        def subprocess_setup():
+            # Python installs a SIGPIPE handler by default. This is usually not what
+            # non-Python subprocesses expect.
+            # SIGPIPE errors are known issues with gzip/bash
+            signal.signal(signal.SIGPIPE, signal.SIG_DFL)
+
+        extension = os.path.splitext(src)[1]
+        decompressor = {
+            ".bz2": "bzip2",
+            ".gz": "gzip",
+            ".xz": "xz",
+            ".zst": "zstd -f",
+        }.get(extension)
+        if not decompressor:
+            raise WicError("Not supported compressor filename extension: %s" % extension)
+        cmd = "%s -dc %s > %s" % (decompressor, src, dst)
+        subprocess.call(cmd, preexec_fn=subprocess_setup, shell=True, cwd=workdir)
+
+    @classmethod
+    def do_prepare_partition(cls, part, source_params, cr, cr_workdir,
+                             oe_builddir, bootimg_dir, kernel_dir,
+                             rootfs_dir, native_sysroot):
+        """
+        Called to do the actual content population for a partition i.e. it
+        'prepares' the partition to be incorporated into the image.
+        """
+        if not kernel_dir:
+            kernel_dir = get_bitbake_var("DEPLOY_DIR_IMAGE")
+            if not kernel_dir:
+                raise WicError("Couldn't find DEPLOY_DIR_IMAGE, exiting")
+
+        logger.debug('Kernel dir: %s', kernel_dir)
+
+        if 'file' not in source_params:
+            raise WicError("No file specified")
+
+        if 'unpack' in source_params:
+            img = os.path.join(kernel_dir, source_params['file'])
+            src = os.path.join(cr_workdir, os.path.splitext(source_params['file'])[0])
+            RawCopyPlugin.do_image_uncompression(img, src, cr_workdir)
+        else:
+            src = os.path.join(kernel_dir, source_params['file'])
+
+        dst = os.path.join(cr_workdir, "%s.%s" % (os.path.basename(source_params['file']), part.lineno))
+
+        if not os.path.exists(os.path.dirname(dst)):
+            os.makedirs(os.path.dirname(dst))
+
+        if 'skip' in source_params:
+            sparse_copy(src, dst, skip=int(source_params['skip']))
+        else:
+            sparse_copy(src, dst)
+
+        # get the size in the right units for kickstart (kB)
+        du_cmd = "du -Lbks %s" % dst
+        out = exec_cmd(du_cmd)
+        filesize = int(out.split()[0])
+
+        if filesize > part.size:
+            part.size = filesize
+
+        if part.label:
+            RawCopyPlugin.do_image_label(part.fstype, dst, part.label)
+
+        part.source_file = dst

sources/poky/scripts/lib/wic/plugins/source/rootfs.py

@@ -0,0 +1,236 @@
+#
+# Copyright (c) 2014, Intel Corporation.
+#
+# SPDX-License-Identifier: GPL-2.0-only
+#
+# DESCRIPTION
+# This implements the 'rootfs' source plugin class for 'wic'
+#
+# AUTHORS
+# Tom Zanussi <tom.zanussi (at] linux.intel.com>
+# Joao Henrique Ferreira de Freitas <joaohf (at] gmail.com>
+#
+
+import logging
+import os
+import shutil
+import sys
+
+from oe.path import copyhardlinktree
+from pathlib import Path
+
+from wic import WicError
+from wic.pluginbase import SourcePlugin
+from wic.misc import get_bitbake_var, exec_native_cmd
+
+logger = logging.getLogger('wic')
+
+class RootfsPlugin(SourcePlugin):
+    """
+    Populate partition content from a rootfs directory.
+    """
+
+    name = 'rootfs'
+
+    @staticmethod
+    def __validate_path(cmd, rootfs_dir, path):
+        if os.path.isabs(path):
+            logger.error("%s: Must be relative: %s" % (cmd, path))
+            sys.exit(1)
+
+        # Disallow climbing outside of parent directory using '..',
+        # because doing so could be quite disastrous (we will delete the
+        # directory, or modify a directory outside OpenEmbedded).
+        full_path = os.path.realpath(os.path.join(rootfs_dir, path))
+        if not full_path.startswith(os.path.realpath(rootfs_dir)):
+            logger.error("%s: Must point inside the rootfs:" % (cmd, path))
+            sys.exit(1)
+
+        return full_path
+
+    @staticmethod
+    def __get_rootfs_dir(rootfs_dir):
+        if rootfs_dir and os.path.isdir(rootfs_dir):
+            return os.path.realpath(rootfs_dir)
+
+        image_rootfs_dir = get_bitbake_var("IMAGE_ROOTFS", rootfs_dir)
+        if not os.path.isdir(image_rootfs_dir):
+            raise WicError("No valid artifact IMAGE_ROOTFS from image "
+                           "named %s has been found at %s, exiting." %
+                           (rootfs_dir, image_rootfs_dir))
+
+        return os.path.realpath(image_rootfs_dir)
+
+    @staticmethod
+    def __get_pseudo(native_sysroot, rootfs, pseudo_dir):
+        pseudo = "export PSEUDO_PREFIX=%s/usr;" % native_sysroot
+        pseudo += "export PSEUDO_LOCALSTATEDIR=%s;" % pseudo_dir
+        pseudo += "export PSEUDO_PASSWD=%s;" % rootfs
+        pseudo += "export PSEUDO_NOSYMLINKEXP=1;"
+        pseudo += "%s " % get_bitbake_var("FAKEROOTCMD")
+        return pseudo
+
+    @classmethod
+    def do_prepare_partition(cls, part, source_params, cr, cr_workdir,
+                             oe_builddir, bootimg_dir, kernel_dir,
+                             krootfs_dir, native_sysroot):
+        """
+        Called to do the actual content population for a partition i.e. it
+        'prepares' the partition to be incorporated into the image.
+        In this case, prepare content for legacy bios boot partition.
+        """
+        if part.rootfs_dir is None:
+            if not 'ROOTFS_DIR' in krootfs_dir:
+                raise WicError("Couldn't find --rootfs-dir, exiting")
+
+            rootfs_dir = krootfs_dir['ROOTFS_DIR']
+        else:
+            if part.rootfs_dir in krootfs_dir:
+                rootfs_dir = krootfs_dir[part.rootfs_dir]
+            elif part.rootfs_dir:
+                rootfs_dir = part.rootfs_dir
+            else:
+                raise WicError("Couldn't find --rootfs-dir=%s connection or "
+                               "it is not a valid path, exiting" % part.rootfs_dir)
+
+        part.rootfs_dir = cls.__get_rootfs_dir(rootfs_dir)
+        part.has_fstab = os.path.exists(os.path.join(part.rootfs_dir, "etc/fstab"))
+        pseudo_dir = os.path.join(part.rootfs_dir, "../pseudo")
+        if not os.path.lexists(pseudo_dir):
+            pseudo_dir = os.path.join(cls.__get_rootfs_dir(None), '../pseudo')
+
+        if not os.path.lexists(pseudo_dir):
+            logger.warn("%s folder does not exist. "
+                        "Usernames and permissions will be invalid " % pseudo_dir)
+            pseudo_dir = None
+
+        new_rootfs = None
+        new_pseudo = None
+        # Handle excluded paths.
+        if part.exclude_path or part.include_path or part.change_directory or part.update_fstab_in_rootfs:
+            # We need a new rootfs directory we can safely modify without
+            # interfering with other tasks. Copy to workdir.
+            new_rootfs = os.path.realpath(os.path.join(cr_workdir, "rootfs%d" % part.lineno))
+
+            if os.path.lexists(new_rootfs):
+                shutil.rmtree(os.path.join(new_rootfs))
+
+            if part.change_directory:
+                cd = part.change_directory
+                if cd[-1] == '/':
+                    cd = cd[:-1]
+                orig_dir = cls.__validate_path("--change-directory", part.rootfs_dir, cd)
+            else:
+                orig_dir = part.rootfs_dir
+            copyhardlinktree(orig_dir, new_rootfs)
+
+            # Convert the pseudo directory to its new location
+            if (pseudo_dir):
+                new_pseudo = os.path.realpath(
+                             os.path.join(cr_workdir, "pseudo%d" % part.lineno))
+                if os.path.lexists(new_pseudo):
+                    shutil.rmtree(new_pseudo)
+                os.mkdir(new_pseudo)
+                shutil.copy(os.path.join(pseudo_dir, "files.db"),
+                            os.path.join(new_pseudo, "files.db"))
+
+                pseudo_cmd = "%s -B -m %s -M %s" % (cls.__get_pseudo(native_sysroot,
+                                                                     new_rootfs,
+                                                                     new_pseudo),
+                                                    orig_dir, new_rootfs)
+                exec_native_cmd(pseudo_cmd, native_sysroot)
+
+            for in_path in part.include_path or []:
+                #parse arguments
+                include_path = in_path[0]
+                if len(in_path) > 2:
+                    logger.error("'Invalid number of arguments for include-path")
+                    sys.exit(1)
+                if len(in_path) == 2:
+                    path = in_path[1]
+                else:
+                    path = None
+
+                # Pack files to be included into a tar file.
+                # We need to create a tar file, because that way we can keep the
+                # permissions from the files even when they belong to different
+                # pseudo enviroments.
+                # If we simply copy files using copyhardlinktree/copytree... the
+                # copied files will belong to the user running wic.
+                tar_file = os.path.realpath(
+                           os.path.join(cr_workdir, "include-path%d.tar" % part.lineno))
+                if os.path.isfile(include_path):
+                    parent = os.path.dirname(os.path.realpath(include_path))
+                    tar_cmd = "tar c --owner=root --group=root -f %s -C %s %s" % (
+                                tar_file, parent, os.path.relpath(include_path, parent))
+                    exec_native_cmd(tar_cmd, native_sysroot)
+                else:
+                    if include_path in krootfs_dir:
+                        include_path = krootfs_dir[include_path]
+                    include_path = cls.__get_rootfs_dir(include_path)
+                    include_pseudo = os.path.join(include_path, "../pseudo")
+                    if os.path.lexists(include_pseudo):
+                        pseudo = cls.__get_pseudo(native_sysroot, include_path,
+                                                  include_pseudo)
+                        tar_cmd = "tar cf %s -C %s ." % (tar_file, include_path)
+                    else:
+                        pseudo = None
+                        tar_cmd = "tar c --owner=root --group=root -f %s -C %s ." % (
+                                tar_file, include_path)
+                    exec_native_cmd(tar_cmd, native_sysroot, pseudo)
+
+                #create destination
+                if path:
+                    destination = cls.__validate_path("--include-path", new_rootfs, path)
+                    Path(destination).mkdir(parents=True, exist_ok=True)
+                else:
+                    destination = new_rootfs
+
+                #extract destination
+                untar_cmd = "tar xf %s -C %s" % (tar_file, destination)
+                if new_pseudo:
+                    pseudo = cls.__get_pseudo(native_sysroot, new_rootfs, new_pseudo)
+                else:
+                    pseudo = None
+                exec_native_cmd(untar_cmd, native_sysroot, pseudo)
+                os.remove(tar_file)
+
+            for orig_path in part.exclude_path or []:
+                path = orig_path
+
+                full_path = cls.__validate_path("--exclude-path", new_rootfs, path)
+
+                if not os.path.lexists(full_path):
+                    continue
+
+                if new_pseudo:
+                    pseudo = cls.__get_pseudo(native_sysroot, new_rootfs, new_pseudo)
+                else:
+                    pseudo = None
+                if path.endswith(os.sep):
+                    # Delete content only.
+                    for entry in os.listdir(full_path):
+                        full_entry = os.path.join(full_path, entry)
+                        rm_cmd = "rm -rf %s" % (full_entry)
+                        exec_native_cmd(rm_cmd, native_sysroot, pseudo)
+                else:
+                    # Delete whole directory.
+                    rm_cmd = "rm -rf %s" % (full_path)
+                    exec_native_cmd(rm_cmd, native_sysroot, pseudo)
+
+            # Update part.has_fstab here as fstab may have been added or
+            # removed by the above modifications.
+            part.has_fstab = os.path.exists(os.path.join(new_rootfs, "etc/fstab"))
+            if part.update_fstab_in_rootfs and part.has_fstab and not part.no_fstab_update:
+                fstab_path = os.path.join(new_rootfs, "etc/fstab")
+                # Assume that fstab should always be owned by root with fixed permissions
+                install_cmd = "install -m 0644 -p %s %s" % (part.updated_fstab_path, fstab_path)
+                if new_pseudo:
+                    pseudo = cls.__get_pseudo(native_sysroot, new_rootfs, new_pseudo)
+                else:
+                    pseudo = None
+                exec_native_cmd(install_cmd, native_sysroot, pseudo)
+
+        part.prepare_rootfs(cr_workdir, oe_builddir,
+                            new_rootfs or part.rootfs_dir, native_sysroot,
+                            pseudo_dir = new_pseudo or pseudo_dir)