Complete Yocto mirror with license table for TQMa6UL (2038-compliance)
- 264 license table entries with exact download URLs (224/264 resolved) - Complete sources/ directory with all BitBake recipes - Build configuration: tqma6ul-multi-mba6ulx, spaetzle (musl) - Full traceability for Softwarefreigabeantrag - GCC 13.4.0, Linux 6.6.102, U-Boot 2023.04, musl 1.2.4 - License distribution: GPL-2.0 (24), MIT (23), GPL-2.0+ (18), BSD-3 (16)
This commit is contained in:
Siggi (OpenClaw Agent)
302
sources/poky/scripts/buildstats-diff
Executable file
302
sources/poky/scripts/buildstats-diff
Executable file
@@ -0,0 +1,302 @@
|
||||
#!/usr/bin/env python3
|
||||
#
|
||||
# Script for comparing buildstats from two different builds
|
||||
#
|
||||
# Copyright (c) 2016, Intel Corporation.
|
||||
#
|
||||
# SPDX-License-Identifier: GPL-2.0-only
|
||||
#
|
||||
|
||||
import argparse
|
||||
import glob
|
||||
import logging
|
||||
import math
|
||||
import os
|
||||
import sys
|
||||
from operator import attrgetter
|
||||
|
||||
# Import oe libs
|
||||
scripts_path = os.path.dirname(os.path.realpath(__file__))
|
||||
sys.path.append(os.path.join(scripts_path, 'lib'))
|
||||
from buildstats import BuildStats, diff_buildstats, taskdiff_fields, BSVerDiff
|
||||
|
||||
|
||||
# Setup logging
|
||||
logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s")
|
||||
log = logging.getLogger()
|
||||
|
||||
|
||||
class ScriptError(Exception):
|
||||
"""Exception for internal error handling of this script"""
|
||||
pass
|
||||
|
||||
|
||||
def read_buildstats(path, multi):
|
||||
"""Read buildstats"""
|
||||
if not os.path.exists(path):
|
||||
raise ScriptError("No such file or directory: {}".format(path))
|
||||
|
||||
if os.path.isfile(path):
|
||||
return BuildStats.from_file_json(path)
|
||||
|
||||
if os.path.isfile(os.path.join(path, 'build_stats')):
|
||||
return BuildStats.from_dir(path)
|
||||
|
||||
# Handle a non-buildstat directory
|
||||
subpaths = sorted(glob.glob(path + '/*'))
|
||||
if len(subpaths) > 1:
|
||||
if multi:
|
||||
log.info("Averaging over {} buildstats from {}".format(
|
||||
len(subpaths), path))
|
||||
else:
|
||||
raise ScriptError("Multiple buildstats found in '{}'. Please give "
|
||||
"a single buildstat directory of use the --multi "
|
||||
"option".format(path))
|
||||
bs = None
|
||||
for subpath in subpaths:
|
||||
if os.path.isfile(subpath):
|
||||
_bs = BuildStats.from_file_json(subpath)
|
||||
else:
|
||||
_bs = BuildStats.from_dir(subpath)
|
||||
if bs is None:
|
||||
bs = _bs
|
||||
else:
|
||||
bs.aggregate(_bs)
|
||||
if not bs:
|
||||
raise ScriptError("No buildstats found under {}".format(path))
|
||||
|
||||
return bs
|
||||
|
||||
|
||||
def print_ver_diff(bs1, bs2):
|
||||
"""Print package version differences"""
|
||||
|
||||
diff = BSVerDiff(bs1, bs2)
|
||||
|
||||
maxlen = max([len(r) for r in set(bs1.keys()).union(set(bs2.keys()))])
|
||||
fmt_str = " {:{maxlen}} ({})"
|
||||
|
||||
if diff.new:
|
||||
print("\nNEW RECIPES:")
|
||||
print("------------")
|
||||
for name, val in sorted(diff.new.items()):
|
||||
print(fmt_str.format(name, val.nevr, maxlen=maxlen))
|
||||
|
||||
if diff.dropped:
|
||||
print("\nDROPPED RECIPES:")
|
||||
print("----------------")
|
||||
for name, val in sorted(diff.dropped.items()):
|
||||
print(fmt_str.format(name, val.nevr, maxlen=maxlen))
|
||||
|
||||
fmt_str = " {0:{maxlen}} {1:<20} ({2})"
|
||||
if diff.rchanged:
|
||||
print("\nREVISION CHANGED:")
|
||||
print("-----------------")
|
||||
for name, val in sorted(diff.rchanged.items()):
|
||||
field1 = "{} -> {}".format(val.left.revision, val.right.revision)
|
||||
field2 = "{} -> {}".format(val.left.nevr, val.right.nevr)
|
||||
print(fmt_str.format(name, field1, field2, maxlen=maxlen))
|
||||
|
||||
if diff.vchanged:
|
||||
print("\nVERSION CHANGED:")
|
||||
print("----------------")
|
||||
for name, val in sorted(diff.vchanged.items()):
|
||||
field1 = "{} -> {}".format(val.left.version, val.right.version)
|
||||
field2 = "{} -> {}".format(val.left.nevr, val.right.nevr)
|
||||
print(fmt_str.format(name, field1, field2, maxlen=maxlen))
|
||||
|
||||
if diff.echanged:
|
||||
print("\nEPOCH CHANGED:")
|
||||
print("--------------")
|
||||
for name, val in sorted(diff.echanged.items()):
|
||||
field1 = "{} -> {}".format(val.left.epoch, val.right.epoch)
|
||||
field2 = "{} -> {}".format(val.left.nevr, val.right.nevr)
|
||||
print(fmt_str.format(name, field1, field2, maxlen=maxlen))
|
||||
|
||||
|
||||
def print_task_diff(bs1, bs2, val_type, min_val=0, min_absdiff=0, sort_by=('absdiff',), only_tasks=[]):
|
||||
"""Diff task execution times"""
|
||||
def val_to_str(val, human_readable=False):
|
||||
"""Convert raw value to printable string"""
|
||||
def hms_time(secs):
|
||||
"""Get time in human-readable HH:MM:SS format"""
|
||||
h = int(secs / 3600)
|
||||
m = int((secs % 3600) / 60)
|
||||
s = secs % 60
|
||||
if h == 0:
|
||||
return "{:02d}:{:04.1f}".format(m, s)
|
||||
else:
|
||||
return "{:d}:{:02d}:{:04.1f}".format(h, m, s)
|
||||
|
||||
if 'time' in val_type:
|
||||
if human_readable:
|
||||
return hms_time(val)
|
||||
else:
|
||||
return "{:.1f}s".format(val)
|
||||
elif 'bytes' in val_type and human_readable:
|
||||
prefix = ['', 'Ki', 'Mi', 'Gi', 'Ti', 'Pi']
|
||||
dec = int(math.log(val, 2) / 10)
|
||||
prec = 1 if dec > 0 else 0
|
||||
return "{:.{prec}f}{}B".format(val / (2 ** (10 * dec)),
|
||||
prefix[dec], prec=prec)
|
||||
elif 'ops' in val_type and human_readable:
|
||||
prefix = ['', 'k', 'M', 'G', 'T', 'P']
|
||||
dec = int(math.log(val, 1000))
|
||||
prec = 1 if dec > 0 else 0
|
||||
return "{:.{prec}f}{}ops".format(val / (1000 ** dec),
|
||||
prefix[dec], prec=prec)
|
||||
return str(int(val))
|
||||
|
||||
def sum_vals(buildstats):
|
||||
"""Get cumulative sum of all tasks"""
|
||||
total = 0.0
|
||||
for recipe_data in buildstats.values():
|
||||
for name, bs_task in recipe_data.tasks.items():
|
||||
if not only_tasks or name in only_tasks:
|
||||
total += getattr(bs_task, val_type)
|
||||
return total
|
||||
|
||||
if min_val:
|
||||
print("Ignoring tasks less than {} ({})".format(
|
||||
val_to_str(min_val, True), val_to_str(min_val)))
|
||||
if min_absdiff:
|
||||
print("Ignoring differences less than {} ({})".format(
|
||||
val_to_str(min_absdiff, True), val_to_str(min_absdiff)))
|
||||
|
||||
# Prepare the data
|
||||
tasks_diff = diff_buildstats(bs1, bs2, val_type, min_val, min_absdiff, only_tasks)
|
||||
|
||||
# Sort our list
|
||||
for field in reversed(sort_by):
|
||||
if field.startswith('-'):
|
||||
field = field[1:]
|
||||
reverse = True
|
||||
else:
|
||||
reverse = False
|
||||
tasks_diff = sorted(tasks_diff, key=attrgetter(field), reverse=reverse)
|
||||
|
||||
linedata = [(' ', 'PKG', ' ', 'TASK', 'ABSDIFF', 'RELDIFF',
|
||||
val_type.upper() + '1', val_type.upper() + '2')]
|
||||
field_lens = dict([('len_{}'.format(i), len(f)) for i, f in enumerate(linedata[0])])
|
||||
|
||||
# Prepare fields in string format and measure field lengths
|
||||
for diff in tasks_diff:
|
||||
task_prefix = diff.task_op if diff.pkg_op == ' ' else ' '
|
||||
linedata.append((diff.pkg_op, diff.pkg, task_prefix, diff.task,
|
||||
val_to_str(diff.absdiff),
|
||||
'{:+.1f}%'.format(diff.reldiff),
|
||||
val_to_str(diff.value1),
|
||||
val_to_str(diff.value2)))
|
||||
for i, field in enumerate(linedata[-1]):
|
||||
key = 'len_{}'.format(i)
|
||||
if len(field) > field_lens[key]:
|
||||
field_lens[key] = len(field)
|
||||
|
||||
# Print data
|
||||
print()
|
||||
for fields in linedata:
|
||||
print("{:{len_0}}{:{len_1}} {:{len_2}}{:{len_3}} {:>{len_4}} {:>{len_5}} {:>{len_6}} -> {:{len_7}}".format(
|
||||
*fields, **field_lens))
|
||||
|
||||
# Print summary of the diffs
|
||||
total1 = sum_vals(bs1)
|
||||
total2 = sum_vals(bs2)
|
||||
print("\nCumulative {}:".format(val_type))
|
||||
print (" {} {:+.1f}% {} ({}) -> {} ({})".format(
|
||||
val_to_str(total2 - total1), 100 * (total2-total1) / total1,
|
||||
val_to_str(total1, True), val_to_str(total1),
|
||||
val_to_str(total2, True), val_to_str(total2)))
|
||||
|
||||
|
||||
def parse_args(argv):
|
||||
"""Parse cmdline arguments"""
|
||||
description="""
|
||||
Script for comparing buildstats of two separate builds."""
|
||||
parser = argparse.ArgumentParser(
|
||||
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
|
||||
description=description)
|
||||
|
||||
min_val_defaults = {'cputime': 3.0,
|
||||
'read_bytes': 524288,
|
||||
'write_bytes': 524288,
|
||||
'read_ops': 500,
|
||||
'write_ops': 500,
|
||||
'walltime': 5}
|
||||
min_absdiff_defaults = {'cputime': 1.0,
|
||||
'read_bytes': 131072,
|
||||
'write_bytes': 131072,
|
||||
'read_ops': 50,
|
||||
'write_ops': 50,
|
||||
'walltime': 2}
|
||||
|
||||
parser.add_argument('--debug', '-d', action='store_true',
|
||||
help="Verbose logging")
|
||||
parser.add_argument('--ver-diff', action='store_true',
|
||||
help="Show package version differences and exit")
|
||||
parser.add_argument('--diff-attr', default='cputime',
|
||||
choices=min_val_defaults.keys(),
|
||||
help="Buildstat attribute which to compare")
|
||||
parser.add_argument('--min-val', default=min_val_defaults, type=float,
|
||||
help="Filter out tasks less than MIN_VAL. "
|
||||
"Default depends on --diff-attr.")
|
||||
parser.add_argument('--min-absdiff', default=min_absdiff_defaults, type=float,
|
||||
help="Filter out tasks whose difference is less than "
|
||||
"MIN_ABSDIFF, Default depends on --diff-attr.")
|
||||
parser.add_argument('--sort-by', default='absdiff',
|
||||
help="Comma-separated list of field sort order. "
|
||||
"Prepend the field name with '-' for reversed sort. "
|
||||
"Available fields are: {}".format(', '.join(taskdiff_fields)))
|
||||
parser.add_argument('--multi', action='store_true',
|
||||
help="Read all buildstats from the given paths and "
|
||||
"average over them")
|
||||
parser.add_argument('--only-task', dest='only_tasks', metavar='TASK', action='append', default=[],
|
||||
help="Only include TASK in report. May be specified multiple times")
|
||||
parser.add_argument('buildstats1', metavar='BUILDSTATS1', help="'Left' buildstat")
|
||||
parser.add_argument('buildstats2', metavar='BUILDSTATS2', help="'Right' buildstat")
|
||||
|
||||
args = parser.parse_args(argv)
|
||||
|
||||
# We do not nedd/want to read all buildstats if we just want to look at the
|
||||
# package versions
|
||||
if args.ver_diff:
|
||||
args.multi = False
|
||||
|
||||
# Handle defaults for the filter arguments
|
||||
if args.min_val is min_val_defaults:
|
||||
args.min_val = min_val_defaults[args.diff_attr]
|
||||
if args.min_absdiff is min_absdiff_defaults:
|
||||
args.min_absdiff = min_absdiff_defaults[args.diff_attr]
|
||||
|
||||
return args
|
||||
|
||||
def main(argv=None):
|
||||
"""Script entry point"""
|
||||
args = parse_args(argv)
|
||||
if args.debug:
|
||||
log.setLevel(logging.DEBUG)
|
||||
|
||||
# Validate sort fields
|
||||
sort_by = []
|
||||
for field in args.sort_by.split(','):
|
||||
if field.lstrip('-') not in taskdiff_fields:
|
||||
log.error("Invalid sort field '%s' (must be one of: %s)" %
|
||||
(field, ', '.join(taskdiff_fields)))
|
||||
sys.exit(1)
|
||||
sort_by.append(field)
|
||||
|
||||
try:
|
||||
bs1 = read_buildstats(args.buildstats1, args.multi)
|
||||
bs2 = read_buildstats(args.buildstats2, args.multi)
|
||||
|
||||
if args.ver_diff:
|
||||
print_ver_diff(bs1, bs2)
|
||||
else:
|
||||
print_task_diff(bs1, bs2, args.diff_attr, args.min_val,
|
||||
args.min_absdiff, sort_by, args.only_tasks)
|
||||
except ScriptError as err:
|
||||
log.error(str(err))
|
||||
return 1
|
||||
return 0
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Reference in New Issue
Block a user