Complete Yocto mirror with license table for TQMa6UL (2038-compliance)
- 264 license table entries with exact download URLs (224/264 resolved) - Complete sources/ directory with all BitBake recipes - Build configuration: tqma6ul-multi-mba6ulx, spaetzle (musl) - Full traceability for Softwarefreigabeantrag - GCC 13.4.0, Linux 6.6.102, U-Boot 2023.04, musl 1.2.4 - License distribution: GPL-2.0 (24), MIT (23), GPL-2.0+ (18), BSD-3 (16)
This commit is contained in:
Siggi (OpenClaw Agent)
@@ -0,0 +1,170 @@
|
||||
#
|
||||
# SPDX-License-Identifier: GPL-2.0-only
|
||||
#
|
||||
|
||||
from django.core.management.base import BaseCommand
|
||||
|
||||
from django.core.management import call_command
|
||||
from bldcontrol.models import BuildRequest, BuildEnvironment, BRError
|
||||
from orm.models import ToasterSetting, Build, Layer
|
||||
|
||||
import os
|
||||
import traceback
|
||||
import warnings
|
||||
|
||||
|
||||
def DN(path):
|
||||
if path is None:
|
||||
return ""
|
||||
else:
|
||||
return os.path.dirname(path)
|
||||
|
||||
|
||||
class Command(BaseCommand):
|
||||
args = ""
|
||||
help = "Verifies that the configured settings are valid and usable, or prompts the user to fix the settings."
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
super(Command, self).__init__(*args, **kwargs)
|
||||
self.guesspath = DN(DN(DN(DN(DN(DN(DN(__file__)))))))
|
||||
|
||||
def _verify_build_environment(self):
|
||||
# provide a local build env. This will be extended later to include non local
|
||||
if BuildEnvironment.objects.count() == 0:
|
||||
BuildEnvironment.objects.create(betype=BuildEnvironment.TYPE_LOCAL)
|
||||
|
||||
# we make sure we have builddir and sourcedir for all defined build envionments
|
||||
for be in BuildEnvironment.objects.all():
|
||||
be.needs_import = False
|
||||
def _verify_be():
|
||||
is_changed = False
|
||||
|
||||
def _update_sourcedir():
|
||||
be.sourcedir = os.environ.get('TOASTER_DIR')
|
||||
return True
|
||||
|
||||
if len(be.sourcedir) == 0:
|
||||
is_changed = _update_sourcedir()
|
||||
|
||||
if not be.sourcedir.startswith("/"):
|
||||
print("\n -- Validation: The layers checkout directory must be set to an absolute path.")
|
||||
is_changed = _update_sourcedir()
|
||||
|
||||
if is_changed:
|
||||
if be.betype == BuildEnvironment.TYPE_LOCAL:
|
||||
be.needs_import = True
|
||||
return True
|
||||
|
||||
def _update_builddir():
|
||||
be.builddir = os.environ.get('TOASTER_DIR')+"/build"
|
||||
return True
|
||||
|
||||
if len(be.builddir) == 0:
|
||||
is_changed = _update_builddir()
|
||||
|
||||
if not be.builddir.startswith("/"):
|
||||
print("\n -- Validation: The build directory must to be set to an absolute path.")
|
||||
is_changed = _update_builddir()
|
||||
|
||||
if is_changed:
|
||||
print("\nBuild configuration saved")
|
||||
be.save()
|
||||
return True
|
||||
|
||||
if be.needs_import:
|
||||
try:
|
||||
print("Loading default settings")
|
||||
call_command("loaddata", "settings")
|
||||
template_conf = os.environ.get("TEMPLATECONF", "")
|
||||
custom_xml_only = os.environ.get("CUSTOM_XML_ONLY")
|
||||
|
||||
if ToasterSetting.objects.filter(name='CUSTOM_XML_ONLY').count() > 0 or custom_xml_only is not None:
|
||||
# only use the custom settings
|
||||
pass
|
||||
elif "poky" in template_conf:
|
||||
print("Loading poky configuration")
|
||||
call_command("loaddata", "poky")
|
||||
else:
|
||||
print("Loading OE-Core configuration")
|
||||
call_command("loaddata", "oe-core")
|
||||
if template_conf:
|
||||
oe_core_path = os.path.realpath(
|
||||
template_conf +
|
||||
"/../")
|
||||
else:
|
||||
print("TEMPLATECONF not found. You may have to"
|
||||
" manually configure layer paths")
|
||||
oe_core_path = input("Please enter the path of"
|
||||
" your openembedded-core "
|
||||
"layer: ")
|
||||
# Update the layer instances of openemebedded-core
|
||||
for layer in Layer.objects.filter(
|
||||
name="openembedded-core",
|
||||
local_source_dir="OE-CORE-LAYER-DIR"):
|
||||
layer.local_path = oe_core_path
|
||||
layer.save()
|
||||
|
||||
# Import the custom fixture if it's present
|
||||
with warnings.catch_warnings():
|
||||
warnings.filterwarnings(
|
||||
action="ignore",
|
||||
message="^.*No fixture named.*$")
|
||||
print("Importing custom settings if present")
|
||||
try:
|
||||
call_command("loaddata", "custom")
|
||||
except:
|
||||
print("NOTE: optional fixture 'custom' not found")
|
||||
|
||||
# we run lsupdates after config update
|
||||
print("\nFetching information from the layer index, "
|
||||
"please wait.\nYou can re-update any time later "
|
||||
"by running bitbake/lib/toaster/manage.py "
|
||||
"lsupdates\n")
|
||||
call_command("lsupdates")
|
||||
|
||||
# we don't look for any other config files
|
||||
return is_changed
|
||||
except Exception as e:
|
||||
print("Failure while trying to setup toaster: %s"
|
||||
% e)
|
||||
traceback.print_exc()
|
||||
|
||||
return is_changed
|
||||
|
||||
while _verify_be():
|
||||
pass
|
||||
return 0
|
||||
|
||||
def _verify_default_settings(self):
|
||||
# verify that default settings are there
|
||||
if ToasterSetting.objects.filter(name='DEFAULT_RELEASE').count() != 1:
|
||||
ToasterSetting.objects.filter(name='DEFAULT_RELEASE').delete()
|
||||
ToasterSetting.objects.get_or_create(name='DEFAULT_RELEASE', value='')
|
||||
return 0
|
||||
|
||||
def _verify_builds_in_progress(self):
|
||||
# we are just starting up. we must not have any builds in progress, or build environments taken
|
||||
for b in BuildRequest.objects.filter(state=BuildRequest.REQ_INPROGRESS):
|
||||
BRError.objects.create(req=b, errtype="toaster",
|
||||
errmsg=
|
||||
"Toaster found this build IN PROGRESS while Toaster started up. This is an inconsistent state, and the build was marked as failed")
|
||||
|
||||
BuildRequest.objects.filter(state=BuildRequest.REQ_INPROGRESS).update(state=BuildRequest.REQ_FAILED)
|
||||
|
||||
BuildEnvironment.objects.update(lock=BuildEnvironment.LOCK_FREE)
|
||||
|
||||
# also mark "In Progress builds as failures"
|
||||
from django.utils import timezone
|
||||
Build.objects.filter(outcome=Build.IN_PROGRESS).update(outcome=Build.FAILED, completed_on=timezone.now())
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
|
||||
def handle(self, **options):
|
||||
retval = 0
|
||||
retval += self._verify_build_environment()
|
||||
retval += self._verify_default_settings()
|
||||
retval += self._verify_builds_in_progress()
|
||||
|
||||
return retval
|
||||
@@ -0,0 +1,275 @@
|
||||
#
|
||||
# SPDX-License-Identifier: GPL-2.0-only
|
||||
#
|
||||
|
||||
from django.core.management.base import BaseCommand
|
||||
from django.db import transaction
|
||||
from django.db.models import Q
|
||||
|
||||
from bldcontrol.bbcontroller import getBuildEnvironmentController
|
||||
from bldcontrol.models import BuildRequest, BuildEnvironment
|
||||
from bldcontrol.models import BRError, BRVariable
|
||||
|
||||
from orm.models import Build, LogMessage, Target
|
||||
|
||||
import logging
|
||||
import traceback
|
||||
import signal
|
||||
import os
|
||||
|
||||
logger = logging.getLogger("toaster")
|
||||
|
||||
|
||||
class Command(BaseCommand):
|
||||
args = ""
|
||||
help = "Schedules and executes build requests as possible. "\
|
||||
"Does not return (interrupt with Ctrl-C)"
|
||||
|
||||
@transaction.atomic
|
||||
def _selectBuildEnvironment(self):
|
||||
bec = getBuildEnvironmentController(lock=BuildEnvironment.LOCK_FREE)
|
||||
bec.be.lock = BuildEnvironment.LOCK_LOCK
|
||||
bec.be.save()
|
||||
return bec
|
||||
|
||||
@transaction.atomic
|
||||
def _selectBuildRequest(self):
|
||||
br = BuildRequest.objects.filter(state=BuildRequest.REQ_QUEUED).first()
|
||||
return br
|
||||
|
||||
def schedule(self):
|
||||
try:
|
||||
# select the build environment and the request to build
|
||||
br = self._selectBuildRequest()
|
||||
if br:
|
||||
br.state = BuildRequest.REQ_INPROGRESS
|
||||
br.save()
|
||||
else:
|
||||
return
|
||||
|
||||
try:
|
||||
bec = self._selectBuildEnvironment()
|
||||
except IndexError as e:
|
||||
# we could not find a BEC; postpone the BR
|
||||
br.state = BuildRequest.REQ_QUEUED
|
||||
br.save()
|
||||
logger.debug("runbuilds: No build env (%s)" % e)
|
||||
return
|
||||
|
||||
logger.info("runbuilds: starting build %s, environment %s" %
|
||||
(br, bec.be))
|
||||
|
||||
# let the build request know where it is being executed
|
||||
br.environment = bec.be
|
||||
br.save()
|
||||
|
||||
# this triggers an async build
|
||||
bec.triggerBuild(br.brbitbake, br.brlayer_set.all(),
|
||||
br.brvariable_set.all(), br.brtarget_set.all(),
|
||||
"%d:%d" % (br.pk, bec.be.pk))
|
||||
|
||||
except Exception as e:
|
||||
logger.error("runbuilds: Error launching build %s" % e)
|
||||
traceback.print_exc()
|
||||
if "[Errno 111] Connection refused" in str(e):
|
||||
# Connection refused, read toaster_server.out
|
||||
errmsg = bec.readServerLogFile()
|
||||
else:
|
||||
errmsg = str(e)
|
||||
|
||||
BRError.objects.create(req=br, errtype=str(type(e)), errmsg=errmsg,
|
||||
traceback=traceback.format_exc())
|
||||
br.state = BuildRequest.REQ_FAILED
|
||||
br.save()
|
||||
bec.be.lock = BuildEnvironment.LOCK_FREE
|
||||
bec.be.save()
|
||||
# Cancel the pending build and report the exception to the UI
|
||||
log_object = LogMessage.objects.create(
|
||||
build = br.build,
|
||||
level = LogMessage.EXCEPTION,
|
||||
message = errmsg)
|
||||
log_object.save()
|
||||
br.build.outcome = Build.FAILED
|
||||
br.build.save()
|
||||
|
||||
def archive(self):
|
||||
for br in BuildRequest.objects.filter(state=BuildRequest.REQ_ARCHIVE):
|
||||
if br.build is None:
|
||||
br.state = BuildRequest.REQ_FAILED
|
||||
else:
|
||||
br.state = BuildRequest.REQ_COMPLETED
|
||||
br.save()
|
||||
|
||||
def cleanup(self):
|
||||
from django.utils import timezone
|
||||
from datetime import timedelta
|
||||
# environments locked for more than 30 seconds
|
||||
# they should be unlocked
|
||||
BuildEnvironment.objects.filter(
|
||||
Q(buildrequest__state__in=[BuildRequest.REQ_FAILED,
|
||||
BuildRequest.REQ_COMPLETED,
|
||||
BuildRequest.REQ_CANCELLING]) &
|
||||
Q(lock=BuildEnvironment.LOCK_LOCK) &
|
||||
Q(updated__lt=timezone.now() - timedelta(seconds=30))
|
||||
).update(lock=BuildEnvironment.LOCK_FREE)
|
||||
|
||||
# update all Builds that were in progress and failed to start
|
||||
for br in BuildRequest.objects.filter(
|
||||
state=BuildRequest.REQ_FAILED,
|
||||
build__outcome=Build.IN_PROGRESS):
|
||||
# transpose the launch errors in ToasterExceptions
|
||||
br.build.outcome = Build.FAILED
|
||||
for brerror in br.brerror_set.all():
|
||||
logger.debug("Saving error %s" % brerror)
|
||||
LogMessage.objects.create(build=br.build,
|
||||
level=LogMessage.EXCEPTION,
|
||||
message=brerror.errmsg)
|
||||
br.build.save()
|
||||
|
||||
# we don't have a true build object here; hence, toasterui
|
||||
# didn't have a change to release the BE lock
|
||||
br.environment.lock = BuildEnvironment.LOCK_FREE
|
||||
br.environment.save()
|
||||
|
||||
# update all BuildRequests without a build created
|
||||
for br in BuildRequest.objects.filter(build=None):
|
||||
br.build = Build.objects.create(project=br.project,
|
||||
completed_on=br.updated,
|
||||
started_on=br.created)
|
||||
br.build.outcome = Build.FAILED
|
||||
try:
|
||||
br.build.machine = br.brvariable_set.get(name='MACHINE').value
|
||||
except BRVariable.DoesNotExist:
|
||||
pass
|
||||
br.save()
|
||||
# transpose target information
|
||||
for brtarget in br.brtarget_set.all():
|
||||
Target.objects.create(build=br.build,
|
||||
target=brtarget.target,
|
||||
task=brtarget.task)
|
||||
# transpose the launch errors in ToasterExceptions
|
||||
for brerror in br.brerror_set.all():
|
||||
LogMessage.objects.create(build=br.build,
|
||||
level=LogMessage.EXCEPTION,
|
||||
message=brerror.errmsg)
|
||||
|
||||
br.build.save()
|
||||
|
||||
# Make sure the LOCK is removed for builds which have been fully
|
||||
# cancelled
|
||||
for br in BuildRequest.objects.filter(
|
||||
Q(build__outcome=Build.CANCELLED) &
|
||||
Q(state=BuildRequest.REQ_CANCELLING) &
|
||||
~Q(environment=None)):
|
||||
br.environment.lock = BuildEnvironment.LOCK_FREE
|
||||
br.environment.save()
|
||||
|
||||
def runbuild(self):
|
||||
try:
|
||||
self.cleanup()
|
||||
except Exception as e:
|
||||
logger.warning("runbuilds: cleanup exception %s" % str(e))
|
||||
|
||||
try:
|
||||
self.archive()
|
||||
except Exception as e:
|
||||
logger.warning("runbuilds: archive exception %s" % str(e))
|
||||
|
||||
try:
|
||||
self.schedule()
|
||||
except Exception as e:
|
||||
logger.warning("runbuilds: schedule exception %s" % str(e))
|
||||
|
||||
# Test to see if a build pre-maturely died due to a bitbake crash
|
||||
def check_dead_builds(self):
|
||||
do_cleanup = False
|
||||
try:
|
||||
for br in BuildRequest.objects.filter(state=BuildRequest.REQ_INPROGRESS):
|
||||
# Get the build directory
|
||||
if br.project.builddir:
|
||||
builddir = br.project.builddir
|
||||
else:
|
||||
builddir = '%s-toaster-%d' % (br.environment.builddir,br.project.id)
|
||||
# Check log to see if there is a recent traceback
|
||||
toaster_ui_log = os.path.join(builddir, 'toaster_ui.log')
|
||||
test_file = os.path.join(builddir, '._toaster_check.txt')
|
||||
os.system("tail -n 50 %s > %s" % (os.path.join(builddir, 'toaster_ui.log'),test_file))
|
||||
traceback_text = ''
|
||||
is_traceback = False
|
||||
with open(test_file,'r') as test_file_fd:
|
||||
test_file_tail = test_file_fd.readlines()
|
||||
for line in test_file_tail:
|
||||
if line.startswith('Traceback (most recent call last):'):
|
||||
traceback_text = line
|
||||
is_traceback = True
|
||||
elif line.startswith('NOTE: ToasterUI waiting for events'):
|
||||
# Ignore any traceback before new build start
|
||||
traceback_text = ''
|
||||
is_traceback = False
|
||||
elif line.startswith('Note: Toaster traceback auto-stop'):
|
||||
# Ignore any traceback before this previous traceback catch
|
||||
traceback_text = ''
|
||||
is_traceback = False
|
||||
elif is_traceback:
|
||||
traceback_text += line
|
||||
# Test the results
|
||||
is_stop = False
|
||||
if is_traceback:
|
||||
# Found a traceback
|
||||
errtype = 'Bitbake crash'
|
||||
errmsg = 'Bitbake crash\n' + traceback_text
|
||||
state = BuildRequest.REQ_FAILED
|
||||
# Clean up bitbake files
|
||||
bitbake_lock = os.path.join(builddir, 'bitbake.lock')
|
||||
if os.path.isfile(bitbake_lock):
|
||||
os.remove(bitbake_lock)
|
||||
bitbake_sock = os.path.join(builddir, 'bitbake.sock')
|
||||
if os.path.isfile(bitbake_sock):
|
||||
os.remove(bitbake_sock)
|
||||
if os.path.isfile(test_file):
|
||||
os.remove(test_file)
|
||||
# Add note to ignore this traceback on next check
|
||||
os.system('echo "Note: Toaster traceback auto-stop" >> %s' % toaster_ui_log)
|
||||
is_stop = True
|
||||
# Add more tests here
|
||||
#elif ...
|
||||
# Stop the build request?
|
||||
if is_stop:
|
||||
brerror = BRError(
|
||||
req = br,
|
||||
errtype = errtype,
|
||||
errmsg = errmsg,
|
||||
traceback = traceback_text,
|
||||
)
|
||||
brerror.save()
|
||||
br.state = state
|
||||
br.save()
|
||||
do_cleanup = True
|
||||
# Do cleanup
|
||||
if do_cleanup:
|
||||
self.cleanup()
|
||||
except Exception as e:
|
||||
logger.error("runbuilds: Error in check_dead_builds %s" % e)
|
||||
|
||||
def handle(self, **options):
|
||||
pidfile_path = os.path.join(os.environ.get("BUILDDIR", "."),
|
||||
".runbuilds.pid")
|
||||
|
||||
with open(pidfile_path, 'w') as pidfile:
|
||||
pidfile.write("%s" % os.getpid())
|
||||
|
||||
# Clean up any stale/failed builds from previous Toaster run
|
||||
self.runbuild()
|
||||
|
||||
signal.signal(signal.SIGUSR1, lambda sig, frame: None)
|
||||
|
||||
while True:
|
||||
sigset = signal.sigtimedwait([signal.SIGUSR1], 5)
|
||||
if sigset:
|
||||
for sig in sigset:
|
||||
# Consume each captured pending event
|
||||
self.runbuild()
|
||||
else:
|
||||
# Check for build exceptions
|
||||
self.check_dead_builds()
|
||||
|
||||
Reference in New Issue
Block a user