Complete Yocto mirror with license table for TQMa6UL (2038-compliance)

- 264 license table entries with exact download URLs (224/264 resolved)
- Complete sources/ directory with all BitBake recipes
- Build configuration: tqma6ul-multi-mba6ulx, spaetzle (musl)
- Full traceability for Softwarefreigabeantrag
- GCC 13.4.0, Linux 6.6.102, U-Boot 2023.04, musl 1.2.4
- License distribution: GPL-2.0 (24), MIT (23), GPL-2.0+ (18), BSD-3 (16)
This commit is contained in:
Siggi (OpenClaw Agent)
2026-03-01 20:58:18 +00:00
commit 16accb6b24
15086 changed files with 1292356 additions and 0 deletions

View File

@@ -0,0 +1,158 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
import os
import logging
import json
from pathlib import Path
from django.http import HttpRequest
BUILDDIR = Path(os.environ.get('BUILDDIR', '/tmp'))
def log_api_request(request, response, view, logger_name='api'):
"""Helper function for LogAPIMixin"""
repjson = {
'view': view,
'path': request.path,
'method': request.method,
'status': response.status_code
}
logger = logging.getLogger(logger_name)
logger.info(
json.dumps(repjson, indent=4, separators=(", ", " : "))
)
def log_view_mixin(view):
def log_view_request(*args, **kwargs):
# get request from args else kwargs
request = None
if len(args) > 0:
for req in args:
if isinstance(req, HttpRequest):
request = req
break
elif request is None:
request = kwargs.get('request')
response = view(*args, **kwargs)
view_name = 'unknown'
if hasattr(request, 'resolver_match'):
if hasattr(request.resolver_match, 'view_name'):
view_name = request.resolver_match.view_name
log_api_request(
request, response, view_name, 'toaster')
return response
return log_view_request
class LogAPIMixin:
"""Logs API requests
tested with:
- APIView
- ModelViewSet
- ReadOnlyModelViewSet
- GenericAPIView
Note: you can set `view_name` attribute in View to override get_view_name()
"""
def get_view_name(self):
if hasattr(self, 'view_name'):
return self.view_name
return super().get_view_name()
def finalize_response(self, request, response, *args, **kwargs):
log_api_request(request, response, self.get_view_name())
return super().finalize_response(request, response, *args, **kwargs)
LOGGING_SETTINGS = {
'version': 1,
'disable_existing_loggers': False,
'filters': {
'require_debug_false': {
'()': 'django.utils.log.RequireDebugFalse'
}
},
'formatters': {
'datetime': {
'format': '%(asctime)s %(levelname)s %(message)s'
},
'verbose': {
'format': '{levelname} {asctime} {module} {name}.{funcName} {process:d} {thread:d} {message}',
'datefmt': "%d/%b/%Y %H:%M:%S",
'style': '{',
},
'api': {
'format': '\n{levelname} {asctime} {name}.{funcName}:\n{message}',
'style': '{'
}
},
'handlers': {
'mail_admins': {
'level': 'ERROR',
'filters': ['require_debug_false'],
'class': 'django.utils.log.AdminEmailHandler'
},
'console': {
'level': 'DEBUG',
'class': 'logging.StreamHandler',
'formatter': 'datetime',
},
'file_django': {
'level': 'INFO',
'class': 'logging.handlers.TimedRotatingFileHandler',
'filename': BUILDDIR / 'toaster_logs/django.log',
'when': 'D', # interval type
'interval': 1, # defaults to 1
'backupCount': 10, # how many files to keep
'formatter': 'verbose',
},
'file_api': {
'level': 'INFO',
'class': 'logging.handlers.TimedRotatingFileHandler',
'filename': BUILDDIR / 'toaster_logs/api.log',
'when': 'D',
'interval': 1,
'backupCount': 10,
'formatter': 'verbose',
},
'file_toaster': {
'level': 'INFO',
'class': 'logging.handlers.TimedRotatingFileHandler',
'filename': BUILDDIR / 'toaster_logs/web.log',
'when': 'D',
'interval': 1,
'backupCount': 10,
'formatter': 'verbose',
},
},
'loggers': {
'django.request': {
'handlers': ['file_django', 'console'],
'level': 'WARN',
'propagate': True,
},
'django': {
'handlers': ['file_django', 'console'],
'level': 'WARNING',
'propogate': True,
},
'toaster': {
'handlers': ['file_toaster'],
'level': 'INFO',
'propagate': False,
},
'api': {
'handlers': ['file_api'],
'level': 'INFO',
'propagate': False,
}
}
}

View File

@@ -0,0 +1,60 @@
#
# SPDX-License-Identifier: GPL-2.0-only
#
from django.core.management.base import BaseCommand
from django.core.exceptions import ObjectDoesNotExist
from orm.models import Build
from django.db import OperationalError
class Command(BaseCommand):
args = '<buildID1 buildID2 .....>'
help = "Deletes selected build(s)"
def add_arguments(self, parser):
parser.add_argument('buildids', metavar='N', type=int, nargs='+',
help="Build ID's to delete")
def handle(self, *args, **options):
for bid in options['buildids']:
try:
b = Build.objects.get(pk = bid)
except ObjectDoesNotExist:
print('build %s does not exist, skipping...' %(bid))
continue
# theoretically, just b.delete() would suffice
# however SQLite runs into problems when you try to
# delete too many rows at once, so we delete some direct
# relationships from Build manually.
for t in b.target_set.all():
t.delete()
for t in b.task_build.all():
t.delete()
for p in b.package_set.all():
p.delete()
for lv in b.layer_version_build.all():
lv.delete()
for v in b.variable_build.all():
v.delete()
for l in b.logmessage_set.all():
l.delete()
# delete the build; some databases might have had problem with migration of the bldcontrol app
retry_count = 0
need_bldcontrol_migration = False
while True:
if retry_count >= 5:
break
retry_count += 1
if need_bldcontrol_migration:
from django.core import management
management.call_command('migrate', 'bldcontrol', interactive=False)
try:
b.delete()
break
except OperationalError as e:
# execute migrations
need_bldcontrol_migration = True

View File

@@ -0,0 +1,579 @@
#
# BitBake Toaster Implementation
#
# Copyright (C) 2018 Wind River Systems
#
# SPDX-License-Identifier: GPL-2.0-only
#
# buildimport: import a project for project specific configuration
#
# Usage:
# (a) Set up Toaster environent
#
# (b) Call buildimport
# $ /path/to/bitbake/lib/toaster/manage.py buildimport \
# --name=$PROJECTNAME \
# --path=$BUILD_DIRECTORY \
# --callback="$CALLBACK_SCRIPT" \
# --command="configure|reconfigure|import"
#
# (c) Return is "|Default_image=%s|Project_id=%d"
#
# (d) Open Toaster to this project using for example:
# $ xdg-open http://localhost:$toaster_port/toastergui/project_specific/$project_id
#
# (e) To delete a project:
# $ /path/to/bitbake/lib/toaster/manage.py buildimport \
# --name=$PROJECTNAME --delete-project
#
# ../bitbake/lib/toaster/manage.py buildimport --name=test --path=`pwd` --callback="" --command=import
from django.core.management.base import BaseCommand
from orm.models import Project, Release, ProjectVariable
from orm.models import Layer, Layer_Version, LayerSource, ProjectLayer
from toastergui.api import scan_layer_content
import os
import re
import os.path
import subprocess
import shutil
# Toaster variable section delimiters
TOASTER_PROLOG = '#=== TOASTER_CONFIG_PROLOG ==='
TOASTER_EPILOG = '#=== TOASTER_CONFIG_EPILOG ==='
# quick development/debugging support
verbose = 2
def _log(msg):
if 1 == verbose:
print(msg)
elif 2 == verbose:
f1=open('/tmp/toaster.log', 'a')
f1.write("|" + msg + "|\n" )
f1.close()
__config_regexp__ = re.compile( r"""
^
(?P<exp>export\s+)?
(?P<var>[a-zA-Z0-9\-_+.${}/~]+?)
(\[(?P<flag>[a-zA-Z0-9\-_+.]+)\])?
\s* (
(?P<colon>:=) |
(?P<lazyques>\?\?=) |
(?P<ques>\?=) |
(?P<append>\+=) |
(?P<prepend>=\+) |
(?P<predot>=\.) |
(?P<postdot>\.=) |
=
) \s*
(?!'[^']*'[^']*'$)
(?!\"[^\"]*\"[^\"]*\"$)
(?P<apo>['\"])
(?P<value>.*)
(?P=apo)
$
""", re.X)
class Command(BaseCommand):
args = "<name> <path> <release>"
help = "Import a command line build directory"
vars = {}
toaster_vars = {}
def add_arguments(self, parser):
parser.add_argument(
'--name', dest='name', required=True,
help='name of the project',
)
parser.add_argument(
'--path', dest='path', required=True,
help='path to the project',
)
parser.add_argument(
'--release', dest='release', required=False,
help='release for the project',
)
parser.add_argument(
'--callback', dest='callback', required=False,
help='callback for project config update',
)
parser.add_argument(
'--delete-project', dest='delete_project', required=False,
help='delete this project from the database',
)
parser.add_argument(
'--command', dest='command', required=False,
help='command (configure,reconfigure,import)',
)
def get_var(self, varname):
value = self.vars.get(varname, '')
if value:
varrefs = re.findall('\${([^}]*)}', value)
for ref in varrefs:
if ref in self.vars:
value = value.replace('${%s}' % ref, self.vars[ref])
return value
# Extract the bb variables from a conf file
def scan_conf(self,fn):
vars = self.vars
toaster_vars = self.toaster_vars
#_log("scan_conf:%s" % fn)
if not os.path.isfile(fn):
return
f = open(fn, 'r')
#statements = ast.StatementGroup()
lineno = 0
is_toaster_section = False
while True:
lineno = lineno + 1
s = f.readline()
if not s:
break
w = s.strip()
# skip empty lines
if not w:
continue
# evaluate Toaster sections
if w.startswith(TOASTER_PROLOG):
is_toaster_section = True
continue
if w.startswith(TOASTER_EPILOG):
is_toaster_section = False
continue
s = s.rstrip()
while s[-1] == '\\':
s2 = f.readline().strip()
lineno = lineno + 1
if (not s2 or s2 and s2[0] != "#") and s[0] == "#" :
echo("There is a confusing multiline, partially commented expression on line %s of file %s (%s).\nPlease clarify whether this is all a comment or should be parsed." % (lineno, fn, s))
s = s[:-1] + s2
# skip comments
if s[0] == '#':
continue
# process the line for just assignments
m = __config_regexp__.match(s)
if m:
groupd = m.groupdict()
var = groupd['var']
value = groupd['value']
if groupd['lazyques']:
if not var in vars:
vars[var] = value
continue
if groupd['ques']:
if not var in vars:
vars[var] = value
continue
# preset empty blank for remaining operators
if not var in vars:
vars[var] = ''
if groupd['append']:
vars[var] += value
elif groupd['prepend']:
vars[var] = "%s%s" % (value,vars[var])
elif groupd['predot']:
vars[var] = "%s %s" % (value,vars[var])
elif groupd['postdot']:
vars[var] = "%s %s" % (vars[var],value)
else:
vars[var] = "%s" % (value)
# capture vars in a Toaster section
if is_toaster_section:
toaster_vars[var] = vars[var]
# DONE WITH PARSING
f.close()
self.vars = vars
self.toaster_vars = toaster_vars
# Update the scanned project variables
def update_project_vars(self,project,name):
pv, create = ProjectVariable.objects.get_or_create(project = project, name = name)
if (not name in self.vars.keys()) or (not self.vars[name]):
self.vars[name] = pv.value
else:
if pv.value != self.vars[name]:
pv.value = self.vars[name]
pv.save()
# Find the git version of the installation
def find_layer_dir_version(self,path):
# * rocko ...
install_version = ''
cwd = os.getcwd()
os.chdir(path)
p = subprocess.Popen(['git', 'branch', '-av'], stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
out, err = p.communicate()
out = out.decode("utf-8")
for branch in out.split('\n'):
if ('*' == branch[0:1]) and ('no branch' not in branch):
install_version = re.sub(' .*','',branch[2:])
break
if 'remotes/m/master' in branch:
install_version = re.sub('.*base/','',branch)
break
os.chdir(cwd)
return install_version
# Compute table of the installation's registered layer versions (branch or commit)
def find_layer_dir_versions(self,INSTALL_URL_PREFIX):
lv_dict = {}
layer_versions = Layer_Version.objects.all()
for lv in layer_versions:
layer = Layer.objects.filter(pk=lv.layer.pk)[0]
if layer.vcs_url:
url_short = layer.vcs_url.replace(INSTALL_URL_PREFIX,'')
else:
url_short = ''
# register the core, branch, and the version variations
lv_dict["%s,%s,%s" % (url_short,lv.dirpath,'')] = (lv.id,layer.name)
lv_dict["%s,%s,%s" % (url_short,lv.dirpath,lv.branch)] = (lv.id,layer.name)
lv_dict["%s,%s,%s" % (url_short,lv.dirpath,lv.commit)] = (lv.id,layer.name)
#_log(" (%s,%s,%s|%s) = (%s,%s)" % (url_short,lv.dirpath,lv.branch,lv.commit,lv.id,layer.name))
return lv_dict
# Apply table of all layer versions
def extract_bblayers(self):
# set up the constants
bblayer_str = self.get_var('BBLAYERS')
TOASTER_DIR = os.environ.get('TOASTER_DIR')
INSTALL_CLONE_PREFIX = os.path.dirname(TOASTER_DIR) + "/"
TOASTER_CLONE_PREFIX = TOASTER_DIR + "/_toaster_clones/"
INSTALL_URL_PREFIX = ''
layers = Layer.objects.filter(name='openembedded-core')
for layer in layers:
if layer.vcs_url:
INSTALL_URL_PREFIX = layer.vcs_url
break
INSTALL_URL_PREFIX = INSTALL_URL_PREFIX.replace("/poky","/")
INSTALL_VERSION_DIR = TOASTER_DIR
INSTALL_URL_POSTFIX = INSTALL_URL_PREFIX.replace(':','_')
INSTALL_URL_POSTFIX = INSTALL_URL_POSTFIX.replace('/','_')
INSTALL_URL_POSTFIX = "%s_%s" % (TOASTER_CLONE_PREFIX,INSTALL_URL_POSTFIX)
# get the set of available layer:layer_versions
lv_dict = self.find_layer_dir_versions(INSTALL_URL_PREFIX)
# compute the layer matches
layers_list = []
for line in bblayer_str.split(' '):
if not line:
continue
if line.endswith('/local'):
continue
# isolate the repo
layer_path = line
line = line.replace(INSTALL_URL_POSTFIX,'').replace(INSTALL_CLONE_PREFIX,'').replace('/layers/','/').replace('/poky/','/')
# isolate the sub-path
path_index = line.rfind('/')
if path_index > 0:
sub_path = line[path_index+1:]
line = line[0:path_index]
else:
sub_path = ''
# isolate the version
if TOASTER_CLONE_PREFIX in layer_path:
is_toaster_clone = True
# extract version from name syntax
version_index = line.find('_')
if version_index > 0:
version = line[version_index+1:]
line = line[0:version_index]
else:
version = ''
_log("TOASTER_CLONE(%s/%s), version=%s" % (line,sub_path,version))
else:
is_toaster_clone = False
# version is from the installation
version = self.find_layer_dir_version(layer_path)
_log("LOCAL_CLONE(%s/%s), version=%s" % (line,sub_path,version))
# capture the layer information into layers_list
layers_list.append( (line,sub_path,version,layer_path,is_toaster_clone) )
return layers_list,lv_dict
#
def find_import_release(self,layers_list,lv_dict,default_release):
# poky,meta,rocko => 4;openembedded-core
release = default_release
for line,path,version,layer_path,is_toaster_clone in layers_list:
key = "%s,%s,%s" % (line,path,version)
if key in lv_dict:
lv_id = lv_dict[key]
if 'openembedded-core' == lv_id[1]:
_log("Find_import_release(%s):version=%s,Toaster=%s" % (lv_id[1],version,is_toaster_clone))
# only versions in Toaster managed layers are accepted
if not is_toaster_clone:
break
try:
release = Release.objects.get(name=version)
except:
pass
break
_log("Find_import_release:RELEASE=%s" % release.name)
return release
# Apply the found conf layers
def apply_conf_bblayers(self,layers_list,lv_dict,project,release=None):
for line,path,version,layer_path,is_toaster_clone in layers_list:
# Assert release promote if present
if release:
version = release
# try to match the key to a layer_version
key = "%s,%s,%s" % (line,path,version)
key_short = "%s,%s,%s" % (line,path,'')
lv_id = ''
if key in lv_dict:
lv_id = lv_dict[key]
lv = Layer_Version.objects.get(pk=int(lv_id[0]))
pl,created = ProjectLayer.objects.get_or_create(project=project,
layercommit=lv)
pl.optional=False
pl.save()
_log(" %s => %s;%s" % (key,lv_id[0],lv_id[1]))
elif key_short in lv_dict:
lv_id = lv_dict[key_short]
lv = Layer_Version.objects.get(pk=int(lv_id[0]))
pl,created = ProjectLayer.objects.get_or_create(project=project,
layercommit=lv)
pl.optional=False
pl.save()
_log(" %s ?> %s" % (key,lv_dict[key_short]))
else:
_log("%s <= %s" % (key,layer_path))
found = False
# does local layer already exist in this project?
try:
for pl in ProjectLayer.objects.filter(project=project):
if pl.layercommit.layer.local_source_dir == layer_path:
found = True
_log(" Project Local Layer found!")
except Exception as e:
_log("ERROR: Local Layer '%s'" % e)
pass
if not found:
# Does Layer name+path already exist?
try:
layer_name_base = os.path.basename(layer_path)
_log("Layer_lookup: try '%s','%s'" % (layer_name_base,layer_path))
layer = Layer.objects.get(name=layer_name_base,local_source_dir = layer_path)
# Found! Attach layer_version and ProjectLayer
layer_version = Layer_Version.objects.create(
layer=layer,
project=project,
layer_source=LayerSource.TYPE_IMPORTED)
layer_version.save()
pl,created = ProjectLayer.objects.get_or_create(project=project,
layercommit=layer_version)
pl.optional=False
pl.save()
found = True
# add layer contents to this layer version
scan_layer_content(layer,layer_version)
_log(" Parent Local Layer found in db!")
except Exception as e:
_log("Layer_exists_test_failed: Local Layer '%s'" % e)
pass
if not found:
# Insure that layer path exists, in case of user typo
if not os.path.isdir(layer_path):
_log("ERROR:Layer path '%s' not found" % layer_path)
continue
# Add layer to db and attach project to it
layer_name_base = os.path.basename(layer_path)
# generate a unique layer name
layer_name_matches = {}
for layer in Layer.objects.filter(name__contains=layer_name_base):
layer_name_matches[layer.name] = '1'
layer_name_idx = 0
layer_name_test = layer_name_base
while layer_name_test in layer_name_matches.keys():
layer_name_idx += 1
layer_name_test = "%s_%d" % (layer_name_base,layer_name_idx)
# create the layer and layer_verion objects
layer = Layer.objects.create(name=layer_name_test)
layer.local_source_dir = layer_path
layer_version = Layer_Version.objects.create(
layer=layer,
project=project,
layer_source=LayerSource.TYPE_IMPORTED)
layer.save()
layer_version.save()
pl,created = ProjectLayer.objects.get_or_create(project=project,
layercommit=layer_version)
pl.optional=False
pl.save()
# register the layer's content
_log(" Local Layer Add content")
scan_layer_content(layer,layer_version)
_log(" Local Layer Added '%s'!" % layer_name_test)
# Scan the project's conf files (if any)
def scan_conf_variables(self,project_path):
self.vars['TOPDIR'] = project_path
# scan the project's settings, add any new layers or variables
if os.path.isfile("%s/conf/local.conf" % project_path):
self.scan_conf("%s/conf/local.conf" % project_path)
self.scan_conf("%s/conf/bblayers.conf" % project_path)
# Import then disable old style Toaster conf files (before 'merged_attr')
old_toaster_local = "%s/conf/toaster.conf" % project_path
if os.path.isfile(old_toaster_local):
self.scan_conf(old_toaster_local)
shutil.move(old_toaster_local, old_toaster_local+"_old")
old_toaster_layer = "%s/conf/toaster-bblayers.conf" % project_path
if os.path.isfile(old_toaster_layer):
self.scan_conf(old_toaster_layer)
shutil.move(old_toaster_layer, old_toaster_layer+"_old")
# Scan the found conf variables (if any)
def apply_conf_variables(self,project,layers_list,lv_dict,release=None):
if self.vars:
# Catch vars relevant to Toaster (in case no Toaster section)
self.update_project_vars(project,'DISTRO')
self.update_project_vars(project,'MACHINE')
self.update_project_vars(project,'IMAGE_INSTALL:append')
self.update_project_vars(project,'IMAGE_FSTYPES')
self.update_project_vars(project,'PACKAGE_CLASSES')
# These vars are typically only assigned by Toaster
#self.update_project_vars(project,'DL_DIR')
#self.update_project_vars(project,'SSTATE_DIR')
# Assert found Toaster vars
for var in self.toaster_vars.keys():
pv, create = ProjectVariable.objects.get_or_create(project = project, name = var)
pv.value = self.toaster_vars[var]
_log("* Add/update Toaster var '%s' = '%s'" % (pv.name,pv.value))
pv.save()
# Assert found BBLAYERS
if 0 < verbose:
for pl in ProjectLayer.objects.filter(project=project):
release_name = 'None' if not pl.layercommit.release else pl.layercommit.release.name
print(" BEFORE:ProjectLayer=%s,%s,%s,%s" % (pl.layercommit.layer.name,release_name,pl.layercommit.branch,pl.layercommit.commit))
self.apply_conf_bblayers(layers_list,lv_dict,project,release)
if 0 < verbose:
for pl in ProjectLayer.objects.filter(project=project):
release_name = 'None' if not pl.layercommit.release else pl.layercommit.release.name
print(" AFTER :ProjectLayer=%s,%s,%s,%s" % (pl.layercommit.layer.name,release_name,pl.layercommit.branch,pl.layercommit.commit))
def handle(self, *args, **options):
project_name = options['name']
project_path = options['path']
project_callback = options['callback'] if options['callback'] else ''
release_name = options['release'] if options['release'] else ''
#
# Delete project
#
if options['delete_project']:
try:
print("Project '%s' delete from Toaster database" % (project_name))
project = Project.objects.get(name=project_name)
# TODO: deep project delete
project.delete()
print("Project '%s' Deleted" % (project_name))
return
except Exception as e:
print("Project '%s' not found, not deleted (%s)" % (project_name,e))
return
#
# Create/Update/Import project
#
# See if project (by name) exists
project = None
try:
# Project already exists
project = Project.objects.get(name=project_name)
except Exception as e:
pass
# Find the installation's default release
default_release = Release.objects.get(id=1)
# SANITY: if 'reconfig' but project does not exist (deleted externally), switch to 'import'
if ("reconfigure" == options['command']) and project is None:
options['command'] = 'import'
# 'Configure':
if "configure" == options['command']:
# Note: ignore any existing conf files
# create project, SANITY: reuse any project of same name
project = Project.objects.create_project(project_name,default_release,project)
# 'Re-configure':
if "reconfigure" == options['command']:
# Scan the directory's conf files
self.scan_conf_variables(project_path)
# Scan the layer list
layers_list,lv_dict = self.extract_bblayers()
# Apply any new layers or variables
self.apply_conf_variables(project,layers_list,lv_dict)
# 'Import':
if "import" == options['command']:
# Scan the directory's conf files
self.scan_conf_variables(project_path)
# Remove these Toaster controlled variables
for var in ('DL_DIR','SSTATE_DIR'):
self.vars.pop(var, None)
self.toaster_vars.pop(var, None)
# Scan the layer list
layers_list,lv_dict = self.extract_bblayers()
# Find the directory's release, and promote to default_release if local paths
release = self.find_import_release(layers_list,lv_dict,default_release)
# create project, SANITY: reuse any project of same name
project = Project.objects.create_project(project_name,release,project, imported=True)
# Apply any new layers or variables
self.apply_conf_variables(project,layers_list,lv_dict,release)
# WORKAROUND: since we now derive the release, redirect 'newproject_specific' to 'project_specific'
project.set_variable('INTERNAL_PROJECT_SPECIFIC_SKIPRELEASE','1')
# Set up the project's meta data
project.builddir = project_path
project.merged_attr = True
project.set_variable(Project.PROJECT_SPECIFIC_CALLBACK,project_callback)
project.set_variable(Project.PROJECT_SPECIFIC_STATUS,Project.PROJECT_SPECIFIC_EDIT)
if ("configure" == options['command']) or ("import" == options['command']):
# preset the mode and default image recipe
project.set_variable(Project.PROJECT_SPECIFIC_ISNEW,Project.PROJECT_SPECIFIC_NEW)
project.set_variable(Project.PROJECT_SPECIFIC_DEFAULTIMAGE,"core-image-minimal")
# Assert any extended/custom actions or variables for new non-Toaster projects
if not len(self.toaster_vars):
pass
else:
project.set_variable(Project.PROJECT_SPECIFIC_ISNEW,Project.PROJECT_SPECIFIC_NONE)
# Save the updated Project
project.save()
_log("Buildimport:project='%s' at '%d'" % (project_name,project.id))
if ('DEFAULT_IMAGE' in self.vars) and (self.vars['DEFAULT_IMAGE']):
print("|Default_image=%s|Project_id=%d" % (self.vars['DEFAULT_IMAGE'],project.id))
else:
print("|Project_id=%d" % (project.id))

View File

@@ -0,0 +1,16 @@
#
# SPDX-License-Identifier: GPL-2.0-only
#
from django.core.management.base import BaseCommand
from orm.models import Build
class Command(BaseCommand):
args = ""
help = "Lists current builds"
def handle(self,**options):
for b in Build.objects.all():
print("%d: %s %s %s" % (b.pk, b.machine, b.distro, ",".join([x.target for x in b.target_set.all()])))

View File

@@ -0,0 +1,57 @@
#!/usr/bin/env python3
#
# BitBake Toaster Implementation
#
# Copyright (C) 2015 Intel Corporation
#
# SPDX-License-Identifier: GPL-2.0-only
#
"""Custom management command checksocket."""
import errno
import socket
from django.core.management.base import BaseCommand, CommandError
from django.utils.encoding import force_str
DEFAULT_ADDRPORT = "0.0.0.0:8000"
class Command(BaseCommand):
"""Custom management command."""
help = 'Check if Toaster can listen on address:port'
def add_arguments(self, parser):
parser.add_argument('addrport', nargs='?', default=DEFAULT_ADDRPORT,
help='ipaddr:port to check, %s by default' % \
DEFAULT_ADDRPORT)
def handle(self, *args, **options):
addrport = options['addrport']
if ':' not in addrport:
raise CommandError('Invalid addr:port specified: %s' % addrport)
splitted = addrport.split(':')
try:
splitted[1] = int(splitted[1])
except ValueError:
raise CommandError('Invalid port specified: %s' % splitted[1])
self.stdout.write('Check if toaster can listen on %s' % addrport)
try:
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
sock.bind(tuple(splitted))
except (socket.error, OverflowError) as err:
errors = {
errno.EACCES: 'You don\'t have permission to access port %s' \
% splitted[1],
errno.EADDRINUSE: 'Port %s is already in use' % splitted[1],
errno.EADDRNOTAVAIL: 'IP address can\'t be assigned to',
}
if hasattr(err, 'errno') and err.errno in errors:
errtext = errors[err.errno]
else:
errtext = force_str(err)
raise CommandError(errtext)
self.stdout.write("OK")

View File

@@ -0,0 +1,62 @@
#
# SPDX-License-Identifier: GPL-2.0-only
#
from django.core.management.base import BaseCommand
from django.test.client import Client
import os, sys, re
import requests
from django.conf import settings
# pylint: disable=E1103
# Instance of 'WSGIRequest' has no 'status_code' member
# (but some types could not be inferred) (maybe-no-member)
class Command(BaseCommand):
help = "Test the response time for all toaster urls"
def handle(self, *args, **options):
root_urlconf = __import__(settings.ROOT_URLCONF)
patterns = root_urlconf.urls.urlpatterns
global full_url
for pat in patterns:
if pat.__class__.__name__ == 'RegexURLResolver':
url_root_res = str(pat).split('^')[1].replace('>', '')
if 'gui' in url_root_res:
for url_patt in pat.url_patterns:
full_url = self.get_full_url(url_patt, url_root_res)
info = self.url_info(full_url)
status_code = info[0]
load_time = info[1]
print('Trying \'' + full_url + '\', ' + str(status_code) + ', ' + str(load_time))
def get_full_url(self, url_patt, url_root_res):
full_url = str(url_patt).split('^')[1].replace('$>', '').replace('(?P<file_path>(?:/[', '/bin/busybox').replace('.*', '')
full_url = str(url_root_res + full_url)
full_url = re.sub('\(\?P<.*?>\\\d\+\)', '1', full_url)
full_url = 'http://localhost:8000/' + full_url
return full_url
def url_info(self, full_url):
client = Client()
info = []
try:
resp = client.get(full_url, follow = True)
except Exception as e_status_code:
self.error('Url: %s, error: %s' % (full_url, e_status_code))
resp = type('object', (), {'status_code':0, 'content': str(e_status_code)})
status_code = resp.status_code
info.append(status_code)
try:
req = requests.get(full_url)
except Exception as e_load_time:
self.error('Url: %s, error: %s' % (full_url, e_load_time))
load_time = req.elapsed
info.append(load_time)
return info
def error(self, *args):
for arg in args:
print(arg, end=' ', file=sys.stderr)
print(file=sys.stderr)

View File

@@ -0,0 +1,348 @@
#
# BitBake Toaster Implementation
#
# Copyright (C) 2013 Intel Corporation
#
# SPDX-License-Identifier: GPL-2.0-only
#
# Django settings for Toaster project.
import os
from pathlib import Path
from toastermain.logs import LOGGING_SETTINGS
DEBUG = True
# Set to True to see the SQL queries in console
SQL_DEBUG = False
if os.environ.get("TOASTER_SQLDEBUG", None) is not None:
SQL_DEBUG = True
ADMINS = (
# ('Your Name', 'your_email@example.com'),
)
MANAGERS = ADMINS
TOASTER_SQLITE_DEFAULT_DIR = os.environ.get('TOASTER_DIR')
DATABASES = {
'default': {
# Add 'postgresql_psycopg2', 'mysql', 'sqlite3' or 'oracle'.
'ENGINE': 'django.db.backends.sqlite3',
# DB name or full path to database file if using sqlite3.
'NAME': "%s/toaster.sqlite" % TOASTER_SQLITE_DEFAULT_DIR,
'USER': '',
'PASSWORD': '',
#'HOST': '127.0.0.1', # e.g. mysql server
#'PORT': '3306', # e.g. mysql port
}
}
# New in Django 3.2
DEFAULT_AUTO_FIELD = 'django.db.models.BigAutoField'
# Needed when Using sqlite especially to add a longer timeout for waiting
# for the database lock to be released
# https://docs.djangoproject.com/en/1.6/ref/databases/#database-is-locked-errors
if 'sqlite' in DATABASES['default']['ENGINE']:
DATABASES['default']['OPTIONS'] = { 'timeout': 20 }
# Update as of django 1.8.16 release, the '*' is needed to allow us to connect while running
# on hosts without explicitly setting the fqdn for the toaster server.
# See https://docs.djangoproject.com/en/dev/ref/settings/ for info on ALLOWED_HOSTS
# Previously this setting was not enforced if DEBUG was set but it is now.
# The previous behavior was such that ALLOWED_HOSTS defaulted to ['localhost','127.0.0.1','::1']
# and if you bound to 0.0.0.0:<port #> then accessing toaster as localhost or fqdn would both work.
# To have that same behavior, with a fqdn explicitly enabled you would set
# ALLOWED_HOSTS= ['localhost','127.0.0.1','::1','myserver.mycompany.com'] for
# Django >= 1.8.16. By default, we are not enforcing this restriction in
# DEBUG mode.
if DEBUG is True:
# this will allow connection via localhost,hostname, or fqdn
ALLOWED_HOSTS = ['*']
# Local time zone for this installation. Choices can be found here:
# http://en.wikipedia.org/wiki/List_of_tz_zones_by_name
# although not all choices may be available on all operating systems.
# In a Windows environment this must be set to your system time zone.
# Always use local computer's time zone, find
import hashlib
if 'TZ' in os.environ:
TIME_ZONE = os.environ['TZ']
else:
# need to read the /etc/localtime file which is the libc standard
# and do a reverse-mapping to /usr/share/zoneinfo/;
# since the timezone may match any number of identical timezone definitions,
zonefilelist = {}
ZONEINFOPATH = '/usr/share/zoneinfo/'
for dirpath, dirnames, filenames in os.walk(ZONEINFOPATH):
for fn in filenames:
filepath = os.path.join(dirpath, fn)
zonename = filepath.lstrip(ZONEINFOPATH).strip()
try:
import pytz
from pytz.exceptions import UnknownTimeZoneError
try:
if pytz.timezone(zonename) is not None:
with open(filepath, 'rb') as f:
zonefilelist[hashlib.md5(f.read()).hexdigest()] = zonename
except UnknownTimeZoneError as ValueError:
# we expect timezone failures here, just move over
pass
except ImportError:
with open(filepath, 'rb') as f:
zonefilelist[hashlib.md5(f.read()).hexdigest()] = zonename
with open('/etc/localtime', 'rb') as f:
TIME_ZONE = zonefilelist[hashlib.md5(f.read()).hexdigest()]
# Language code for this installation. All choices can be found here:
# http://www.i18nguy.com/unicode/language-identifiers.html
LANGUAGE_CODE = 'en-us'
SITE_ID = 1
# If you set this to False, Django will make some optimizations so as not
# to load the internationalization machinery.
USE_I18N = True
# If you set this to False, Django will not use timezone-aware datetimes.
USE_TZ = True
# Absolute filesystem path to the directory that will hold user-uploaded files.
# Example: "/var/www/example.com/media/"
MEDIA_ROOT = ''
# URL that handles the media served from MEDIA_ROOT. Make sure to use a
# trailing slash.
# Examples: "http://example.com/media/", "http://media.example.com/"
MEDIA_URL = ''
# Absolute path to the directory static files should be collected to.
# Don't put anything in this directory yourself; store your static files
# in apps' "static/" subdirectories and in STATICFILES_DIRS.
# Example: "/var/www/example.com/static/"
STATIC_ROOT = ''
# URL prefix for static files.
# Example: "http://example.com/static/", "http://static.example.com/"
STATIC_URL = '/static/'
# Additional locations of static files
STATICFILES_DIRS = (
# Put strings here, like "/home/html/static" or "C:/www/django/static".
# Always use forward slashes, even on Windows.
# Don't forget to use absolute paths, not relative paths.
)
# List of finder classes that know how to find static files in
# various locations.
STATICFILES_FINDERS = (
'django.contrib.staticfiles.finders.FileSystemFinder',
'django.contrib.staticfiles.finders.AppDirectoriesFinder',
# 'django.contrib.staticfiles.finders.DefaultStorageFinder',
)
# Make this unique, and don't share it with anybody.
SECRET_KEY = 'NOT_SUITABLE_FOR_HOSTED_DEPLOYMENT'
TMPDIR = os.environ.get('TOASTER_DJANGO_TMPDIR', '/tmp')
class InvalidString(str):
def __mod__(self, other):
from django.template.base import TemplateSyntaxError
raise TemplateSyntaxError(
"Undefined variable or unknown value for: \"%s\"" % other)
TEMPLATES = [
{
'BACKEND': 'django.template.backends.django.DjangoTemplates',
'DIRS': [
# Put strings here, like "/home/html/django_templates" or "C:/www/django/templates".
# Always use forward slashes, even on Windows.
# Don't forget to use absolute paths, not relative paths.
],
'OPTIONS': {
'context_processors': [
# Insert your TEMPLATE_CONTEXT_PROCESSORS here or use this
# list if you haven't customized them:
'django.contrib.auth.context_processors.auth',
'django.template.context_processors.debug',
'django.template.context_processors.i18n',
'django.template.context_processors.media',
'django.template.context_processors.static',
'django.template.context_processors.tz',
'django.contrib.messages.context_processors.messages',
# Custom
'django.template.context_processors.request',
'toastergui.views.managedcontextprocessor',
],
'loaders': [
# List of callables that know how to import templates from various sources.
'django.template.loaders.filesystem.Loader',
'django.template.loaders.app_directories.Loader',
#'django.template.loaders.eggs.Loader',
],
# https://docs.djangoproject.com/en/4.2/ref/templates/api/#how-invalid-variables-are-handled
# Generally, string_if_invalid should only be enabled in order to debug
# a specific template problem, then cleared once debugging is complete.
# If you assign a value other than '' to string_if_invalid,
# you will experience rendering problems with these templates and sites.
# 'string_if_invalid': InvalidString("%s"),
'string_if_invalid': "",
'debug': DEBUG,
},
},
]
MIDDLEWARE = [
'django.middleware.common.CommonMiddleware',
'django.contrib.sessions.middleware.SessionMiddleware',
'django.middleware.csrf.CsrfViewMiddleware',
'django.contrib.auth.middleware.AuthenticationMiddleware',
'django.contrib.messages.middleware.MessageMiddleware',
'django.contrib.auth.middleware.AuthenticationMiddleware',
'django.contrib.messages.middleware.MessageMiddleware',
'django.contrib.sessions.middleware.SessionMiddleware',
]
CACHES = {
# 'default': {
# 'BACKEND': 'django.core.cache.backends.memcached.MemcachedCache',
# 'LOCATION': '127.0.0.1:11211',
# },
'default': {
'BACKEND': 'django.core.cache.backends.filebased.FileBasedCache',
'LOCATION': '%s/toaster_cache_%d' % (TMPDIR, os.getuid()),
'TIMEOUT': 1,
}
}
from os.path import dirname as DN
SITE_ROOT=DN(DN(os.path.abspath(__file__)))
import subprocess
TOASTER_BRANCH = subprocess.Popen('git branch | grep "^* " | tr -d "* "', cwd = SITE_ROOT, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()[0]
TOASTER_REVISION = subprocess.Popen('git rev-parse HEAD ', cwd = SITE_ROOT, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()[0]
ROOT_URLCONF = 'toastermain.urls'
# Python dotted path to the WSGI application used by Django's runserver.
WSGI_APPLICATION = 'toastermain.wsgi.application'
INSTALLED_APPS = (
'django.contrib.auth',
'django.contrib.contenttypes',
'django.contrib.messages',
'django.contrib.sessions',
'django.contrib.admin',
'django.contrib.staticfiles',
# Uncomment the next line to enable admin documentation:
# 'django.contrib.admindocs',
'django.contrib.humanize',
'bldcollector',
'toastermain',
# 3rd-lib
"log_viewer",
)
INTERNAL_IPS = ['127.0.0.1', '192.168.2.28']
# Load django-fresh is TOASTER_DEVEL is set, and the module is available
FRESH_ENABLED = False
if os.environ.get('TOASTER_DEVEL', None) is not None:
try:
import fresh
MIDDLEWARE = ["fresh.middleware.FreshMiddleware",] + MIDDLEWARE
INSTALLED_APPS = INSTALLED_APPS + ('fresh',)
FRESH_ENABLED = True
except:
pass
DEBUG_PANEL_ENABLED = False
if os.environ.get('TOASTER_DEVEL', None) is not None:
try:
import debug_toolbar, debug_panel
MIDDLEWARE = ['debug_panel.middleware.DebugPanelMiddleware',] + MIDDLEWARE
#MIDDLEWARE = MIDDLEWARE + ['debug_toolbar.middleware.DebugToolbarMiddleware',]
INSTALLED_APPS = INSTALLED_APPS + ('debug_toolbar','debug_panel',)
DEBUG_PANEL_ENABLED = True
# this cache backend will be used by django-debug-panel
CACHES['debug-panel'] = {
'BACKEND': 'django.core.cache.backends.filebased.FileBasedCache',
'LOCATION': '/var/tmp/debug-panel-cache',
'TIMEOUT': 300,
'OPTIONS': {
'MAX_ENTRIES': 200
}
}
except:
pass
SOUTH_TESTS_MIGRATE = False
# We automatically detect and install applications here if
# they have a 'models.py' or 'views.py' file
import os
currentdir = os.path.dirname(__file__)
for t in os.walk(os.path.dirname(currentdir)):
modulename = os.path.basename(t[0])
#if we have a virtualenv skip it to avoid incorrect imports
if 'VIRTUAL_ENV' in os.environ and os.environ['VIRTUAL_ENV'] in t[0]:
continue
if ("views.py" in t[2] or "models.py" in t[2]) and not modulename in INSTALLED_APPS:
INSTALLED_APPS = INSTALLED_APPS + (modulename,)
# A sample logging configuration. The only tangible logging
# performed by this configuration is to send an email to
# the site admins on every HTTP 500 error when DEBUG=False.
# See http://docs.djangoproject.com/en/dev/topics/logging for
# more details on how to customize your logging configuration.
LOGGING = LOGGING_SETTINGS
# Build paths inside the project like this: BASE_DIR / 'subdir'.
BUILDDIR = os.environ.get("BUILDDIR", TMPDIR)
# LOG VIEWER
# https://pypi.org/project/django-log-viewer/
LOG_VIEWER_FILES_PATTERN = '*.log*'
LOG_VIEWER_FILES_DIR = os.path.join(BUILDDIR, "toaster_logs/")
LOG_VIEWER_PAGE_LENGTH = 25 # total log lines per-page
LOG_VIEWER_MAX_READ_LINES = 100000 # total log lines will be read
LOG_VIEWER_PATTERNS = ['INFO', 'DEBUG', 'WARNING', 'ERROR', 'CRITICAL']
# Optionally you can set the next variables in order to customize the admin:
LOG_VIEWER_FILE_LIST_TITLE = "Logs list"
if DEBUG and SQL_DEBUG:
LOGGING['loggers']['django.db.backends'] = {
'level': 'DEBUG',
'handlers': ['console'],
}
# If we're using sqlite, we need to tweak the performance a bit
from django.db.backends.signals import connection_created
def activate_synchronous_off(sender, connection, **kwargs):
if connection.vendor == 'sqlite':
cursor = connection.cursor()
cursor.execute('PRAGMA synchronous = 0;')
connection_created.connect(activate_synchronous_off)
#

View File

@@ -0,0 +1,45 @@
#
# BitBake Toaster Implementation
#
# Copyright (C) 2016 Intel Corporation
#
# SPDX-License-Identifier: GPL-2.0-only
#
# See Django documentation for more information about deployment
# https://docs.djangoproject.com/en/1.8/howto/deployment/checklist/
# Toaster production settings example overlay
# To use this copy this example to "settings_production.py" and set in your
# environment DJANGO_SETTINGS_MODULE=toastermain.settings_production
# This can be permanently set in a new .wsgi file
from toastermain.settings import * # NOQA
# Set this value!
SECRET_KEY = None
# Switch off any debugging
DEBUG = True
TEMPLATE_DEBUG = DEBUG
DATABASES = {
'default': {
'ENGINE': 'django.db.backends.mysql',
'NAME': 'toaster_data',
'USER': 'toaster',
'PASSWORD': 'yourpasswordhere',
'HOST': '127.0.0.1',
'PORT': '3306',
}
}
# Location where static files will be placed by "manage.py collectstatic"
STATIC_ROOT = '/var/www/static-toaster/'
# URL prefix for static files.
STATIC_URL = '/static-toaster/'
# Hosts that Django will serve
# https://docs.djangoproject.com/en/1.8/ref/settings/#std:setting-ALLOWED_HOSTS
ALLOWED_HOSTS = ['toaster-example.example.com']

View File

@@ -0,0 +1,28 @@
#
# BitBake Toaster Implementation
#
# Copyright (C) 2016 Intel Corporation
#
# SPDX-License-Identifier: GPL-2.0-only
#
# Django settings for Toaster project.
# Settings overlay to use for running tests
# DJANGO_SETTINGS_MODULE=toastermain.settings-test
from toastermain.settings import *
DEBUG = True
TEMPLATE_DEBUG = DEBUG
DATABASES = {
'default': {
'ENGINE': 'django.db.backends.sqlite3',
'NAME': '%s/toaster-test-db.sqlite' % TMPDIR,
'TEST': {
'ENGINE': 'django.db.backends.sqlite3',
'NAME': '%s/toaster-test-db.sqlite' % TMPDIR,
}
}
}

View File

@@ -0,0 +1,82 @@
#
# BitBake Toaster Implementation
#
# Copyright (C) 2013 Intel Corporation
#
# SPDX-License-Identifier: GPL-2.0-only
#
from django.urls import re_path as url, include
from django.views.generic import RedirectView, TemplateView
from django.views.decorators.cache import never_cache
import bldcollector.views
import logging
logger = logging.getLogger("toaster")
# Uncomment the next two lines to enable the admin:
from django.contrib import admin
admin.autodiscover()
urlpatterns = [
# Examples:
# url(r'^toaster/', include('toaster.foo.urls')),
# Uncomment the admin/doc line below to enable admin documentation:
# url(r'^admin/doc/', include('django.contrib.admindocs.urls')),
url(r'^logs/', include('log_viewer.urls')),
# This is here to maintain backward compatibility and will be deprecated
# in the future.
url(r'^orm/eventfile$', bldcollector.views.eventfile),
url(r'^health$', TemplateView.as_view(template_name="health.html"), name='Toaster Health'),
# if no application is selected, we have the magic toastergui app here
url(r'^$', never_cache(RedirectView.as_view(url='/toastergui/', permanent=True))),
]
import toastermain.settings
if toastermain.settings.FRESH_ENABLED:
urlpatterns.insert(1, url(r'', include('fresh.urls')))
#logger.info("Enabled django-fresh extension")
if toastermain.settings.DEBUG_PANEL_ENABLED:
import debug_toolbar
urlpatterns.insert(1, url(r'', include(debug_toolbar.urls)))
#logger.info("Enabled django_toolbar extension")
urlpatterns = [
# Uncomment the next line to enable the admin:
url(r'^admin/', admin.site.urls),
] + urlpatterns
# Automatically discover urls.py in various apps, beside our own
# and map module directories to the patterns
import os
currentdir = os.path.dirname(__file__)
for t in os.walk(os.path.dirname(currentdir)):
#if we have a virtualenv skip it to avoid incorrect imports
if 'VIRTUAL_ENV' in os.environ and os.environ['VIRTUAL_ENV'] in t[0]:
continue
if "urls.py" in t[2] and t[0] != currentdir:
modulename = os.path.basename(t[0])
# make sure we don't have this module name in
conflict = False
for p in urlpatterns:
if p.pattern.regex.pattern == '^' + modulename + '/':
conflict = True
if not conflict:
urlpatterns.insert(0, url(r'^' + modulename + '/', include ( modulename + '.urls')))
else:
logger.warning("Module \'%s\' has a regexp conflict, was not added to the urlpatterns" % modulename)
from pprint import pformat
#logger.debug("urlpatterns list %s", pformat(urlpatterns))

View File

@@ -0,0 +1,36 @@
#
# SPDX-License-Identifier: GPL-2.0-only
#
"""
WSGI config for Toaster project.
This module contains the WSGI application used by Django's development server
and any production WSGI deployments. It should expose a module-level variable
named ``application``. Django's ``runserver`` and ``runfcgi`` commands discover
this application via the ``WSGI_APPLICATION`` setting.
Usually you will have the standard Django WSGI application here, but it also
might make sense to replace the whole Django WSGI application with a custom one
that later delegates to the Django one. For example, you could introduce WSGI
middleware here, or combine a Django application with an application of another
framework.
"""
import os
# We defer to a DJANGO_SETTINGS_MODULE already in the environment. This breaks
# if running multiple sites in the same mod_wsgi process. To fix this, use
# mod_wsgi daemon mode with each site in its own daemon process, or use
# os.environ["DJANGO_SETTINGS_MODULE"] = "Toaster.settings"
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "toastermain.settings")
# This application object is used by any WSGI server configured to use this
# file. This includes Django's development server, if the WSGI_APPLICATION
# setting points here.
from django.core.wsgi import get_wsgi_application
application = get_wsgi_application()
# Apply WSGI middleware here.
# from helloworld.wsgi import HelloWorldApplication
# application = HelloWorldApplication(application)