Complete Yocto mirror with license table for TQMa6UL (2038-compliance)
- 264 license table entries with exact download URLs (224/264 resolved) - Complete sources/ directory with all BitBake recipes - Build configuration: tqma6ul-multi-mba6ulx, spaetzle (musl) - Full traceability for Softwarefreigabeantrag - GCC 13.4.0, Linux 6.6.102, U-Boot 2023.04, musl 1.2.4 - License distribution: GPL-2.0 (24), MIT (23), GPL-2.0+ (18), BSD-3 (16)
This commit is contained in:
Siggi (OpenClaw Agent)
0
sources/poky/scripts/lib/resulttool/__init__.py
Normal file
0
sources/poky/scripts/lib/resulttool/__init__.py
Normal file
107
sources/poky/scripts/lib/resulttool/log.py
Normal file
107
sources/poky/scripts/lib/resulttool/log.py
Normal file
@@ -0,0 +1,107 @@
|
||||
# resulttool - Show logs
|
||||
#
|
||||
# Copyright (c) 2019 Garmin International
|
||||
#
|
||||
# SPDX-License-Identifier: GPL-2.0-only
|
||||
#
|
||||
import os
|
||||
import resulttool.resultutils as resultutils
|
||||
|
||||
def show_ptest(result, ptest, logger):
|
||||
logdata = resultutils.ptestresult_get_log(result, ptest)
|
||||
if logdata is not None:
|
||||
print(logdata)
|
||||
return 0
|
||||
|
||||
print("ptest '%s' log not found" % ptest)
|
||||
return 1
|
||||
|
||||
def show_reproducible(result, reproducible, logger):
|
||||
try:
|
||||
print(result['reproducible'][reproducible]['diffoscope.text'])
|
||||
return 0
|
||||
|
||||
except KeyError:
|
||||
print("reproducible '%s' not found" % reproducible)
|
||||
return 1
|
||||
|
||||
def log(args, logger):
|
||||
results = resultutils.load_resultsdata(args.source)
|
||||
|
||||
for _, run_name, _, r in resultutils.test_run_results(results):
|
||||
if args.list_ptest:
|
||||
print('\n'.join(sorted(r['ptestresult.sections'].keys())))
|
||||
|
||||
if args.dump_ptest:
|
||||
for sectname in ['ptestresult.sections', 'ltpposixresult.sections', 'ltpresult.sections']:
|
||||
if sectname in r:
|
||||
for name, ptest in r[sectname].items():
|
||||
logdata = resultutils.generic_get_log(sectname, r, name)
|
||||
if logdata is not None:
|
||||
dest_dir = args.dump_ptest
|
||||
if args.prepend_run:
|
||||
dest_dir = os.path.join(dest_dir, run_name)
|
||||
if not sectname.startswith("ptest"):
|
||||
dest_dir = os.path.join(dest_dir, sectname.split(".")[0])
|
||||
|
||||
os.makedirs(dest_dir, exist_ok=True)
|
||||
dest = os.path.join(dest_dir, '%s.log' % name)
|
||||
if os.path.exists(dest):
|
||||
print("Overlapping ptest logs found, skipping %s. The '--prepend-run' option would avoid this" % name)
|
||||
continue
|
||||
print(dest)
|
||||
with open(dest, 'w') as f:
|
||||
f.write(logdata)
|
||||
|
||||
if args.raw_ptest:
|
||||
found = False
|
||||
for sectname in ['ptestresult.rawlogs', 'ltpposixresult.rawlogs', 'ltpresult.rawlogs']:
|
||||
rawlog = resultutils.generic_get_rawlogs(sectname, r)
|
||||
if rawlog is not None:
|
||||
print(rawlog)
|
||||
found = True
|
||||
if not found:
|
||||
print('Raw ptest logs not found')
|
||||
return 1
|
||||
|
||||
if args.raw_reproducible:
|
||||
if 'reproducible.rawlogs' in r:
|
||||
print(r['reproducible.rawlogs']['log'])
|
||||
else:
|
||||
print('Raw reproducible logs not found')
|
||||
return 1
|
||||
|
||||
for ptest in args.ptest:
|
||||
if not show_ptest(r, ptest, logger):
|
||||
return 1
|
||||
|
||||
for reproducible in args.reproducible:
|
||||
if not show_reproducible(r, reproducible, logger):
|
||||
return 1
|
||||
|
||||
def register_commands(subparsers):
|
||||
"""Register subcommands from this plugin"""
|
||||
parser = subparsers.add_parser('log', help='show logs',
|
||||
description='show the logs from test results',
|
||||
group='analysis')
|
||||
parser.set_defaults(func=log)
|
||||
parser.add_argument('source',
|
||||
help='the results file/directory/URL to import')
|
||||
parser.add_argument('--list-ptest', action='store_true',
|
||||
help='list the ptest test names')
|
||||
parser.add_argument('--ptest', action='append', default=[],
|
||||
help='show logs for a ptest')
|
||||
parser.add_argument('--dump-ptest', metavar='DIR',
|
||||
help='Dump all ptest log files to the specified directory.')
|
||||
parser.add_argument('--reproducible', action='append', default=[],
|
||||
help='show logs for a reproducible test')
|
||||
parser.add_argument('--prepend-run', action='store_true',
|
||||
help='''Dump ptest results to a subdirectory named after the test run when using --dump-ptest.
|
||||
Required if more than one test run is present in the result file''')
|
||||
parser.add_argument('--raw', action='store_true',
|
||||
help='show raw (ptest) logs. Deprecated. Alias for "--raw-ptest"', dest='raw_ptest')
|
||||
parser.add_argument('--raw-ptest', action='store_true',
|
||||
help='show raw ptest log')
|
||||
parser.add_argument('--raw-reproducible', action='store_true',
|
||||
help='show raw reproducible build logs')
|
||||
|
||||
235
sources/poky/scripts/lib/resulttool/manualexecution.py
Executable file
235
sources/poky/scripts/lib/resulttool/manualexecution.py
Executable file
@@ -0,0 +1,235 @@
|
||||
# test case management tool - manual execution from testopia test cases
|
||||
#
|
||||
# Copyright (c) 2018, Intel Corporation.
|
||||
#
|
||||
# SPDX-License-Identifier: GPL-2.0-only
|
||||
#
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import datetime
|
||||
import re
|
||||
import copy
|
||||
from oeqa.core.runner import OETestResultJSONHelper
|
||||
|
||||
|
||||
def load_json_file(f):
|
||||
with open(f, "r") as filedata:
|
||||
return json.load(filedata)
|
||||
|
||||
def write_json_file(f, json_data):
|
||||
os.makedirs(os.path.dirname(f), exist_ok=True)
|
||||
with open(f, 'w') as filedata:
|
||||
filedata.write(json.dumps(json_data, sort_keys=True, indent=1))
|
||||
|
||||
class ManualTestRunner(object):
|
||||
|
||||
def _get_test_module(self, case_file):
|
||||
return os.path.basename(case_file).split('.')[0]
|
||||
|
||||
def _get_input(self, config):
|
||||
while True:
|
||||
output = input('{} = '.format(config))
|
||||
if re.match('^[a-z0-9-.]+$', output):
|
||||
break
|
||||
print('Only lowercase alphanumeric, hyphen and dot are allowed. Please try again')
|
||||
return output
|
||||
|
||||
def _get_available_config_options(self, config_options, test_module, target_config):
|
||||
avail_config_options = None
|
||||
if test_module in config_options:
|
||||
avail_config_options = config_options[test_module].get(target_config)
|
||||
return avail_config_options
|
||||
|
||||
def _choose_config_option(self, options):
|
||||
while True:
|
||||
output = input('{} = '.format('Option index number'))
|
||||
if output in options:
|
||||
break
|
||||
print('Only integer index inputs from above available configuration options are allowed. Please try again.')
|
||||
return options[output]
|
||||
|
||||
def _get_config(self, config_options, test_module):
|
||||
from oeqa.utils.metadata import get_layers
|
||||
from oeqa.utils.commands import get_bb_var
|
||||
from resulttool.resultutils import store_map
|
||||
|
||||
layers = get_layers(get_bb_var('BBLAYERS'))
|
||||
configurations = {}
|
||||
configurations['LAYERS'] = layers
|
||||
configurations['STARTTIME'] = datetime.datetime.now().strftime('%Y%m%d%H%M%S')
|
||||
configurations['TEST_TYPE'] = 'manual'
|
||||
configurations['TEST_MODULE'] = test_module
|
||||
|
||||
extra_config = set(store_map['manual']) - set(configurations)
|
||||
for config in sorted(extra_config):
|
||||
avail_config_options = self._get_available_config_options(config_options, test_module, config)
|
||||
if avail_config_options:
|
||||
print('---------------------------------------------')
|
||||
print('These are available configuration #%s options:' % config)
|
||||
print('---------------------------------------------')
|
||||
for option, _ in sorted(avail_config_options.items(), key=lambda x: int(x[0])):
|
||||
print('%s: %s' % (option, avail_config_options[option]))
|
||||
print('Please select configuration option, enter the integer index number.')
|
||||
value_conf = self._choose_config_option(avail_config_options)
|
||||
print('---------------------------------------------\n')
|
||||
else:
|
||||
print('---------------------------------------------')
|
||||
print('This is configuration #%s. Please provide configuration value(use "None" if not applicable).' % config)
|
||||
print('---------------------------------------------')
|
||||
value_conf = self._get_input('Configuration Value')
|
||||
print('---------------------------------------------\n')
|
||||
configurations[config] = value_conf
|
||||
return configurations
|
||||
|
||||
def _execute_test_steps(self, case):
|
||||
test_result = {}
|
||||
print('------------------------------------------------------------------------')
|
||||
print('Executing test case: %s' % case['test']['@alias'])
|
||||
print('------------------------------------------------------------------------')
|
||||
print('You have total %s test steps to be executed.' % len(case['test']['execution']))
|
||||
print('------------------------------------------------------------------------\n')
|
||||
for step, _ in sorted(case['test']['execution'].items(), key=lambda x: int(x[0])):
|
||||
print('Step %s: %s' % (step, case['test']['execution'][step]['action']))
|
||||
expected_output = case['test']['execution'][step]['expected_results']
|
||||
if expected_output:
|
||||
print('Expected output: %s' % expected_output)
|
||||
while True:
|
||||
done = input('\nPlease provide test results: (P)assed/(F)ailed/(B)locked/(S)kipped? \n').lower()
|
||||
result_types = {'p':'PASSED',
|
||||
'f':'FAILED',
|
||||
'b':'BLOCKED',
|
||||
's':'SKIPPED'}
|
||||
if done in result_types:
|
||||
for r in result_types:
|
||||
if done == r:
|
||||
res = result_types[r]
|
||||
if res == 'FAILED':
|
||||
log_input = input('\nPlease enter the error and the description of the log: (Ex:log:211 Error Bitbake)\n')
|
||||
test_result.update({case['test']['@alias']: {'status': '%s' % res, 'log': '%s' % log_input}})
|
||||
else:
|
||||
test_result.update({case['test']['@alias']: {'status': '%s' % res}})
|
||||
break
|
||||
print('Invalid input!')
|
||||
return test_result
|
||||
|
||||
def _get_write_dir(self):
|
||||
return os.environ['BUILDDIR'] + '/tmp/log/manual/'
|
||||
|
||||
def run_test(self, case_file, config_options_file, testcase_config_file):
|
||||
test_module = self._get_test_module(case_file)
|
||||
cases = load_json_file(case_file)
|
||||
config_options = {}
|
||||
if config_options_file:
|
||||
config_options = load_json_file(config_options_file)
|
||||
configurations = self._get_config(config_options, test_module)
|
||||
result_id = 'manual_%s_%s' % (test_module, configurations['STARTTIME'])
|
||||
test_results = {}
|
||||
if testcase_config_file:
|
||||
test_case_config = load_json_file(testcase_config_file)
|
||||
test_case_to_execute = test_case_config['testcases']
|
||||
for case in copy.deepcopy(cases) :
|
||||
if case['test']['@alias'] not in test_case_to_execute:
|
||||
cases.remove(case)
|
||||
|
||||
print('\nTotal number of test cases in this test suite: %s\n' % len(cases))
|
||||
for c in cases:
|
||||
test_result = self._execute_test_steps(c)
|
||||
test_results.update(test_result)
|
||||
return configurations, result_id, self._get_write_dir(), test_results
|
||||
|
||||
def _get_true_false_input(self, input_message):
|
||||
yes_list = ['Y', 'YES']
|
||||
no_list = ['N', 'NO']
|
||||
while True:
|
||||
more_config_option = input(input_message).upper()
|
||||
if more_config_option in yes_list or more_config_option in no_list:
|
||||
break
|
||||
print('Invalid input!')
|
||||
if more_config_option in no_list:
|
||||
return False
|
||||
return True
|
||||
|
||||
def make_config_option_file(self, logger, case_file, config_options_file):
|
||||
config_options = {}
|
||||
if config_options_file:
|
||||
config_options = load_json_file(config_options_file)
|
||||
new_test_module = self._get_test_module(case_file)
|
||||
print('Creating configuration options file for test module: %s' % new_test_module)
|
||||
new_config_options = {}
|
||||
|
||||
while True:
|
||||
config_name = input('\nPlease provide test configuration to create:\n').upper()
|
||||
new_config_options[config_name] = {}
|
||||
while True:
|
||||
config_value = self._get_input('Configuration possible option value')
|
||||
config_option_index = len(new_config_options[config_name]) + 1
|
||||
new_config_options[config_name][config_option_index] = config_value
|
||||
more_config_option = self._get_true_false_input('\nIs there more configuration option input: (Y)es/(N)o\n')
|
||||
if not more_config_option:
|
||||
break
|
||||
more_config = self._get_true_false_input('\nIs there more configuration to create: (Y)es/(N)o\n')
|
||||
if not more_config:
|
||||
break
|
||||
|
||||
if new_config_options:
|
||||
config_options[new_test_module] = new_config_options
|
||||
if not config_options_file:
|
||||
config_options_file = os.path.join(self._get_write_dir(), 'manual_config_options.json')
|
||||
write_json_file(config_options_file, config_options)
|
||||
logger.info('Configuration option file created at %s' % config_options_file)
|
||||
|
||||
def make_testcase_config_file(self, logger, case_file, testcase_config_file):
|
||||
if testcase_config_file:
|
||||
if os.path.exists(testcase_config_file):
|
||||
print('\nTest configuration file with name %s already exists. Please provide a unique file name' % (testcase_config_file))
|
||||
return 0
|
||||
|
||||
if not testcase_config_file:
|
||||
testcase_config_file = os.path.join(self._get_write_dir(), "testconfig_new.json")
|
||||
|
||||
testcase_config = {}
|
||||
cases = load_json_file(case_file)
|
||||
new_test_module = self._get_test_module(case_file)
|
||||
new_testcase_config = {}
|
||||
new_testcase_config['testcases'] = []
|
||||
|
||||
print('\nAdd testcases for this configuration file:')
|
||||
for case in cases:
|
||||
print('\n' + case['test']['@alias'])
|
||||
add_tc_config = self._get_true_false_input('\nDo you want to add this test case to test configuration : (Y)es/(N)o\n')
|
||||
if add_tc_config:
|
||||
new_testcase_config['testcases'].append(case['test']['@alias'])
|
||||
write_json_file(testcase_config_file, new_testcase_config)
|
||||
logger.info('Testcase Configuration file created at %s' % testcase_config_file)
|
||||
|
||||
def manualexecution(args, logger):
|
||||
testrunner = ManualTestRunner()
|
||||
if args.make_config_options_file:
|
||||
testrunner.make_config_option_file(logger, args.file, args.config_options_file)
|
||||
return 0
|
||||
if args.make_testcase_config_file:
|
||||
testrunner.make_testcase_config_file(logger, args.file, args.testcase_config_file)
|
||||
return 0
|
||||
configurations, result_id, write_dir, test_results = testrunner.run_test(args.file, args.config_options_file, args.testcase_config_file)
|
||||
resultjsonhelper = OETestResultJSONHelper()
|
||||
resultjsonhelper.dump_testresult_file(write_dir, configurations, result_id, test_results)
|
||||
return 0
|
||||
|
||||
def register_commands(subparsers):
|
||||
"""Register subcommands from this plugin"""
|
||||
parser_build = subparsers.add_parser('manualexecution', help='helper script for results populating during manual test execution.',
|
||||
description='helper script for results populating during manual test execution. You can find manual test case JSON file in meta/lib/oeqa/manual/',
|
||||
group='manualexecution')
|
||||
parser_build.set_defaults(func=manualexecution)
|
||||
parser_build.add_argument('file', help='specify path to manual test case JSON file.Note: Please use \"\" to encapsulate the file path.')
|
||||
parser_build.add_argument('-c', '--config-options-file', default='',
|
||||
help='the config options file to import and used as available configuration option selection or make config option file')
|
||||
parser_build.add_argument('-m', '--make-config-options-file', action='store_true',
|
||||
help='make the configuration options file based on provided inputs')
|
||||
parser_build.add_argument('-t', '--testcase-config-file', default='',
|
||||
help='the testcase configuration file to enable user to run a selected set of test case or make a testcase configuration file')
|
||||
parser_build.add_argument('-d', '--make-testcase-config-file', action='store_true',
|
||||
help='make the testcase configuration file to run a set of test cases based on user selection')
|
||||
46
sources/poky/scripts/lib/resulttool/merge.py
Normal file
46
sources/poky/scripts/lib/resulttool/merge.py
Normal file
@@ -0,0 +1,46 @@
|
||||
# resulttool - merge multiple testresults.json files into a file or directory
|
||||
#
|
||||
# Copyright (c) 2019, Intel Corporation.
|
||||
# Copyright (c) 2019, Linux Foundation
|
||||
#
|
||||
# SPDX-License-Identifier: GPL-2.0-only
|
||||
#
|
||||
|
||||
import os
|
||||
import json
|
||||
import resulttool.resultutils as resultutils
|
||||
|
||||
def merge(args, logger):
|
||||
configvars = {}
|
||||
if not args.not_add_testseries:
|
||||
configvars = resultutils.extra_configvars.copy()
|
||||
if args.executed_by:
|
||||
configvars['EXECUTED_BY'] = args.executed_by
|
||||
if resultutils.is_url(args.target_results) or os.path.isdir(args.target_results):
|
||||
results = resultutils.load_resultsdata(args.target_results, configmap=resultutils.store_map, configvars=configvars)
|
||||
resultutils.append_resultsdata(results, args.base_results, configmap=resultutils.store_map, configvars=configvars)
|
||||
resultutils.save_resultsdata(results, args.target_results)
|
||||
else:
|
||||
results = resultutils.load_resultsdata(args.base_results, configmap=resultutils.flatten_map, configvars=configvars)
|
||||
if os.path.exists(args.target_results):
|
||||
resultutils.append_resultsdata(results, args.target_results, configmap=resultutils.flatten_map, configvars=configvars)
|
||||
resultutils.save_resultsdata(results, os.path.dirname(args.target_results), fn=os.path.basename(args.target_results))
|
||||
|
||||
logger.info('Merged results to %s' % os.path.dirname(args.target_results))
|
||||
|
||||
return 0
|
||||
|
||||
def register_commands(subparsers):
|
||||
"""Register subcommands from this plugin"""
|
||||
parser_build = subparsers.add_parser('merge', help='merge test result files/directories/URLs',
|
||||
description='merge the results from multiple files/directories/URLs into the target file or directory',
|
||||
group='setup')
|
||||
parser_build.set_defaults(func=merge)
|
||||
parser_build.add_argument('base_results',
|
||||
help='the results file/directory/URL to import')
|
||||
parser_build.add_argument('target_results',
|
||||
help='the target file or directory to merge the base_results with')
|
||||
parser_build.add_argument('-t', '--not-add-testseries', action='store_true',
|
||||
help='do not add testseries configuration to results')
|
||||
parser_build.add_argument('-x', '--executed-by', default='',
|
||||
help='add executed-by configuration to each result file')
|
||||
447
sources/poky/scripts/lib/resulttool/regression.py
Normal file
447
sources/poky/scripts/lib/resulttool/regression.py
Normal file
@@ -0,0 +1,447 @@
|
||||
# resulttool - regression analysis
|
||||
#
|
||||
# Copyright (c) 2019, Intel Corporation.
|
||||
# Copyright (c) 2019, Linux Foundation
|
||||
#
|
||||
# SPDX-License-Identifier: GPL-2.0-only
|
||||
#
|
||||
|
||||
import resulttool.resultutils as resultutils
|
||||
|
||||
from oeqa.utils.git import GitRepo
|
||||
import oeqa.utils.gitarchive as gitarchive
|
||||
|
||||
METADATA_MATCH_TABLE = {
|
||||
"oeselftest": "OESELFTEST_METADATA"
|
||||
}
|
||||
|
||||
OESELFTEST_METADATA_GUESS_TABLE={
|
||||
"trigger-build-posttrigger": {
|
||||
"run_all_tests": False,
|
||||
"run_tests":["buildoptions.SourceMirroring.test_yocto_source_mirror"],
|
||||
"skips": None,
|
||||
"machine": None,
|
||||
"select_tags":None,
|
||||
"exclude_tags": None
|
||||
},
|
||||
"reproducible": {
|
||||
"run_all_tests": False,
|
||||
"run_tests":["reproducible"],
|
||||
"skips": None,
|
||||
"machine": None,
|
||||
"select_tags":None,
|
||||
"exclude_tags": None
|
||||
},
|
||||
"arch-qemu-quick": {
|
||||
"run_all_tests": True,
|
||||
"run_tests":None,
|
||||
"skips": None,
|
||||
"machine": None,
|
||||
"select_tags":["machine"],
|
||||
"exclude_tags": None
|
||||
},
|
||||
"arch-qemu-full-x86-or-x86_64": {
|
||||
"run_all_tests": True,
|
||||
"run_tests":None,
|
||||
"skips": None,
|
||||
"machine": None,
|
||||
"select_tags":["machine", "toolchain-system"],
|
||||
"exclude_tags": None
|
||||
},
|
||||
"arch-qemu-full-others": {
|
||||
"run_all_tests": True,
|
||||
"run_tests":None,
|
||||
"skips": None,
|
||||
"machine": None,
|
||||
"select_tags":["machine", "toolchain-user"],
|
||||
"exclude_tags": None
|
||||
},
|
||||
"selftest": {
|
||||
"run_all_tests": True,
|
||||
"run_tests":None,
|
||||
"skips": ["distrodata.Distrodata.test_checkpkg", "buildoptions.SourceMirroring.test_yocto_source_mirror", "reproducible"],
|
||||
"machine": None,
|
||||
"select_tags":None,
|
||||
"exclude_tags": ["machine", "toolchain-system", "toolchain-user"]
|
||||
},
|
||||
"bringup": {
|
||||
"run_all_tests": True,
|
||||
"run_tests":None,
|
||||
"skips": ["distrodata.Distrodata.test_checkpkg", "buildoptions.SourceMirroring.test_yocto_source_mirror"],
|
||||
"machine": None,
|
||||
"select_tags":None,
|
||||
"exclude_tags": ["machine", "toolchain-system", "toolchain-user"]
|
||||
}
|
||||
}
|
||||
|
||||
STATUS_STRINGS = {
|
||||
"None": "No matching test result"
|
||||
}
|
||||
|
||||
REGRESSIONS_DISPLAY_LIMIT=50
|
||||
|
||||
MISSING_TESTS_BANNER = "-------------------------- Missing tests --------------------------"
|
||||
ADDITIONAL_DATA_BANNER = "--------------------- Matches and improvements --------------------"
|
||||
|
||||
def test_has_at_least_one_matching_tag(test, tag_list):
|
||||
return "oetags" in test and any(oetag in tag_list for oetag in test["oetags"])
|
||||
|
||||
def all_tests_have_at_least_one_matching_tag(results, tag_list):
|
||||
return all(test_has_at_least_one_matching_tag(test_result, tag_list) or test_name.startswith("ptestresult") for (test_name, test_result) in results.items())
|
||||
|
||||
def any_test_have_any_matching_tag(results, tag_list):
|
||||
return any(test_has_at_least_one_matching_tag(test, tag_list) for test in results.values())
|
||||
|
||||
def have_skipped_test(result, test_prefix):
|
||||
return all( result[test]['status'] == "SKIPPED" for test in result if test.startswith(test_prefix))
|
||||
|
||||
def have_all_tests_skipped(result, test_prefixes_list):
|
||||
return all(have_skipped_test(result, test_prefix) for test_prefix in test_prefixes_list)
|
||||
|
||||
def guess_oeselftest_metadata(results):
|
||||
"""
|
||||
When an oeselftest test result is lacking OESELFTEST_METADATA, we can try to guess it based on results content.
|
||||
Check results for specific values (absence/presence of oetags, number and name of executed tests...),
|
||||
and if it matches one of known configuration from autobuilder configuration, apply guessed OSELFTEST_METADATA
|
||||
to it to allow proper test filtering.
|
||||
This guessing process is tightly coupled to config.json in autobuilder. It should trigger less and less,
|
||||
as new tests will have OESELFTEST_METADATA properly appended at test reporting time
|
||||
"""
|
||||
|
||||
if len(results) == 1 and "buildoptions.SourceMirroring.test_yocto_source_mirror" in results:
|
||||
return OESELFTEST_METADATA_GUESS_TABLE['trigger-build-posttrigger']
|
||||
elif all(result.startswith("reproducible") for result in results):
|
||||
return OESELFTEST_METADATA_GUESS_TABLE['reproducible']
|
||||
elif all_tests_have_at_least_one_matching_tag(results, ["machine"]):
|
||||
return OESELFTEST_METADATA_GUESS_TABLE['arch-qemu-quick']
|
||||
elif all_tests_have_at_least_one_matching_tag(results, ["machine", "toolchain-system"]):
|
||||
return OESELFTEST_METADATA_GUESS_TABLE['arch-qemu-full-x86-or-x86_64']
|
||||
elif all_tests_have_at_least_one_matching_tag(results, ["machine", "toolchain-user"]):
|
||||
return OESELFTEST_METADATA_GUESS_TABLE['arch-qemu-full-others']
|
||||
elif not any_test_have_any_matching_tag(results, ["machine", "toolchain-user", "toolchain-system"]):
|
||||
if have_all_tests_skipped(results, ["distrodata.Distrodata.test_checkpkg", "buildoptions.SourceMirroring.test_yocto_source_mirror", "reproducible"]):
|
||||
return OESELFTEST_METADATA_GUESS_TABLE['selftest']
|
||||
elif have_all_tests_skipped(results, ["distrodata.Distrodata.test_checkpkg", "buildoptions.SourceMirroring.test_yocto_source_mirror"]):
|
||||
return OESELFTEST_METADATA_GUESS_TABLE['bringup']
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def metadata_matches(base_configuration, target_configuration):
|
||||
"""
|
||||
For passed base and target, check test type. If test type matches one of
|
||||
properties described in METADATA_MATCH_TABLE, compare metadata if it is
|
||||
present in base. Return true if metadata matches, or if base lacks some
|
||||
data (either TEST_TYPE or the corresponding metadata)
|
||||
"""
|
||||
test_type = base_configuration.get('TEST_TYPE')
|
||||
if test_type not in METADATA_MATCH_TABLE:
|
||||
return True
|
||||
|
||||
metadata_key = METADATA_MATCH_TABLE.get(test_type)
|
||||
if target_configuration.get(metadata_key) != base_configuration.get(metadata_key):
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
|
||||
def machine_matches(base_configuration, target_configuration):
|
||||
return base_configuration.get('MACHINE') == target_configuration.get('MACHINE')
|
||||
|
||||
|
||||
def can_be_compared(logger, base, target):
|
||||
"""
|
||||
Some tests are not relevant to be compared, for example some oeselftest
|
||||
run with different tests sets or parameters. Return true if tests can be
|
||||
compared
|
||||
"""
|
||||
ret = True
|
||||
base_configuration = base['configuration']
|
||||
target_configuration = target['configuration']
|
||||
|
||||
# Older test results lack proper OESELFTEST_METADATA: if not present, try to guess it based on tests results.
|
||||
if base_configuration.get('TEST_TYPE') == 'oeselftest' and 'OESELFTEST_METADATA' not in base_configuration:
|
||||
guess = guess_oeselftest_metadata(base['result'])
|
||||
if guess is None:
|
||||
logger.error(f"ERROR: did not manage to guess oeselftest metadata for {base_configuration['STARTTIME']}")
|
||||
else:
|
||||
logger.debug(f"Enriching {base_configuration['STARTTIME']} with {guess}")
|
||||
base_configuration['OESELFTEST_METADATA'] = guess
|
||||
if target_configuration.get('TEST_TYPE') == 'oeselftest' and 'OESELFTEST_METADATA' not in target_configuration:
|
||||
guess = guess_oeselftest_metadata(target['result'])
|
||||
if guess is None:
|
||||
logger.error(f"ERROR: did not manage to guess oeselftest metadata for {target_configuration['STARTTIME']}")
|
||||
else:
|
||||
logger.debug(f"Enriching {target_configuration['STARTTIME']} with {guess}")
|
||||
target_configuration['OESELFTEST_METADATA'] = guess
|
||||
|
||||
# Test runs with LTP results in should only be compared with other runs with LTP tests in them
|
||||
if base_configuration.get('TEST_TYPE') == 'runtime' and any(result.startswith("ltpresult") for result in base['result']):
|
||||
ret = target_configuration.get('TEST_TYPE') == 'runtime' and any(result.startswith("ltpresult") for result in target['result'])
|
||||
|
||||
return ret and metadata_matches(base_configuration, target_configuration) \
|
||||
and machine_matches(base_configuration, target_configuration)
|
||||
|
||||
def get_status_str(raw_status):
|
||||
raw_status_lower = raw_status.lower() if raw_status else "None"
|
||||
return STATUS_STRINGS.get(raw_status_lower, raw_status)
|
||||
|
||||
def get_additional_info_line(new_pass_count, new_tests):
|
||||
result=[]
|
||||
if new_tests:
|
||||
result.append(f'+{new_tests} test(s) present')
|
||||
if new_pass_count:
|
||||
result.append(f'+{new_pass_count} test(s) now passing')
|
||||
|
||||
if not result:
|
||||
return ""
|
||||
|
||||
return ' -> ' + ', '.join(result) + '\n'
|
||||
|
||||
def compare_result(logger, base_name, target_name, base_result, target_result, display_limit=None):
|
||||
base_result = base_result.get('result')
|
||||
target_result = target_result.get('result')
|
||||
result = {}
|
||||
new_tests = 0
|
||||
regressions = {}
|
||||
resultstring = ""
|
||||
new_tests = 0
|
||||
new_pass_count = 0
|
||||
|
||||
display_limit = int(display_limit) if display_limit else REGRESSIONS_DISPLAY_LIMIT
|
||||
|
||||
if base_result and target_result:
|
||||
for k in base_result:
|
||||
base_testcase = base_result[k]
|
||||
base_status = base_testcase.get('status')
|
||||
if base_status:
|
||||
target_testcase = target_result.get(k, {})
|
||||
target_status = target_testcase.get('status')
|
||||
if base_status != target_status:
|
||||
result[k] = {'base': base_status, 'target': target_status}
|
||||
else:
|
||||
logger.error('Failed to retrieved base test case status: %s' % k)
|
||||
|
||||
# Also count new tests that were not present in base results: it
|
||||
# could be newly added tests, but it could also highlights some tests
|
||||
# renames or fixed faulty ptests
|
||||
for k in target_result:
|
||||
if k not in base_result:
|
||||
new_tests += 1
|
||||
if result:
|
||||
new_pass_count = sum(test['target'] is not None and test['target'].startswith("PASS") for test in result.values())
|
||||
# Print a regression report only if at least one test has a regression status (FAIL, SKIPPED, absent...)
|
||||
if new_pass_count < len(result):
|
||||
resultstring = "Regression: %s\n %s\n" % (base_name, target_name)
|
||||
for k in sorted(result):
|
||||
if not result[k]['target'] or not result[k]['target'].startswith("PASS"):
|
||||
# Differentiate each ptest kind when listing regressions
|
||||
key_parts = k.split('.')
|
||||
key = '.'.join(key_parts[:2]) if k.startswith('ptest') else key_parts[0]
|
||||
# Append new regression to corresponding test family
|
||||
regressions[key] = regressions.setdefault(key, []) + [' %s: %s -> %s\n' % (k, get_status_str(result[k]['base']), get_status_str(result[k]['target']))]
|
||||
resultstring += f" Total: {sum([len(regressions[r]) for r in regressions])} new regression(s):\n"
|
||||
for k in regressions:
|
||||
resultstring += f" {len(regressions[k])} regression(s) for {k}\n"
|
||||
count_to_print=min([display_limit, len(regressions[k])]) if display_limit > 0 else len(regressions[k])
|
||||
resultstring += ''.join(regressions[k][:count_to_print])
|
||||
if count_to_print < len(regressions[k]):
|
||||
resultstring+=' [...]\n'
|
||||
if new_pass_count > 0:
|
||||
resultstring += f' Additionally, {new_pass_count} previously failing test(s) is/are now passing\n'
|
||||
if new_tests > 0:
|
||||
resultstring += f' Additionally, {new_tests} new test(s) is/are present\n'
|
||||
else:
|
||||
resultstring = "%s\n%s\n" % (base_name, target_name)
|
||||
result = None
|
||||
else:
|
||||
resultstring = "%s\n%s\n" % (base_name, target_name)
|
||||
|
||||
if not result:
|
||||
additional_info = get_additional_info_line(new_pass_count, new_tests)
|
||||
if additional_info:
|
||||
resultstring += additional_info
|
||||
|
||||
return result, resultstring
|
||||
|
||||
def get_results(logger, source):
|
||||
return resultutils.load_resultsdata(source, configmap=resultutils.regression_map)
|
||||
|
||||
def regression(args, logger):
|
||||
base_results = get_results(logger, args.base_result)
|
||||
target_results = get_results(logger, args.target_result)
|
||||
|
||||
regression_common(args, logger, base_results, target_results)
|
||||
|
||||
# Some test case naming is poor and contains random strings, particularly lttng/babeltrace.
|
||||
# Truncating the test names works since they contain file and line number identifiers
|
||||
# which allows us to match them without the random components.
|
||||
def fixup_ptest_names(results, logger):
|
||||
for r in results:
|
||||
for i in results[r]:
|
||||
tests = list(results[r][i]['result'].keys())
|
||||
for test in tests:
|
||||
new = None
|
||||
if test.startswith(("ptestresult.lttng-tools.", "ptestresult.babeltrace.", "ptestresult.babeltrace2")) and "_-_" in test:
|
||||
new = test.split("_-_")[0]
|
||||
elif test.startswith(("ptestresult.curl.")) and "__" in test:
|
||||
new = test.split("__")[0]
|
||||
elif test.startswith(("ptestresult.dbus.")) and "__" in test:
|
||||
new = test.split("__")[0]
|
||||
elif test.startswith("ptestresult.binutils") and "build-st-" in test:
|
||||
new = test.split(" ")[0]
|
||||
elif test.startswith("ptestresult.gcc") and "/tmp/runtest." in test:
|
||||
new = ".".join(test.split(".")[:2])
|
||||
if new:
|
||||
results[r][i]['result'][new] = results[r][i]['result'][test]
|
||||
del results[r][i]['result'][test]
|
||||
|
||||
def regression_common(args, logger, base_results, target_results):
|
||||
if args.base_result_id:
|
||||
base_results = resultutils.filter_resultsdata(base_results, args.base_result_id)
|
||||
if args.target_result_id:
|
||||
target_results = resultutils.filter_resultsdata(target_results, args.target_result_id)
|
||||
|
||||
fixup_ptest_names(base_results, logger)
|
||||
fixup_ptest_names(target_results, logger)
|
||||
|
||||
matches = []
|
||||
regressions = []
|
||||
notfound = []
|
||||
|
||||
for a in base_results:
|
||||
if a in target_results:
|
||||
base = list(base_results[a].keys())
|
||||
target = list(target_results[a].keys())
|
||||
# We may have multiple base/targets which are for different configurations. Start by
|
||||
# removing any pairs which match
|
||||
for c in base.copy():
|
||||
for b in target.copy():
|
||||
if not can_be_compared(logger, base_results[a][c], target_results[a][b]):
|
||||
continue
|
||||
res, resstr = compare_result(logger, c, b, base_results[a][c], target_results[a][b], args.limit)
|
||||
if not res:
|
||||
matches.append(resstr)
|
||||
base.remove(c)
|
||||
target.remove(b)
|
||||
break
|
||||
# Should only now see regressions, we may not be able to match multiple pairs directly
|
||||
for c in base:
|
||||
for b in target:
|
||||
if not can_be_compared(logger, base_results[a][c], target_results[a][b]):
|
||||
continue
|
||||
res, resstr = compare_result(logger, c, b, base_results[a][c], target_results[a][b], args.limit)
|
||||
if res:
|
||||
regressions.append(resstr)
|
||||
else:
|
||||
notfound.append("%s not found in target" % a)
|
||||
print("\n".join(sorted(regressions)))
|
||||
print("\n" + MISSING_TESTS_BANNER + "\n")
|
||||
print("\n".join(sorted(notfound)))
|
||||
print("\n" + ADDITIONAL_DATA_BANNER + "\n")
|
||||
print("\n".join(sorted(matches)))
|
||||
return 0
|
||||
|
||||
def regression_git(args, logger):
|
||||
base_results = {}
|
||||
target_results = {}
|
||||
|
||||
tag_name = "{branch}/{commit_number}-g{commit}/{tag_number}"
|
||||
repo = GitRepo(args.repo)
|
||||
|
||||
revs = gitarchive.get_test_revs(logger, repo, tag_name, branch=args.branch)
|
||||
|
||||
if args.branch2:
|
||||
revs2 = gitarchive.get_test_revs(logger, repo, tag_name, branch=args.branch2)
|
||||
if not len(revs2):
|
||||
logger.error("No revisions found to compare against")
|
||||
return 1
|
||||
if not len(revs):
|
||||
logger.error("No revision to report on found")
|
||||
return 1
|
||||
else:
|
||||
if len(revs) < 2:
|
||||
logger.error("Only %d tester revisions found, unable to generate report" % len(revs))
|
||||
return 1
|
||||
|
||||
# Pick revisions
|
||||
if args.commit:
|
||||
if args.commit_number:
|
||||
logger.warning("Ignoring --commit-number as --commit was specified")
|
||||
index1 = gitarchive.rev_find(revs, 'commit', args.commit)
|
||||
elif args.commit_number:
|
||||
index1 = gitarchive.rev_find(revs, 'commit_number', args.commit_number)
|
||||
else:
|
||||
index1 = len(revs) - 1
|
||||
|
||||
if args.branch2:
|
||||
revs2.append(revs[index1])
|
||||
index1 = len(revs2) - 1
|
||||
revs = revs2
|
||||
|
||||
if args.commit2:
|
||||
if args.commit_number2:
|
||||
logger.warning("Ignoring --commit-number2 as --commit2 was specified")
|
||||
index2 = gitarchive.rev_find(revs, 'commit', args.commit2)
|
||||
elif args.commit_number2:
|
||||
index2 = gitarchive.rev_find(revs, 'commit_number', args.commit_number2)
|
||||
else:
|
||||
if index1 > 0:
|
||||
index2 = index1 - 1
|
||||
# Find the closest matching commit number for comparision
|
||||
# In future we could check the commit is a common ancestor and
|
||||
# continue back if not but this good enough for now
|
||||
while index2 > 0 and revs[index2].commit_number > revs[index1].commit_number:
|
||||
index2 = index2 - 1
|
||||
else:
|
||||
logger.error("Unable to determine the other commit, use "
|
||||
"--commit2 or --commit-number2 to specify it")
|
||||
return 1
|
||||
|
||||
logger.info("Comparing:\n%s\nto\n%s\n" % (revs[index1], revs[index2]))
|
||||
|
||||
base_results = resultutils.git_get_result(repo, revs[index1][2])
|
||||
target_results = resultutils.git_get_result(repo, revs[index2][2])
|
||||
|
||||
regression_common(args, logger, base_results, target_results)
|
||||
|
||||
return 0
|
||||
|
||||
def register_commands(subparsers):
|
||||
"""Register subcommands from this plugin"""
|
||||
|
||||
parser_build = subparsers.add_parser('regression', help='regression file/directory analysis',
|
||||
description='regression analysis comparing the base set of results to the target results',
|
||||
group='analysis')
|
||||
parser_build.set_defaults(func=regression)
|
||||
parser_build.add_argument('base_result',
|
||||
help='base result file/directory/URL for the comparison')
|
||||
parser_build.add_argument('target_result',
|
||||
help='target result file/directory/URL to compare with')
|
||||
parser_build.add_argument('-b', '--base-result-id', default='',
|
||||
help='(optional) filter the base results to this result ID')
|
||||
parser_build.add_argument('-t', '--target-result-id', default='',
|
||||
help='(optional) filter the target results to this result ID')
|
||||
|
||||
parser_build = subparsers.add_parser('regression-git', help='regression git analysis',
|
||||
description='regression analysis comparing base result set to target '
|
||||
'result set',
|
||||
group='analysis')
|
||||
parser_build.set_defaults(func=regression_git)
|
||||
parser_build.add_argument('repo',
|
||||
help='the git repository containing the data')
|
||||
parser_build.add_argument('-b', '--base-result-id', default='',
|
||||
help='(optional) default select regression based on configurations unless base result '
|
||||
'id was provided')
|
||||
parser_build.add_argument('-t', '--target-result-id', default='',
|
||||
help='(optional) default select regression based on configurations unless target result '
|
||||
'id was provided')
|
||||
|
||||
parser_build.add_argument('--branch', '-B', default='master', help="Branch to find commit in")
|
||||
parser_build.add_argument('--branch2', help="Branch to find comparision revisions in")
|
||||
parser_build.add_argument('--commit', help="Revision to search for")
|
||||
parser_build.add_argument('--commit-number', help="Revision number to search for, redundant if --commit is specified")
|
||||
parser_build.add_argument('--commit2', help="Revision to compare with")
|
||||
parser_build.add_argument('--commit-number2', help="Revision number to compare with, redundant if --commit2 is specified")
|
||||
parser_build.add_argument('-l', '--limit', default=REGRESSIONS_DISPLAY_LIMIT, help="Maximum number of changes to display per test. Can be set to 0 to print all changes")
|
||||
|
||||
315
sources/poky/scripts/lib/resulttool/report.py
Normal file
315
sources/poky/scripts/lib/resulttool/report.py
Normal file
@@ -0,0 +1,315 @@
|
||||
# test result tool - report text based test results
|
||||
#
|
||||
# Copyright (c) 2019, Intel Corporation.
|
||||
# Copyright (c) 2019, Linux Foundation
|
||||
#
|
||||
# SPDX-License-Identifier: GPL-2.0-only
|
||||
#
|
||||
|
||||
import os
|
||||
import glob
|
||||
import json
|
||||
import resulttool.resultutils as resultutils
|
||||
from oeqa.utils.git import GitRepo
|
||||
import oeqa.utils.gitarchive as gitarchive
|
||||
|
||||
|
||||
class ResultsTextReport(object):
|
||||
def __init__(self):
|
||||
self.ptests = {}
|
||||
self.ltptests = {}
|
||||
self.ltpposixtests = {}
|
||||
self.result_types = {'passed': ['PASSED', 'passed', 'PASS', 'XFAIL'],
|
||||
'failed': ['FAILED', 'failed', 'FAIL', 'ERROR', 'error', 'UNKNOWN', 'XPASS'],
|
||||
'skipped': ['SKIPPED', 'skipped', 'UNSUPPORTED', 'UNTESTED', 'UNRESOLVED']}
|
||||
|
||||
|
||||
def handle_ptest_result(self, k, status, result, machine):
|
||||
if machine not in self.ptests:
|
||||
self.ptests[machine] = {}
|
||||
|
||||
if k == 'ptestresult.sections':
|
||||
# Ensure tests without any test results still show up on the report
|
||||
for suite in result['ptestresult.sections']:
|
||||
if suite not in self.ptests[machine]:
|
||||
self.ptests[machine][suite] = {
|
||||
'passed': 0, 'failed': 0, 'skipped': 0, 'duration' : '-',
|
||||
'failed_testcases': [], "testcases": set(),
|
||||
}
|
||||
if 'duration' in result['ptestresult.sections'][suite]:
|
||||
self.ptests[machine][suite]['duration'] = result['ptestresult.sections'][suite]['duration']
|
||||
if 'timeout' in result['ptestresult.sections'][suite]:
|
||||
self.ptests[machine][suite]['duration'] += " T"
|
||||
return True
|
||||
|
||||
# process test result
|
||||
try:
|
||||
_, suite, test = k.split(".", 2)
|
||||
except ValueError:
|
||||
return True
|
||||
|
||||
# Handle 'glib-2.0'
|
||||
if 'ptestresult.sections' in result and suite not in result['ptestresult.sections']:
|
||||
try:
|
||||
_, suite, suite1, test = k.split(".", 3)
|
||||
if suite + "." + suite1 in result['ptestresult.sections']:
|
||||
suite = suite + "." + suite1
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
if suite not in self.ptests[machine]:
|
||||
self.ptests[machine][suite] = {
|
||||
'passed': 0, 'failed': 0, 'skipped': 0, 'duration' : '-',
|
||||
'failed_testcases': [], "testcases": set(),
|
||||
}
|
||||
|
||||
# do not process duplicate results
|
||||
if test in self.ptests[machine][suite]["testcases"]:
|
||||
print("Warning duplicate ptest result '{}.{}' for {}".format(suite, test, machine))
|
||||
return False
|
||||
|
||||
for tk in self.result_types:
|
||||
if status in self.result_types[tk]:
|
||||
self.ptests[machine][suite][tk] += 1
|
||||
self.ptests[machine][suite]["testcases"].add(test)
|
||||
return True
|
||||
|
||||
def handle_ltptest_result(self, k, status, result, machine):
|
||||
if machine not in self.ltptests:
|
||||
self.ltptests[machine] = {}
|
||||
|
||||
if k == 'ltpresult.sections':
|
||||
# Ensure tests without any test results still show up on the report
|
||||
for suite in result['ltpresult.sections']:
|
||||
if suite not in self.ltptests[machine]:
|
||||
self.ltptests[machine][suite] = {'passed': 0, 'failed': 0, 'skipped': 0, 'duration' : '-', 'failed_testcases': []}
|
||||
if 'duration' in result['ltpresult.sections'][suite]:
|
||||
self.ltptests[machine][suite]['duration'] = result['ltpresult.sections'][suite]['duration']
|
||||
if 'timeout' in result['ltpresult.sections'][suite]:
|
||||
self.ltptests[machine][suite]['duration'] += " T"
|
||||
return
|
||||
try:
|
||||
_, suite, test = k.split(".", 2)
|
||||
except ValueError:
|
||||
return
|
||||
# Handle 'glib-2.0'
|
||||
if 'ltpresult.sections' in result and suite not in result['ltpresult.sections']:
|
||||
try:
|
||||
_, suite, suite1, test = k.split(".", 3)
|
||||
if suite + "." + suite1 in result['ltpresult.sections']:
|
||||
suite = suite + "." + suite1
|
||||
except ValueError:
|
||||
pass
|
||||
if suite not in self.ltptests[machine]:
|
||||
self.ltptests[machine][suite] = {'passed': 0, 'failed': 0, 'skipped': 0, 'duration' : '-', 'failed_testcases': []}
|
||||
for tk in self.result_types:
|
||||
if status in self.result_types[tk]:
|
||||
self.ltptests[machine][suite][tk] += 1
|
||||
|
||||
def handle_ltpposixtest_result(self, k, status, result, machine):
|
||||
if machine not in self.ltpposixtests:
|
||||
self.ltpposixtests[machine] = {}
|
||||
|
||||
if k == 'ltpposixresult.sections':
|
||||
# Ensure tests without any test results still show up on the report
|
||||
for suite in result['ltpposixresult.sections']:
|
||||
if suite not in self.ltpposixtests[machine]:
|
||||
self.ltpposixtests[machine][suite] = {'passed': 0, 'failed': 0, 'skipped': 0, 'duration' : '-', 'failed_testcases': []}
|
||||
if 'duration' in result['ltpposixresult.sections'][suite]:
|
||||
self.ltpposixtests[machine][suite]['duration'] = result['ltpposixresult.sections'][suite]['duration']
|
||||
return
|
||||
try:
|
||||
_, suite, test = k.split(".", 2)
|
||||
except ValueError:
|
||||
return
|
||||
# Handle 'glib-2.0'
|
||||
if 'ltpposixresult.sections' in result and suite not in result['ltpposixresult.sections']:
|
||||
try:
|
||||
_, suite, suite1, test = k.split(".", 3)
|
||||
if suite + "." + suite1 in result['ltpposixresult.sections']:
|
||||
suite = suite + "." + suite1
|
||||
except ValueError:
|
||||
pass
|
||||
if suite not in self.ltpposixtests[machine]:
|
||||
self.ltpposixtests[machine][suite] = {'passed': 0, 'failed': 0, 'skipped': 0, 'duration' : '-', 'failed_testcases': []}
|
||||
for tk in self.result_types:
|
||||
if status in self.result_types[tk]:
|
||||
self.ltpposixtests[machine][suite][tk] += 1
|
||||
|
||||
def get_aggregated_test_result(self, logger, testresult, machine):
|
||||
test_count_report = {'passed': 0, 'failed': 0, 'skipped': 0, 'failed_testcases': []}
|
||||
result = testresult.get('result', [])
|
||||
for k in result:
|
||||
test_status = result[k].get('status', [])
|
||||
if k.startswith("ptestresult."):
|
||||
if not self.handle_ptest_result(k, test_status, result, machine):
|
||||
continue
|
||||
elif k.startswith("ltpresult."):
|
||||
self.handle_ltptest_result(k, test_status, result, machine)
|
||||
elif k.startswith("ltpposixresult."):
|
||||
self.handle_ltpposixtest_result(k, test_status, result, machine)
|
||||
|
||||
# process result if it was not skipped by a handler
|
||||
for tk in self.result_types:
|
||||
if test_status in self.result_types[tk]:
|
||||
test_count_report[tk] += 1
|
||||
if test_status in self.result_types['failed']:
|
||||
test_count_report['failed_testcases'].append(k)
|
||||
return test_count_report
|
||||
|
||||
def print_test_report(self, template_file_name, test_count_reports):
|
||||
from jinja2 import Environment, FileSystemLoader
|
||||
script_path = os.path.dirname(os.path.realpath(__file__))
|
||||
file_loader = FileSystemLoader(script_path + '/template')
|
||||
env = Environment(loader=file_loader, trim_blocks=True)
|
||||
template = env.get_template(template_file_name)
|
||||
havefailed = False
|
||||
reportvalues = []
|
||||
machines = []
|
||||
cols = ['passed', 'failed', 'skipped']
|
||||
maxlen = {'passed' : 0, 'failed' : 0, 'skipped' : 0, 'result_id': 0, 'testseries' : 0, 'ptest' : 0 ,'ltptest': 0, 'ltpposixtest': 0}
|
||||
for line in test_count_reports:
|
||||
total_tested = line['passed'] + line['failed'] + line['skipped']
|
||||
vals = {}
|
||||
vals['result_id'] = line['result_id']
|
||||
vals['testseries'] = line['testseries']
|
||||
vals['sort'] = line['testseries'] + "_" + line['result_id']
|
||||
vals['failed_testcases'] = line['failed_testcases']
|
||||
for k in cols:
|
||||
if total_tested:
|
||||
vals[k] = "%d (%s%%)" % (line[k], format(line[k] / total_tested * 100, '.0f'))
|
||||
else:
|
||||
vals[k] = "0 (0%)"
|
||||
for k in maxlen:
|
||||
if k in vals and len(vals[k]) > maxlen[k]:
|
||||
maxlen[k] = len(vals[k])
|
||||
reportvalues.append(vals)
|
||||
if line['failed_testcases']:
|
||||
havefailed = True
|
||||
if line['machine'] not in machines:
|
||||
machines.append(line['machine'])
|
||||
reporttotalvalues = {}
|
||||
for k in cols:
|
||||
reporttotalvalues[k] = '%s' % sum([line[k] for line in test_count_reports])
|
||||
reporttotalvalues['count'] = '%s' % len(test_count_reports)
|
||||
for (machine, report) in self.ptests.items():
|
||||
for ptest in self.ptests[machine]:
|
||||
if len(ptest) > maxlen['ptest']:
|
||||
maxlen['ptest'] = len(ptest)
|
||||
for (machine, report) in self.ltptests.items():
|
||||
for ltptest in self.ltptests[machine]:
|
||||
if len(ltptest) > maxlen['ltptest']:
|
||||
maxlen['ltptest'] = len(ltptest)
|
||||
for (machine, report) in self.ltpposixtests.items():
|
||||
for ltpposixtest in self.ltpposixtests[machine]:
|
||||
if len(ltpposixtest) > maxlen['ltpposixtest']:
|
||||
maxlen['ltpposixtest'] = len(ltpposixtest)
|
||||
output = template.render(reportvalues=reportvalues,
|
||||
reporttotalvalues=reporttotalvalues,
|
||||
havefailed=havefailed,
|
||||
machines=machines,
|
||||
ptests=self.ptests,
|
||||
ltptests=self.ltptests,
|
||||
ltpposixtests=self.ltpposixtests,
|
||||
maxlen=maxlen)
|
||||
print(output)
|
||||
|
||||
def view_test_report(self, logger, source_dir, branch, commit, tag, use_regression_map, raw_test, selected_test_case_only):
|
||||
def print_selected_testcase_result(testresults, selected_test_case_only):
|
||||
for testsuite in testresults:
|
||||
for resultid in testresults[testsuite]:
|
||||
result = testresults[testsuite][resultid]['result']
|
||||
test_case_result = result.get(selected_test_case_only, {})
|
||||
if test_case_result.get('status'):
|
||||
print('Found selected test case result for %s from %s' % (selected_test_case_only,
|
||||
resultid))
|
||||
print(test_case_result['status'])
|
||||
else:
|
||||
print('Could not find selected test case result for %s from %s' % (selected_test_case_only,
|
||||
resultid))
|
||||
if test_case_result.get('log'):
|
||||
print(test_case_result['log'])
|
||||
test_count_reports = []
|
||||
configmap = resultutils.store_map
|
||||
if use_regression_map:
|
||||
configmap = resultutils.regression_map
|
||||
if commit:
|
||||
if tag:
|
||||
logger.warning("Ignoring --tag as --commit was specified")
|
||||
tag_name = "{branch}/{commit_number}-g{commit}/{tag_number}"
|
||||
repo = GitRepo(source_dir)
|
||||
revs = gitarchive.get_test_revs(logger, repo, tag_name, branch=branch)
|
||||
rev_index = gitarchive.rev_find(revs, 'commit', commit)
|
||||
testresults = resultutils.git_get_result(repo, revs[rev_index][2], configmap=configmap)
|
||||
elif tag:
|
||||
repo = GitRepo(source_dir)
|
||||
testresults = resultutils.git_get_result(repo, [tag], configmap=configmap)
|
||||
else:
|
||||
testresults = resultutils.load_resultsdata(source_dir, configmap=configmap)
|
||||
if raw_test:
|
||||
raw_results = {}
|
||||
for testsuite in testresults:
|
||||
result = testresults[testsuite].get(raw_test, {})
|
||||
if result:
|
||||
raw_results[testsuite] = {raw_test: result}
|
||||
if raw_results:
|
||||
if selected_test_case_only:
|
||||
print_selected_testcase_result(raw_results, selected_test_case_only)
|
||||
else:
|
||||
print(json.dumps(raw_results, sort_keys=True, indent=1))
|
||||
else:
|
||||
print('Could not find raw test result for %s' % raw_test)
|
||||
return 0
|
||||
if selected_test_case_only:
|
||||
print_selected_testcase_result(testresults, selected_test_case_only)
|
||||
return 0
|
||||
for testsuite in testresults:
|
||||
for resultid in testresults[testsuite]:
|
||||
skip = False
|
||||
result = testresults[testsuite][resultid]
|
||||
machine = result['configuration']['MACHINE']
|
||||
|
||||
# Check to see if there is already results for these kinds of tests for the machine
|
||||
for key in result['result'].keys():
|
||||
testtype = str(key).split('.')[0]
|
||||
if ((machine in self.ltptests and testtype == "ltpiresult" and self.ltptests[machine]) or
|
||||
(machine in self.ltpposixtests and testtype == "ltpposixresult" and self.ltpposixtests[machine])):
|
||||
print("Already have test results for %s on %s, skipping %s" %(str(key).split('.')[0], machine, resultid))
|
||||
skip = True
|
||||
break
|
||||
if skip:
|
||||
break
|
||||
|
||||
test_count_report = self.get_aggregated_test_result(logger, result, machine)
|
||||
test_count_report['machine'] = machine
|
||||
test_count_report['testseries'] = result['configuration']['TESTSERIES']
|
||||
test_count_report['result_id'] = resultid
|
||||
test_count_reports.append(test_count_report)
|
||||
self.print_test_report('test_report_full_text.txt', test_count_reports)
|
||||
|
||||
def report(args, logger):
|
||||
report = ResultsTextReport()
|
||||
report.view_test_report(logger, args.source_dir, args.branch, args.commit, args.tag, args.use_regression_map,
|
||||
args.raw_test_only, args.selected_test_case_only)
|
||||
return 0
|
||||
|
||||
def register_commands(subparsers):
|
||||
"""Register subcommands from this plugin"""
|
||||
parser_build = subparsers.add_parser('report', help='summarise test results',
|
||||
description='print a text-based summary of the test results',
|
||||
group='analysis')
|
||||
parser_build.set_defaults(func=report)
|
||||
parser_build.add_argument('source_dir',
|
||||
help='source file/directory/URL that contain the test result files to summarise')
|
||||
parser_build.add_argument('--branch', '-B', default='master', help="Branch to find commit in")
|
||||
parser_build.add_argument('--commit', help="Revision to report")
|
||||
parser_build.add_argument('-t', '--tag', default='',
|
||||
help='source_dir is a git repository, report on the tag specified from that repository')
|
||||
parser_build.add_argument('-m', '--use_regression_map', action='store_true',
|
||||
help='instead of the default "store_map", use the "regression_map" for report')
|
||||
parser_build.add_argument('-r', '--raw_test_only', default='',
|
||||
help='output raw test result only for the user provided test result id')
|
||||
parser_build.add_argument('-s', '--selected_test_case_only', default='',
|
||||
help='output selected test case result for the user provided test case id, if both test '
|
||||
'result id and test case id are provided then output the selected test case result '
|
||||
'from the provided test result id')
|
||||
274
sources/poky/scripts/lib/resulttool/resultutils.py
Normal file
274
sources/poky/scripts/lib/resulttool/resultutils.py
Normal file
@@ -0,0 +1,274 @@
|
||||
# resulttool - common library/utility functions
|
||||
#
|
||||
# Copyright (c) 2019, Intel Corporation.
|
||||
# Copyright (c) 2019, Linux Foundation
|
||||
#
|
||||
# SPDX-License-Identifier: GPL-2.0-only
|
||||
#
|
||||
|
||||
import os
|
||||
import base64
|
||||
import zlib
|
||||
import json
|
||||
import scriptpath
|
||||
import copy
|
||||
import urllib.request
|
||||
import posixpath
|
||||
import logging
|
||||
scriptpath.add_oe_lib_path()
|
||||
|
||||
logger = logging.getLogger('resulttool')
|
||||
|
||||
flatten_map = {
|
||||
"oeselftest": [],
|
||||
"runtime": [],
|
||||
"sdk": [],
|
||||
"sdkext": [],
|
||||
"manual": []
|
||||
}
|
||||
regression_map = {
|
||||
"oeselftest": ['TEST_TYPE', 'MACHINE'],
|
||||
"runtime": ['TESTSERIES', 'TEST_TYPE', 'IMAGE_BASENAME', 'MACHINE', 'IMAGE_PKGTYPE', 'DISTRO'],
|
||||
"sdk": ['TESTSERIES', 'TEST_TYPE', 'IMAGE_BASENAME', 'MACHINE', 'SDKMACHINE'],
|
||||
"sdkext": ['TESTSERIES', 'TEST_TYPE', 'IMAGE_BASENAME', 'MACHINE', 'SDKMACHINE'],
|
||||
"manual": ['TEST_TYPE', 'TEST_MODULE', 'IMAGE_BASENAME', 'MACHINE']
|
||||
}
|
||||
store_map = {
|
||||
"oeselftest": ['TEST_TYPE', 'TESTSERIES', 'MACHINE'],
|
||||
"runtime": ['TEST_TYPE', 'DISTRO', 'MACHINE', 'IMAGE_BASENAME'],
|
||||
"sdk": ['TEST_TYPE', 'MACHINE', 'SDKMACHINE', 'IMAGE_BASENAME'],
|
||||
"sdkext": ['TEST_TYPE', 'MACHINE', 'SDKMACHINE', 'IMAGE_BASENAME'],
|
||||
"manual": ['TEST_TYPE', 'TEST_MODULE', 'MACHINE', 'IMAGE_BASENAME']
|
||||
}
|
||||
|
||||
rawlog_sections = {
|
||||
"ptestresult.rawlogs": "ptest",
|
||||
"ltpresult.rawlogs": "ltp",
|
||||
"ltpposixresult.rawlogs": "ltpposix"
|
||||
}
|
||||
|
||||
def is_url(p):
|
||||
"""
|
||||
Helper for determining if the given path is a URL
|
||||
"""
|
||||
return p.startswith('http://') or p.startswith('https://')
|
||||
|
||||
extra_configvars = {'TESTSERIES': ''}
|
||||
|
||||
#
|
||||
# Load the json file and append the results data into the provided results dict
|
||||
#
|
||||
def append_resultsdata(results, f, configmap=store_map, configvars=extra_configvars):
|
||||
if type(f) is str:
|
||||
if is_url(f):
|
||||
with urllib.request.urlopen(f) as response:
|
||||
data = json.loads(response.read().decode('utf-8'))
|
||||
url = urllib.parse.urlparse(f)
|
||||
testseries = posixpath.basename(posixpath.dirname(url.path))
|
||||
else:
|
||||
with open(f, "r") as filedata:
|
||||
try:
|
||||
data = json.load(filedata)
|
||||
except json.decoder.JSONDecodeError:
|
||||
print("Cannot decode {}. Possible corruption. Skipping.".format(f))
|
||||
data = ""
|
||||
testseries = os.path.basename(os.path.dirname(f))
|
||||
else:
|
||||
data = f
|
||||
for res in data:
|
||||
if "configuration" not in data[res] or "result" not in data[res]:
|
||||
raise ValueError("Test results data without configuration or result section?")
|
||||
for config in configvars:
|
||||
if config == "TESTSERIES" and "TESTSERIES" not in data[res]["configuration"]:
|
||||
data[res]["configuration"]["TESTSERIES"] = testseries
|
||||
continue
|
||||
if config not in data[res]["configuration"]:
|
||||
data[res]["configuration"][config] = configvars[config]
|
||||
testtype = data[res]["configuration"].get("TEST_TYPE")
|
||||
if testtype not in configmap:
|
||||
raise ValueError("Unknown test type %s" % testtype)
|
||||
testpath = "/".join(data[res]["configuration"].get(i) for i in configmap[testtype])
|
||||
if testpath not in results:
|
||||
results[testpath] = {}
|
||||
results[testpath][res] = data[res]
|
||||
|
||||
#
|
||||
# Walk a directory and find/load results data
|
||||
# or load directly from a file
|
||||
#
|
||||
def load_resultsdata(source, configmap=store_map, configvars=extra_configvars):
|
||||
results = {}
|
||||
if is_url(source) or os.path.isfile(source):
|
||||
append_resultsdata(results, source, configmap, configvars)
|
||||
return results
|
||||
for root, dirs, files in os.walk(source):
|
||||
for name in files:
|
||||
f = os.path.join(root, name)
|
||||
if name == "testresults.json":
|
||||
append_resultsdata(results, f, configmap, configvars)
|
||||
return results
|
||||
|
||||
def filter_resultsdata(results, resultid):
|
||||
newresults = {}
|
||||
for r in results:
|
||||
for i in results[r]:
|
||||
if i == resultsid:
|
||||
newresults[r] = {}
|
||||
newresults[r][i] = results[r][i]
|
||||
return newresults
|
||||
|
||||
def strip_logs(results):
|
||||
newresults = copy.deepcopy(results)
|
||||
for res in newresults:
|
||||
if 'result' not in newresults[res]:
|
||||
continue
|
||||
for logtype in rawlog_sections:
|
||||
if logtype in newresults[res]['result']:
|
||||
del newresults[res]['result'][logtype]
|
||||
if 'ptestresult.sections' in newresults[res]['result']:
|
||||
for i in newresults[res]['result']['ptestresult.sections']:
|
||||
if 'log' in newresults[res]['result']['ptestresult.sections'][i]:
|
||||
del newresults[res]['result']['ptestresult.sections'][i]['log']
|
||||
return newresults
|
||||
|
||||
# For timing numbers, crazy amounts of precision don't make sense and just confuse
|
||||
# the logs. For numbers over 1, trim to 3 decimal places, for numbers less than 1,
|
||||
# trim to 4 significant digits
|
||||
def trim_durations(results):
|
||||
for res in results:
|
||||
if 'result' not in results[res]:
|
||||
continue
|
||||
for entry in results[res]['result']:
|
||||
if 'duration' in results[res]['result'][entry]:
|
||||
duration = results[res]['result'][entry]['duration']
|
||||
if duration > 1:
|
||||
results[res]['result'][entry]['duration'] = float("%.3f" % duration)
|
||||
elif duration < 1:
|
||||
results[res]['result'][entry]['duration'] = float("%.4g" % duration)
|
||||
return results
|
||||
|
||||
def handle_cleanups(results):
|
||||
# Remove pointless path duplication from old format reproducibility results
|
||||
for res2 in results:
|
||||
try:
|
||||
section = results[res2]['result']['reproducible']['files']
|
||||
for pkgtype in section:
|
||||
for filelist in section[pkgtype].copy():
|
||||
if section[pkgtype][filelist] and type(section[pkgtype][filelist][0]) == dict:
|
||||
newlist = []
|
||||
for entry in section[pkgtype][filelist]:
|
||||
newlist.append(entry["reference"].split("/./")[1])
|
||||
section[pkgtype][filelist] = newlist
|
||||
|
||||
except KeyError:
|
||||
pass
|
||||
# Remove pointless duplicate rawlogs data
|
||||
try:
|
||||
del results[res2]['result']['reproducible.rawlogs']
|
||||
except KeyError:
|
||||
pass
|
||||
|
||||
def decode_log(logdata):
|
||||
if isinstance(logdata, str):
|
||||
return logdata
|
||||
elif isinstance(logdata, dict):
|
||||
if "compressed" in logdata:
|
||||
data = logdata.get("compressed")
|
||||
data = base64.b64decode(data.encode("utf-8"))
|
||||
data = zlib.decompress(data)
|
||||
return data.decode("utf-8", errors='ignore')
|
||||
return None
|
||||
|
||||
def generic_get_log(sectionname, results, section):
|
||||
if sectionname not in results:
|
||||
return None
|
||||
if section not in results[sectionname]:
|
||||
return None
|
||||
|
||||
ptest = results[sectionname][section]
|
||||
if 'log' not in ptest:
|
||||
return None
|
||||
return decode_log(ptest['log'])
|
||||
|
||||
def ptestresult_get_log(results, section):
|
||||
return generic_get_log('ptestresult.sections', results, section)
|
||||
|
||||
def generic_get_rawlogs(sectname, results):
|
||||
if sectname not in results:
|
||||
return None
|
||||
if 'log' not in results[sectname]:
|
||||
return None
|
||||
return decode_log(results[sectname]['log'])
|
||||
|
||||
def save_resultsdata(results, destdir, fn="testresults.json", ptestjson=False, ptestlogs=False):
|
||||
for res in results:
|
||||
if res:
|
||||
dst = destdir + "/" + res + "/" + fn
|
||||
else:
|
||||
dst = destdir + "/" + fn
|
||||
os.makedirs(os.path.dirname(dst), exist_ok=True)
|
||||
resultsout = results[res]
|
||||
if not ptestjson:
|
||||
resultsout = strip_logs(results[res])
|
||||
trim_durations(resultsout)
|
||||
handle_cleanups(resultsout)
|
||||
with open(dst, 'w') as f:
|
||||
f.write(json.dumps(resultsout, sort_keys=True, indent=1))
|
||||
for res2 in results[res]:
|
||||
if ptestlogs and 'result' in results[res][res2]:
|
||||
seriesresults = results[res][res2]['result']
|
||||
for logtype in rawlog_sections:
|
||||
logdata = generic_get_rawlogs(logtype, seriesresults)
|
||||
if logdata is not None:
|
||||
logger.info("Extracting " + rawlog_sections[logtype] + "-raw.log")
|
||||
with open(dst.replace(fn, rawlog_sections[logtype] + "-raw.log"), "w+") as f:
|
||||
f.write(logdata)
|
||||
if 'ptestresult.sections' in seriesresults:
|
||||
for i in seriesresults['ptestresult.sections']:
|
||||
sectionlog = ptestresult_get_log(seriesresults, i)
|
||||
if sectionlog is not None:
|
||||
with open(dst.replace(fn, "ptest-%s.log" % i), "w+") as f:
|
||||
f.write(sectionlog)
|
||||
|
||||
def git_get_result(repo, tags, configmap=store_map):
|
||||
git_objs = []
|
||||
for tag in tags:
|
||||
files = repo.run_cmd(['ls-tree', "--name-only", "-r", tag]).splitlines()
|
||||
git_objs.extend([tag + ':' + f for f in files if f.endswith("testresults.json")])
|
||||
|
||||
def parse_json_stream(data):
|
||||
"""Parse multiple concatenated JSON objects"""
|
||||
objs = []
|
||||
json_d = ""
|
||||
for line in data.splitlines():
|
||||
if line == '}{':
|
||||
json_d += '}'
|
||||
objs.append(json.loads(json_d))
|
||||
json_d = '{'
|
||||
else:
|
||||
json_d += line
|
||||
objs.append(json.loads(json_d))
|
||||
return objs
|
||||
|
||||
# Optimize by reading all data with one git command
|
||||
results = {}
|
||||
for obj in parse_json_stream(repo.run_cmd(['show'] + git_objs + ['--'])):
|
||||
append_resultsdata(results, obj, configmap=configmap)
|
||||
|
||||
return results
|
||||
|
||||
def test_run_results(results):
|
||||
"""
|
||||
Convenient generator function that iterates over all test runs that have a
|
||||
result section.
|
||||
|
||||
Generates a tuple of:
|
||||
(result json file path, test run name, test run (dict), test run "results" (dict))
|
||||
for each test run that has a "result" section
|
||||
"""
|
||||
for path in results:
|
||||
for run_name, test_run in results[path].items():
|
||||
if not 'result' in test_run:
|
||||
continue
|
||||
yield path, run_name, test_run, test_run['result']
|
||||
|
||||
124
sources/poky/scripts/lib/resulttool/store.py
Normal file
124
sources/poky/scripts/lib/resulttool/store.py
Normal file
@@ -0,0 +1,124 @@
|
||||
# resulttool - store test results
|
||||
#
|
||||
# Copyright (c) 2019, Intel Corporation.
|
||||
# Copyright (c) 2019, Linux Foundation
|
||||
#
|
||||
# SPDX-License-Identifier: GPL-2.0-only
|
||||
#
|
||||
|
||||
import tempfile
|
||||
import os
|
||||
import subprocess
|
||||
import json
|
||||
import shutil
|
||||
import scriptpath
|
||||
scriptpath.add_bitbake_lib_path()
|
||||
scriptpath.add_oe_lib_path()
|
||||
import resulttool.resultutils as resultutils
|
||||
import oeqa.utils.gitarchive as gitarchive
|
||||
|
||||
|
||||
def store(args, logger):
|
||||
tempdir = tempfile.mkdtemp(prefix='testresults.')
|
||||
try:
|
||||
configvars = resultutils.extra_configvars.copy()
|
||||
if args.executed_by:
|
||||
configvars['EXECUTED_BY'] = args.executed_by
|
||||
if args.extra_test_env:
|
||||
configvars['EXTRA_TEST_ENV'] = args.extra_test_env
|
||||
results = {}
|
||||
logger.info('Reading files from %s' % args.source)
|
||||
if resultutils.is_url(args.source) or os.path.isfile(args.source):
|
||||
resultutils.append_resultsdata(results, args.source, configvars=configvars)
|
||||
else:
|
||||
for root, dirs, files in os.walk(args.source):
|
||||
for name in files:
|
||||
f = os.path.join(root, name)
|
||||
if name == "testresults.json":
|
||||
resultutils.append_resultsdata(results, f, configvars=configvars)
|
||||
elif args.all:
|
||||
dst = f.replace(args.source, tempdir + "/")
|
||||
os.makedirs(os.path.dirname(dst), exist_ok=True)
|
||||
shutil.copyfile(f, dst)
|
||||
|
||||
revisions = {}
|
||||
|
||||
if not results and not args.all:
|
||||
if args.allow_empty:
|
||||
logger.info("No results found to store")
|
||||
return 0
|
||||
logger.error("No results found to store")
|
||||
return 1
|
||||
|
||||
# Find the branch/commit/commit_count and ensure they all match
|
||||
for suite in results:
|
||||
for result in results[suite]:
|
||||
config = results[suite][result]['configuration']['LAYERS']['meta']
|
||||
revision = (config['commit'], config['branch'], str(config['commit_count']))
|
||||
if revision not in revisions:
|
||||
revisions[revision] = {}
|
||||
if suite not in revisions[revision]:
|
||||
revisions[revision][suite] = {}
|
||||
revisions[revision][suite][result] = results[suite][result]
|
||||
|
||||
logger.info("Found %d revisions to store" % len(revisions))
|
||||
|
||||
for r in revisions:
|
||||
results = revisions[r]
|
||||
if args.revision and r[0] != args.revision:
|
||||
logger.info('skipping %s as non-matching' % r[0])
|
||||
continue
|
||||
keywords = {'commit': r[0], 'branch': r[1], "commit_count": r[2]}
|
||||
subprocess.check_call(["find", tempdir, "-name", "testresults.json", "!", "-path", "./.git/*", "-delete"])
|
||||
resultutils.save_resultsdata(results, tempdir, ptestlogs=True)
|
||||
|
||||
logger.info('Storing test result into git repository %s' % args.git_dir)
|
||||
|
||||
excludes = []
|
||||
if args.logfile_archive:
|
||||
excludes = ['*.log', "*.log.zst"]
|
||||
|
||||
tagname = gitarchive.gitarchive(tempdir, args.git_dir, False, False,
|
||||
"Results of {branch}:{commit}", "branch: {branch}\ncommit: {commit}", "{branch}",
|
||||
False, "{branch}/{commit_count}-g{commit}/{tag_number}",
|
||||
'Test run #{tag_number} of {branch}:{commit}', '',
|
||||
excludes, [], False, keywords, logger)
|
||||
|
||||
if args.logfile_archive:
|
||||
logdir = args.logfile_archive + "/" + tagname
|
||||
shutil.copytree(tempdir, logdir)
|
||||
for root, dirs, files in os.walk(logdir):
|
||||
for name in files:
|
||||
if not name.endswith(".log"):
|
||||
continue
|
||||
f = os.path.join(root, name)
|
||||
subprocess.run(["zstd", f, "--rm"], check=True, capture_output=True)
|
||||
finally:
|
||||
subprocess.check_call(["rm", "-rf", tempdir])
|
||||
|
||||
return 0
|
||||
|
||||
def register_commands(subparsers):
|
||||
"""Register subcommands from this plugin"""
|
||||
parser_build = subparsers.add_parser('store', help='store test results into a git repository',
|
||||
description='takes a results file or directory of results files and stores '
|
||||
'them into the destination git repository, splitting out the results '
|
||||
'files as configured',
|
||||
group='setup')
|
||||
parser_build.set_defaults(func=store)
|
||||
parser_build.add_argument('source',
|
||||
help='source file/directory/URL that contain the test result files to be stored')
|
||||
parser_build.add_argument('git_dir',
|
||||
help='the location of the git repository to store the results in')
|
||||
parser_build.add_argument('-a', '--all', action='store_true',
|
||||
help='include all files, not just testresults.json files')
|
||||
parser_build.add_argument('-e', '--allow-empty', action='store_true',
|
||||
help='don\'t error if no results to store are found')
|
||||
parser_build.add_argument('-x', '--executed-by', default='',
|
||||
help='add executed-by configuration to each result file')
|
||||
parser_build.add_argument('-t', '--extra-test-env', default='',
|
||||
help='add extra test environment data to each result file configuration')
|
||||
parser_build.add_argument('-r', '--revision', default='',
|
||||
help='only store data for the specified revision')
|
||||
parser_build.add_argument('-l', '--logfile-archive', default='',
|
||||
help='directory to separately archive log files along with a copy of the results')
|
||||
@@ -0,0 +1,79 @@
|
||||
==============================================================================================================
|
||||
Test Result Status Summary (Counts/Percentages sorted by testseries, ID)
|
||||
==============================================================================================================
|
||||
--------------------------------------------------------------------------------------------------------------
|
||||
{{ 'Test Series'.ljust(maxlen['testseries']) }} | {{ 'ID'.ljust(maxlen['result_id']) }} | {{ 'Passed'.ljust(maxlen['passed']) }} | {{ 'Failed'.ljust(maxlen['failed']) }} | {{ 'Skipped'.ljust(maxlen['skipped']) }}
|
||||
--------------------------------------------------------------------------------------------------------------
|
||||
{% for report in reportvalues |sort(attribute='sort') %}
|
||||
{{ report.testseries.ljust(maxlen['testseries']) }} | {{ report.result_id.ljust(maxlen['result_id']) }} | {{ (report.passed|string).ljust(maxlen['passed']) }} | {{ (report.failed|string).ljust(maxlen['failed']) }} | {{ (report.skipped|string).ljust(maxlen['skipped']) }}
|
||||
{% endfor %}
|
||||
--------------------------------------------------------------------------------------------------------------
|
||||
{{ 'Total'.ljust(maxlen['testseries']) }} | {{ reporttotalvalues['count'].ljust(maxlen['result_id']) }} | {{ reporttotalvalues['passed'].ljust(maxlen['passed']) }} | {{ reporttotalvalues['failed'].ljust(maxlen['failed']) }} | {{ reporttotalvalues['skipped'].ljust(maxlen['skipped']) }}
|
||||
--------------------------------------------------------------------------------------------------------------
|
||||
|
||||
{% for machine in machines %}
|
||||
{% if ptests[machine] %}
|
||||
==============================================================================================================
|
||||
{{ machine }} PTest Result Summary
|
||||
==============================================================================================================
|
||||
--------------------------------------------------------------------------------------------------------------
|
||||
{{ 'Recipe'.ljust(maxlen['ptest']) }} | {{ 'Passed'.ljust(maxlen['passed']) }} | {{ 'Failed'.ljust(maxlen['failed']) }} | {{ 'Skipped'.ljust(maxlen['skipped']) }} | {{ 'Time(s)'.ljust(10) }}
|
||||
--------------------------------------------------------------------------------------------------------------
|
||||
{% for ptest in ptests[machine] |sort %}
|
||||
{{ ptest.ljust(maxlen['ptest']) }} | {{ (ptests[machine][ptest]['passed']|string).ljust(maxlen['passed']) }} | {{ (ptests[machine][ptest]['failed']|string).ljust(maxlen['failed']) }} | {{ (ptests[machine][ptest]['skipped']|string).ljust(maxlen['skipped']) }} | {{ (ptests[machine][ptest]['duration']|string) }}
|
||||
{% endfor %}
|
||||
--------------------------------------------------------------------------------------------------------------
|
||||
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
|
||||
{% for machine in machines %}
|
||||
{% if ltptests[machine] %}
|
||||
==============================================================================================================
|
||||
{{ machine }} Ltp Test Result Summary
|
||||
==============================================================================================================
|
||||
--------------------------------------------------------------------------------------------------------------
|
||||
{{ 'Recipe'.ljust(maxlen['ltptest']) }} | {{ 'Passed'.ljust(maxlen['passed']) }} | {{ 'Failed'.ljust(maxlen['failed']) }} | {{ 'Skipped'.ljust(maxlen['skipped']) }} | {{ 'Time(s)'.ljust(10) }}
|
||||
--------------------------------------------------------------------------------------------------------------
|
||||
{% for ltptest in ltptests[machine] |sort %}
|
||||
{{ ltptest.ljust(maxlen['ltptest']) }} | {{ (ltptests[machine][ltptest]['passed']|string).ljust(maxlen['passed']) }} | {{ (ltptests[machine][ltptest]['failed']|string).ljust(maxlen['failed']) }} | {{ (ltptests[machine][ltptest]['skipped']|string).ljust(maxlen['skipped']) }} | {{ (ltptests[machine][ltptest]['duration']|string) }}
|
||||
{% endfor %}
|
||||
--------------------------------------------------------------------------------------------------------------
|
||||
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
|
||||
{% for machine in machines %}
|
||||
{% if ltpposixtests[machine] %}
|
||||
==============================================================================================================
|
||||
{{ machine }} Ltp Posix Result Summary
|
||||
==============================================================================================================
|
||||
--------------------------------------------------------------------------------------------------------------
|
||||
{{ 'Recipe'.ljust(maxlen['ltpposixtest']) }} | {{ 'Passed'.ljust(maxlen['passed']) }} | {{ 'Failed'.ljust(maxlen['failed']) }} | {{ 'Skipped'.ljust(maxlen['skipped']) }} | {{ 'Time(s)'.ljust(10) }}
|
||||
--------------------------------------------------------------------------------------------------------------
|
||||
{% for ltpposixtest in ltpposixtests[machine] |sort %}
|
||||
{{ ltpposixtest.ljust(maxlen['ltpposixtest']) }} | {{ (ltpposixtests[machine][ltpposixtest]['passed']|string).ljust(maxlen['passed']) }} | {{ (ltpposixtests[machine][ltpposixtest]['failed']|string).ljust(maxlen['failed']) }} | {{ (ltpposixtests[machine][ltpposixtest]['skipped']|string).ljust(maxlen['skipped']) }} | {{ (ltpposixtests[machine][ltpposixtest]['duration']|string) }}
|
||||
{% endfor %}
|
||||
--------------------------------------------------------------------------------------------------------------
|
||||
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
|
||||
|
||||
==============================================================================================================
|
||||
Failed test cases (sorted by testseries, ID)
|
||||
==============================================================================================================
|
||||
{% if havefailed %}
|
||||
--------------------------------------------------------------------------------------------------------------
|
||||
{% for report in reportvalues |sort(attribute='sort') %}
|
||||
{% if report.failed_testcases %}
|
||||
testseries | result_id : {{ report.testseries }} | {{ report.result_id }}
|
||||
{% for testcase in report.failed_testcases %}
|
||||
{{ testcase }}
|
||||
{% endfor %}
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
--------------------------------------------------------------------------------------------------------------
|
||||
{% else %}
|
||||
There were no test failures
|
||||
{% endif %}
|
||||
Reference in New Issue
Block a user