1# 2# Copyright (C) 2017 Intel Corporation 3# 4# SPDX-License-Identifier: MIT 5# 6 7import os 8import time 9import glob 10import sys 11import importlib 12import subprocess 13import unittest 14from random import choice 15 16import oeqa 17import oe 18import bb.utils 19 20from oeqa.core.context import OETestContext, OETestContextExecutor 21from oeqa.core.exception import OEQAPreRun, OEQATestNotFound 22 23from oeqa.utils.commands import runCmd, get_bb_vars, get_test_layer 24 25class NonConcurrentTestSuite(unittest.TestSuite): 26 def __init__(self, suite, processes, setupfunc, removefunc): 27 super().__init__([suite]) 28 self.processes = processes 29 self.suite = suite 30 self.setupfunc = setupfunc 31 self.removefunc = removefunc 32 33 def run(self, result): 34 (builddir, newbuilddir) = self.setupfunc("-st", None, self.suite) 35 ret = super().run(result) 36 os.chdir(builddir) 37 if newbuilddir and ret.wasSuccessful() and self.removefunc: 38 self.removefunc(newbuilddir) 39 40def removebuilddir(d): 41 delay = 5 42 while delay and (os.path.exists(d + "/bitbake.lock") or os.path.exists(d + "/cache/hashserv.db-wal")): 43 time.sleep(1) 44 delay = delay - 1 45 # Deleting these directories takes a lot of time, use autobuilder 46 # clobberdir if its available 47 clobberdir = os.path.expanduser("~/yocto-autobuilder-helper/janitor/clobberdir") 48 if os.path.exists(clobberdir): 49 try: 50 subprocess.check_call([clobberdir, d]) 51 return 52 except subprocess.CalledProcessError: 53 pass 54 bb.utils.prunedir(d, ionice=True) 55 56class OESelftestTestContext(OETestContext): 57 def __init__(self, td=None, logger=None, machines=None, config_paths=None, newbuilddir=None, keep_builddir=None): 58 super(OESelftestTestContext, self).__init__(td, logger) 59 60 self.machines = machines 61 self.custommachine = None 62 self.config_paths = config_paths 63 self.newbuilddir = newbuilddir 64 65 if keep_builddir: 66 self.removebuilddir = None 67 else: 68 self.removebuilddir = removebuilddir 69 70 def setup_builddir(self, suffix, selftestdir, suite): 71 builddir = os.environ['BUILDDIR'] 72 if not selftestdir: 73 selftestdir = get_test_layer() 74 if self.newbuilddir: 75 newbuilddir = os.path.join(self.newbuilddir, 'build' + suffix) 76 else: 77 newbuilddir = builddir + suffix 78 newselftestdir = newbuilddir + "/meta-selftest" 79 80 if os.path.exists(newbuilddir): 81 self.logger.error("Build directory %s already exists, aborting" % newbuilddir) 82 sys.exit(1) 83 84 bb.utils.mkdirhier(newbuilddir) 85 oe.path.copytree(builddir + "/conf", newbuilddir + "/conf") 86 oe.path.copytree(builddir + "/cache", newbuilddir + "/cache") 87 oe.path.copytree(selftestdir, newselftestdir) 88 89 for e in os.environ: 90 if builddir + "/" in os.environ[e]: 91 os.environ[e] = os.environ[e].replace(builddir + "/", newbuilddir + "/") 92 if os.environ[e].endswith(builddir): 93 os.environ[e] = os.environ[e].replace(builddir, newbuilddir) 94 95 subprocess.check_output("git init; git add *; git commit -a -m 'initial'", cwd=newselftestdir, shell=True) 96 97 # Tried to used bitbake-layers add/remove but it requires recipe parsing and hence is too slow 98 subprocess.check_output("sed %s/conf/bblayers.conf -i -e 's#%s#%s#g'" % (newbuilddir, selftestdir, newselftestdir), cwd=newbuilddir, shell=True) 99 100 os.chdir(newbuilddir) 101 102 def patch_test(t): 103 if not hasattr(t, "tc"): 104 return 105 cp = t.tc.config_paths 106 for p in cp: 107 if selftestdir in cp[p] and newselftestdir not in cp[p]: 108 cp[p] = cp[p].replace(selftestdir, newselftestdir) 109 if builddir in cp[p] and newbuilddir not in cp[p]: 110 cp[p] = cp[p].replace(builddir, newbuilddir) 111 112 def patch_suite(s): 113 for x in s: 114 if isinstance(x, unittest.TestSuite): 115 patch_suite(x) 116 else: 117 patch_test(x) 118 119 patch_suite(suite) 120 121 return (builddir, newbuilddir) 122 123 def prepareSuite(self, suites, processes): 124 if processes: 125 from oeqa.core.utils.concurrencytest import ConcurrentTestSuite 126 127 return ConcurrentTestSuite(suites, processes, self.setup_builddir, self.removebuilddir) 128 else: 129 return NonConcurrentTestSuite(suites, processes, self.setup_builddir, self.removebuilddir) 130 131 def runTests(self, processes=None, machine=None, skips=[]): 132 if machine: 133 self.custommachine = machine 134 if machine == 'random': 135 self.custommachine = choice(self.machines) 136 self.logger.info('Run tests with custom MACHINE set to: %s' % \ 137 self.custommachine) 138 return super(OESelftestTestContext, self).runTests(processes, skips) 139 140 def listTests(self, display_type, machine=None): 141 return super(OESelftestTestContext, self).listTests(display_type) 142 143class OESelftestTestContextExecutor(OETestContextExecutor): 144 _context_class = OESelftestTestContext 145 _script_executor = 'oe-selftest' 146 147 name = 'oe-selftest' 148 help = 'oe-selftest test component' 149 description = 'Executes selftest tests' 150 151 def register_commands(self, logger, parser): 152 group = parser.add_mutually_exclusive_group(required=True) 153 154 group.add_argument('-a', '--run-all-tests', default=False, 155 action="store_true", dest="run_all_tests", 156 help='Run all (unhidden) tests') 157 group.add_argument('-R', '--skip-tests', required=False, action='store', 158 nargs='+', dest="skips", default=None, 159 help='Run all (unhidden) tests except the ones specified. Format should be <module>[.<class>[.<test_method>]]') 160 group.add_argument('-r', '--run-tests', required=False, action='store', 161 nargs='+', dest="run_tests", default=None, 162 help='Select what tests to run (modules, classes or test methods). Format should be: <module>.<class>.<test_method>') 163 164 group.add_argument('-m', '--list-modules', required=False, 165 action="store_true", default=False, 166 help='List all available test modules.') 167 group.add_argument('--list-classes', required=False, 168 action="store_true", default=False, 169 help='List all available test classes.') 170 group.add_argument('-l', '--list-tests', required=False, 171 action="store_true", default=False, 172 help='List all available tests.') 173 174 parser.add_argument('-j', '--num-processes', dest='processes', action='store', 175 type=int, help="number of processes to execute in parallel with") 176 177 parser.add_argument('--machine', required=False, choices=['random', 'all'], 178 help='Run tests on different machines (random/all).') 179 180 parser.add_argument('-t', '--select-tag', dest="select_tags", 181 action='append', default=None, 182 help='Filter all (unhidden) tests to any that match any of the specified tag(s).') 183 parser.add_argument('-T', '--exclude-tag', dest="exclude_tags", 184 action='append', default=None, 185 help='Exclude all (unhidden) tests that match any of the specified tag(s). (exclude applies before select)') 186 187 parser.add_argument('-K', '--keep-builddir', action='store_true', 188 help='Keep the test build directory even if all tests pass') 189 190 parser.add_argument('-B', '--newbuilddir', help='New build directory to use for tests.') 191 parser.add_argument('-v', '--verbose', action='store_true') 192 parser.set_defaults(func=self.run) 193 194 def _get_available_machines(self): 195 machines = [] 196 197 bbpath = self.tc_kwargs['init']['td']['BBPATH'].split(':') 198 199 for path in bbpath: 200 found_machines = glob.glob(os.path.join(path, 'conf', 'machine', '*.conf')) 201 if found_machines: 202 for i in found_machines: 203 # eg: '/home/<user>/poky/meta-intel/conf/machine/intel-core2-32.conf' 204 machines.append(os.path.splitext(os.path.basename(i))[0]) 205 206 return machines 207 208 def _get_cases_paths(self, bbpath): 209 cases_paths = [] 210 for layer in bbpath: 211 cases_dir = os.path.join(layer, 'lib', 'oeqa', 'selftest', 'cases') 212 if os.path.isdir(cases_dir): 213 cases_paths.append(cases_dir) 214 return cases_paths 215 216 def _process_args(self, logger, args): 217 args.test_start_time = time.strftime("%Y%m%d%H%M%S") 218 args.test_data_file = None 219 args.CASES_PATHS = None 220 221 bbvars = get_bb_vars() 222 logdir = os.environ.get("BUILDDIR") 223 if 'LOG_DIR' in bbvars: 224 logdir = bbvars['LOG_DIR'] 225 bb.utils.mkdirhier(logdir) 226 args.output_log = logdir + '/%s-results-%s.log' % (self.name, args.test_start_time) 227 228 super(OESelftestTestContextExecutor, self)._process_args(logger, args) 229 230 if args.list_modules: 231 args.list_tests = 'module' 232 elif args.list_classes: 233 args.list_tests = 'class' 234 elif args.list_tests: 235 args.list_tests = 'name' 236 237 self.tc_kwargs['init']['td'] = bbvars 238 self.tc_kwargs['init']['machines'] = self._get_available_machines() 239 240 builddir = os.environ.get("BUILDDIR") 241 self.tc_kwargs['init']['config_paths'] = {} 242 self.tc_kwargs['init']['config_paths']['testlayer_path'] = get_test_layer() 243 self.tc_kwargs['init']['config_paths']['builddir'] = builddir 244 self.tc_kwargs['init']['config_paths']['localconf'] = os.path.join(builddir, "conf/local.conf") 245 self.tc_kwargs['init']['config_paths']['bblayers'] = os.path.join(builddir, "conf/bblayers.conf") 246 self.tc_kwargs['init']['newbuilddir'] = args.newbuilddir 247 self.tc_kwargs['init']['keep_builddir'] = args.keep_builddir 248 249 def tag_filter(tags): 250 if args.exclude_tags: 251 if any(tag in args.exclude_tags for tag in tags): 252 return True 253 if args.select_tags: 254 if not tags or not any(tag in args.select_tags for tag in tags): 255 return True 256 return False 257 258 if args.select_tags or args.exclude_tags: 259 self.tc_kwargs['load']['tags_filter'] = tag_filter 260 261 self.tc_kwargs['run']['skips'] = args.skips 262 self.tc_kwargs['run']['processes'] = args.processes 263 264 def _pre_run(self): 265 def _check_required_env_variables(vars): 266 for var in vars: 267 if not os.environ.get(var): 268 self.tc.logger.error("%s is not set. Did you forget to source your build environment setup script?" % var) 269 raise OEQAPreRun 270 271 def _check_presence_meta_selftest(): 272 builddir = os.environ.get("BUILDDIR") 273 if os.getcwd() != builddir: 274 self.tc.logger.info("Changing cwd to %s" % builddir) 275 os.chdir(builddir) 276 277 if not "meta-selftest" in self.tc.td["BBLAYERS"]: 278 self.tc.logger.warning("meta-selftest layer not found in BBLAYERS, adding it") 279 meta_selftestdir = os.path.join( 280 self.tc.td["BBLAYERS_FETCH_DIR"], 'meta-selftest') 281 if os.path.isdir(meta_selftestdir): 282 runCmd("bitbake-layers add-layer %s" %meta_selftestdir) 283 # reload data is needed because a meta-selftest layer was add 284 self.tc.td = get_bb_vars() 285 self.tc.config_paths['testlayer_path'] = get_test_layer() 286 else: 287 self.tc.logger.error("could not locate meta-selftest in:\n%s" % meta_selftestdir) 288 raise OEQAPreRun 289 290 def _add_layer_libs(): 291 bbpath = self.tc.td['BBPATH'].split(':') 292 layer_libdirs = [p for p in (os.path.join(l, 'lib') \ 293 for l in bbpath) if os.path.exists(p)] 294 if layer_libdirs: 295 self.tc.logger.info("Adding layer libraries:") 296 for l in layer_libdirs: 297 self.tc.logger.info("\t%s" % l) 298 299 sys.path.extend(layer_libdirs) 300 importlib.reload(oeqa.selftest) 301 302 _check_required_env_variables(["BUILDDIR"]) 303 _check_presence_meta_selftest() 304 305 if "buildhistory.bbclass" in self.tc.td["BBINCLUDED"]: 306 self.tc.logger.error("You have buildhistory enabled already and this isn't recommended for selftest, please disable it first.") 307 raise OEQAPreRun 308 309 if "rm_work.bbclass" in self.tc.td["BBINCLUDED"]: 310 self.tc.logger.error("You have rm_work enabled which isn't recommended while running oe-selftest. Please disable it before continuing.") 311 raise OEQAPreRun 312 313 if "PRSERV_HOST" in self.tc.td: 314 self.tc.logger.error("Please unset PRSERV_HOST in order to run oe-selftest") 315 raise OEQAPreRun 316 317 if "SANITY_TESTED_DISTROS" in self.tc.td: 318 self.tc.logger.error("Please unset SANITY_TESTED_DISTROS in order to run oe-selftest") 319 raise OEQAPreRun 320 321 _add_layer_libs() 322 323 self.tc.logger.info("Running bitbake -e to test the configuration is valid/parsable") 324 runCmd("bitbake -e") 325 326 def get_json_result_dir(self, args): 327 json_result_dir = os.path.join(self.tc.td["LOG_DIR"], 'oeqa') 328 if "OEQA_JSON_RESULT_DIR" in self.tc.td: 329 json_result_dir = self.tc.td["OEQA_JSON_RESULT_DIR"] 330 331 return json_result_dir 332 333 def get_configuration(self, args): 334 import platform 335 from oeqa.utils.metadata import metadata_from_bb 336 metadata = metadata_from_bb() 337 configuration = {'TEST_TYPE': 'oeselftest', 338 'STARTTIME': args.test_start_time, 339 'MACHINE': self.tc.td["MACHINE"], 340 'HOST_DISTRO': oe.lsb.distro_identifier().replace(' ', '-'), 341 'HOST_NAME': metadata['hostname'], 342 'LAYERS': metadata['layers']} 343 return configuration 344 345 def get_result_id(self, configuration): 346 return '%s_%s_%s_%s' % (configuration['TEST_TYPE'], configuration['HOST_DISTRO'], configuration['MACHINE'], configuration['STARTTIME']) 347 348 def _internal_run(self, logger, args): 349 self.module_paths = self._get_cases_paths( 350 self.tc_kwargs['init']['td']['BBPATH'].split(':')) 351 352 self.tc = self._context_class(**self.tc_kwargs['init']) 353 try: 354 self.tc.loadTests(self.module_paths, **self.tc_kwargs['load']) 355 except OEQATestNotFound as ex: 356 logger.error(ex) 357 sys.exit(1) 358 359 if args.list_tests: 360 rc = self.tc.listTests(args.list_tests, **self.tc_kwargs['list']) 361 else: 362 self._pre_run() 363 rc = self.tc.runTests(**self.tc_kwargs['run']) 364 configuration = self.get_configuration(args) 365 rc.logDetails(self.get_json_result_dir(args), 366 configuration, 367 self.get_result_id(configuration)) 368 rc.logSummary(self.name) 369 370 return rc 371 372 def run(self, logger, args): 373 self._process_args(logger, args) 374 375 rc = None 376 try: 377 if args.machine: 378 logger.info('Custom machine mode enabled. MACHINE set to %s' % 379 args.machine) 380 381 if args.machine == 'all': 382 results = [] 383 for m in self.tc_kwargs['init']['machines']: 384 self.tc_kwargs['run']['machine'] = m 385 results.append(self._internal_run(logger, args)) 386 387 # XXX: the oe-selftest script only needs to know if one 388 # machine run fails 389 for r in results: 390 rc = r 391 if not r.wasSuccessful(): 392 break 393 394 else: 395 self.tc_kwargs['run']['machine'] = args.machine 396 return self._internal_run(logger, args) 397 398 else: 399 self.tc_kwargs['run']['machine'] = args.machine 400 rc = self._internal_run(logger, args) 401 finally: 402 config_paths = self.tc_kwargs['init']['config_paths'] 403 404 output_link = os.path.join(os.path.dirname(args.output_log), 405 "%s-results.log" % self.name) 406 if os.path.lexists(output_link): 407 os.remove(output_link) 408 os.symlink(args.output_log, output_link) 409 410 return rc 411 412_executor_class = OESelftestTestContextExecutor 413