1# 2# SPDX-License-Identifier: MIT 3# 4 5import os 6import shutil 7import tempfile 8import urllib.parse 9 10from oeqa.utils.commands import runCmd, bitbake, get_bb_var 11from oeqa.utils.commands import get_bb_vars, create_temp_layer 12from oeqa.selftest.cases import devtool 13 14templayerdir = None 15 16def setUpModule(): 17 global templayerdir 18 templayerdir = tempfile.mkdtemp(prefix='recipetoolqa') 19 create_temp_layer(templayerdir, 'selftestrecipetool') 20 runCmd('bitbake-layers add-layer %s' % templayerdir) 21 22 23def tearDownModule(): 24 runCmd('bitbake-layers remove-layer %s' % templayerdir, ignore_status=True) 25 runCmd('rm -rf %s' % templayerdir) 26 27 28class RecipetoolBase(devtool.DevtoolTestCase): 29 30 def setUpLocal(self): 31 super(RecipetoolBase, self).setUpLocal() 32 self.templayerdir = templayerdir 33 self.tempdir = tempfile.mkdtemp(prefix='recipetoolqa') 34 self.track_for_cleanup(self.tempdir) 35 self.testfile = os.path.join(self.tempdir, 'testfile') 36 with open(self.testfile, 'w') as f: 37 f.write('Test file\n') 38 39 def tearDownLocal(self): 40 runCmd('rm -rf %s/recipes-*' % self.templayerdir) 41 super(RecipetoolBase, self).tearDownLocal() 42 43 def _try_recipetool_appendcmd(self, cmd, testrecipe, expectedfiles, expectedlines=None): 44 result = runCmd(cmd) 45 self.assertNotIn('Traceback', result.output) 46 47 # Check the bbappend was created and applies properly 48 recipefile = get_bb_var('FILE', testrecipe) 49 bbappendfile = self._check_bbappend(testrecipe, recipefile, self.templayerdir) 50 51 # Check the bbappend contents 52 if expectedlines is not None: 53 with open(bbappendfile, 'r') as f: 54 self.assertEqual(expectedlines, f.readlines(), "Expected lines are not present in %s" % bbappendfile) 55 56 # Check file was copied 57 filesdir = os.path.join(os.path.dirname(bbappendfile), testrecipe) 58 for expectedfile in expectedfiles: 59 self.assertTrue(os.path.isfile(os.path.join(filesdir, expectedfile)), 'Expected file %s to be copied next to bbappend, but it wasn\'t' % expectedfile) 60 61 # Check no other files created 62 createdfiles = [] 63 for root, _, files in os.walk(filesdir): 64 for f in files: 65 createdfiles.append(os.path.relpath(os.path.join(root, f), filesdir)) 66 self.assertTrue(sorted(createdfiles), sorted(expectedfiles)) 67 68 return bbappendfile, result.output 69 70 71class RecipetoolAppendTests(RecipetoolBase): 72 73 @classmethod 74 def setUpClass(cls): 75 super(RecipetoolAppendTests, cls).setUpClass() 76 # Ensure we have the right data in shlibs/pkgdata 77 cls.logger.info('Running bitbake to generate pkgdata') 78 bitbake('-c packagedata base-files coreutils busybox selftest-recipetool-appendfile') 79 bb_vars = get_bb_vars(['COREBASE']) 80 cls.corebase = bb_vars['COREBASE'] 81 82 def _try_recipetool_appendfile(self, testrecipe, destfile, newfile, options, expectedlines, expectedfiles): 83 cmd = 'recipetool appendfile %s %s %s %s' % (self.templayerdir, destfile, newfile, options) 84 return self._try_recipetool_appendcmd(cmd, testrecipe, expectedfiles, expectedlines) 85 86 def _try_recipetool_appendfile_fail(self, destfile, newfile, checkerror): 87 cmd = 'recipetool appendfile %s %s %s' % (self.templayerdir, destfile, newfile) 88 result = runCmd(cmd, ignore_status=True) 89 self.assertNotEqual(result.status, 0, 'Command "%s" should have failed but didn\'t' % cmd) 90 self.assertNotIn('Traceback', result.output) 91 for errorstr in checkerror: 92 self.assertIn(errorstr, result.output) 93 94 def test_recipetool_appendfile_basic(self): 95 # Basic test 96 expectedlines = ['FILESEXTRAPATHS:prepend := "${THISDIR}/${PN}:"\n', 97 '\n'] 98 _, output = self._try_recipetool_appendfile('base-files', '/etc/motd', self.testfile, '', expectedlines, ['motd']) 99 self.assertNotIn('WARNING: ', output) 100 101 def test_recipetool_appendfile_invalid(self): 102 # Test some commands that should error 103 self._try_recipetool_appendfile_fail('/etc/passwd', self.testfile, ['ERROR: /etc/passwd cannot be handled by this tool', 'useradd', 'extrausers']) 104 self._try_recipetool_appendfile_fail('/etc/timestamp', self.testfile, ['ERROR: /etc/timestamp cannot be handled by this tool']) 105 self._try_recipetool_appendfile_fail('/dev/console', self.testfile, ['ERROR: /dev/console cannot be handled by this tool']) 106 107 def test_recipetool_appendfile_alternatives(self): 108 # Now try with a file we know should be an alternative 109 # (this is very much a fake example, but one we know is reliably an alternative) 110 self._try_recipetool_appendfile_fail('/bin/ls', self.testfile, ['ERROR: File /bin/ls is an alternative possibly provided by the following recipes:', 'coreutils', 'busybox']) 111 # Need a test file - should be executable 112 testfile2 = os.path.join(self.corebase, 'oe-init-build-env') 113 testfile2name = os.path.basename(testfile2) 114 expectedlines = ['FILESEXTRAPATHS:prepend := "${THISDIR}/${PN}:"\n', 115 '\n', 116 'SRC_URI += "file://%s"\n' % testfile2name, 117 '\n', 118 'do_install:append() {\n', 119 ' install -d ${D}${base_bindir}\n', 120 ' install -m 0755 ${WORKDIR}/%s ${D}${base_bindir}/ls\n' % testfile2name, 121 '}\n'] 122 self._try_recipetool_appendfile('coreutils', '/bin/ls', testfile2, '-r coreutils', expectedlines, [testfile2name]) 123 # Now try bbappending the same file again, contents should not change 124 bbappendfile, _ = self._try_recipetool_appendfile('coreutils', '/bin/ls', self.testfile, '-r coreutils', expectedlines, [testfile2name]) 125 # But file should have 126 copiedfile = os.path.join(os.path.dirname(bbappendfile), 'coreutils', testfile2name) 127 result = runCmd('diff -q %s %s' % (testfile2, copiedfile), ignore_status=True) 128 self.assertNotEqual(result.status, 0, 'New file should have been copied but was not %s' % result.output) 129 130 def test_recipetool_appendfile_binary(self): 131 # Try appending a binary file 132 # /bin/ls can be a symlink to /usr/bin/ls 133 ls = os.path.realpath("/bin/ls") 134 result = runCmd('recipetool appendfile %s /bin/ls %s -r coreutils' % (self.templayerdir, ls)) 135 self.assertIn('WARNING: ', result.output) 136 self.assertIn('is a binary', result.output) 137 138 def test_recipetool_appendfile_add(self): 139 # Try arbitrary file add to a recipe 140 expectedlines = ['FILESEXTRAPATHS:prepend := "${THISDIR}/${PN}:"\n', 141 '\n', 142 'SRC_URI += "file://testfile"\n', 143 '\n', 144 'do_install:append() {\n', 145 ' install -d ${D}${datadir}\n', 146 ' install -m 0644 ${WORKDIR}/testfile ${D}${datadir}/something\n', 147 '}\n'] 148 self._try_recipetool_appendfile('netbase', '/usr/share/something', self.testfile, '-r netbase', expectedlines, ['testfile']) 149 # Try adding another file, this time where the source file is executable 150 # (so we're testing that, plus modifying an existing bbappend) 151 testfile2 = os.path.join(self.corebase, 'oe-init-build-env') 152 testfile2name = os.path.basename(testfile2) 153 expectedlines = ['FILESEXTRAPATHS:prepend := "${THISDIR}/${PN}:"\n', 154 '\n', 155 'SRC_URI += "file://testfile \\\n', 156 ' file://%s \\\n' % testfile2name, 157 ' "\n', 158 '\n', 159 'do_install:append() {\n', 160 ' install -d ${D}${datadir}\n', 161 ' install -m 0644 ${WORKDIR}/testfile ${D}${datadir}/something\n', 162 ' install -m 0755 ${WORKDIR}/%s ${D}${datadir}/scriptname\n' % testfile2name, 163 '}\n'] 164 self._try_recipetool_appendfile('netbase', '/usr/share/scriptname', testfile2, '-r netbase', expectedlines, ['testfile', testfile2name]) 165 166 def test_recipetool_appendfile_add_bindir(self): 167 # Try arbitrary file add to a recipe, this time to a location such that should be installed as executable 168 expectedlines = ['FILESEXTRAPATHS:prepend := "${THISDIR}/${PN}:"\n', 169 '\n', 170 'SRC_URI += "file://testfile"\n', 171 '\n', 172 'do_install:append() {\n', 173 ' install -d ${D}${bindir}\n', 174 ' install -m 0755 ${WORKDIR}/testfile ${D}${bindir}/selftest-recipetool-testbin\n', 175 '}\n'] 176 _, output = self._try_recipetool_appendfile('netbase', '/usr/bin/selftest-recipetool-testbin', self.testfile, '-r netbase', expectedlines, ['testfile']) 177 self.assertNotIn('WARNING: ', output) 178 179 def test_recipetool_appendfile_add_machine(self): 180 # Try arbitrary file add to a recipe, this time to a location such that should be installed as executable 181 expectedlines = ['FILESEXTRAPATHS:prepend := "${THISDIR}/${PN}:"\n', 182 '\n', 183 'PACKAGE_ARCH = "${MACHINE_ARCH}"\n', 184 '\n', 185 'SRC_URI:append:mymachine = " file://testfile"\n', 186 '\n', 187 'do_install:append:mymachine() {\n', 188 ' install -d ${D}${datadir}\n', 189 ' install -m 0644 ${WORKDIR}/testfile ${D}${datadir}/something\n', 190 '}\n'] 191 _, output = self._try_recipetool_appendfile('netbase', '/usr/share/something', self.testfile, '-r netbase -m mymachine', expectedlines, ['mymachine/testfile']) 192 self.assertNotIn('WARNING: ', output) 193 194 def test_recipetool_appendfile_orig(self): 195 # A file that's in SRC_URI and in do_install with the same name 196 expectedlines = ['FILESEXTRAPATHS:prepend := "${THISDIR}/${PN}:"\n', 197 '\n'] 198 _, output = self._try_recipetool_appendfile('selftest-recipetool-appendfile', '/usr/share/selftest-replaceme-orig', self.testfile, '', expectedlines, ['selftest-replaceme-orig']) 199 self.assertNotIn('WARNING: ', output) 200 201 def test_recipetool_appendfile_todir(self): 202 # A file that's in SRC_URI and in do_install with destination directory rather than file 203 expectedlines = ['FILESEXTRAPATHS:prepend := "${THISDIR}/${PN}:"\n', 204 '\n'] 205 _, output = self._try_recipetool_appendfile('selftest-recipetool-appendfile', '/usr/share/selftest-replaceme-todir', self.testfile, '', expectedlines, ['selftest-replaceme-todir']) 206 self.assertNotIn('WARNING: ', output) 207 208 def test_recipetool_appendfile_renamed(self): 209 # A file that's in SRC_URI with a different name to the destination file 210 expectedlines = ['FILESEXTRAPATHS:prepend := "${THISDIR}/${PN}:"\n', 211 '\n'] 212 _, output = self._try_recipetool_appendfile('selftest-recipetool-appendfile', '/usr/share/selftest-replaceme-renamed', self.testfile, '', expectedlines, ['file1']) 213 self.assertNotIn('WARNING: ', output) 214 215 def test_recipetool_appendfile_subdir(self): 216 # A file that's in SRC_URI in a subdir 217 expectedlines = ['FILESEXTRAPATHS:prepend := "${THISDIR}/${PN}:"\n', 218 '\n', 219 'SRC_URI += "file://testfile"\n', 220 '\n', 221 'do_install:append() {\n', 222 ' install -d ${D}${datadir}\n', 223 ' install -m 0644 ${WORKDIR}/testfile ${D}${datadir}/selftest-replaceme-subdir\n', 224 '}\n'] 225 _, output = self._try_recipetool_appendfile('selftest-recipetool-appendfile', '/usr/share/selftest-replaceme-subdir', self.testfile, '', expectedlines, ['testfile']) 226 self.assertNotIn('WARNING: ', output) 227 228 def test_recipetool_appendfile_inst_glob(self): 229 # A file that's in do_install as a glob 230 expectedlines = ['FILESEXTRAPATHS:prepend := "${THISDIR}/${PN}:"\n', 231 '\n'] 232 _, output = self._try_recipetool_appendfile('selftest-recipetool-appendfile', '/usr/share/selftest-replaceme-inst-globfile', self.testfile, '', expectedlines, ['selftest-replaceme-inst-globfile']) 233 self.assertNotIn('WARNING: ', output) 234 235 def test_recipetool_appendfile_inst_todir_glob(self): 236 # A file that's in do_install as a glob with destination as a directory 237 expectedlines = ['FILESEXTRAPATHS:prepend := "${THISDIR}/${PN}:"\n', 238 '\n'] 239 _, output = self._try_recipetool_appendfile('selftest-recipetool-appendfile', '/usr/share/selftest-replaceme-inst-todir-globfile', self.testfile, '', expectedlines, ['selftest-replaceme-inst-todir-globfile']) 240 self.assertNotIn('WARNING: ', output) 241 242 def test_recipetool_appendfile_patch(self): 243 # A file that's added by a patch in SRC_URI 244 expectedlines = ['FILESEXTRAPATHS:prepend := "${THISDIR}/${PN}:"\n', 245 '\n', 246 'SRC_URI += "file://testfile"\n', 247 '\n', 248 'do_install:append() {\n', 249 ' install -d ${D}${sysconfdir}\n', 250 ' install -m 0644 ${WORKDIR}/testfile ${D}${sysconfdir}/selftest-replaceme-patched\n', 251 '}\n'] 252 _, output = self._try_recipetool_appendfile('selftest-recipetool-appendfile', '/etc/selftest-replaceme-patched', self.testfile, '', expectedlines, ['testfile']) 253 for line in output.splitlines(): 254 if 'WARNING: ' in line: 255 self.assertIn('add-file.patch', line, 'Unexpected warning found in output:\n%s' % line) 256 break 257 else: 258 self.fail('Patch warning not found in output:\n%s' % output) 259 260 def test_recipetool_appendfile_script(self): 261 # Now, a file that's in SRC_URI but installed by a script (so no mention in do_install) 262 expectedlines = ['FILESEXTRAPATHS:prepend := "${THISDIR}/${PN}:"\n', 263 '\n', 264 'SRC_URI += "file://testfile"\n', 265 '\n', 266 'do_install:append() {\n', 267 ' install -d ${D}${datadir}\n', 268 ' install -m 0644 ${WORKDIR}/testfile ${D}${datadir}/selftest-replaceme-scripted\n', 269 '}\n'] 270 _, output = self._try_recipetool_appendfile('selftest-recipetool-appendfile', '/usr/share/selftest-replaceme-scripted', self.testfile, '', expectedlines, ['testfile']) 271 self.assertNotIn('WARNING: ', output) 272 273 def test_recipetool_appendfile_inst_func(self): 274 # A file that's installed from a function called by do_install 275 expectedlines = ['FILESEXTRAPATHS:prepend := "${THISDIR}/${PN}:"\n', 276 '\n'] 277 _, output = self._try_recipetool_appendfile('selftest-recipetool-appendfile', '/usr/share/selftest-replaceme-inst-func', self.testfile, '', expectedlines, ['selftest-replaceme-inst-func']) 278 self.assertNotIn('WARNING: ', output) 279 280 def test_recipetool_appendfile_postinstall(self): 281 # A file that's created by a postinstall script (and explicitly mentioned in it) 282 # First try without specifying recipe 283 self._try_recipetool_appendfile_fail('/usr/share/selftest-replaceme-postinst', self.testfile, ['File /usr/share/selftest-replaceme-postinst may be written out in a pre/postinstall script of the following recipes:', 'selftest-recipetool-appendfile']) 284 # Now specify recipe 285 expectedlines = ['FILESEXTRAPATHS:prepend := "${THISDIR}/${PN}:"\n', 286 '\n', 287 'SRC_URI += "file://testfile"\n', 288 '\n', 289 'do_install:append() {\n', 290 ' install -d ${D}${datadir}\n', 291 ' install -m 0644 ${WORKDIR}/testfile ${D}${datadir}/selftest-replaceme-postinst\n', 292 '}\n'] 293 _, output = self._try_recipetool_appendfile('selftest-recipetool-appendfile', '/usr/share/selftest-replaceme-postinst', self.testfile, '-r selftest-recipetool-appendfile', expectedlines, ['testfile']) 294 295 def test_recipetool_appendfile_extlayer(self): 296 # Try creating a bbappend in a layer that's not in bblayers.conf and has a different structure 297 exttemplayerdir = os.path.join(self.tempdir, 'extlayer') 298 self._create_temp_layer(exttemplayerdir, False, 'oeselftestextlayer', recipepathspec='metadata/recipes/recipes-*/*') 299 result = runCmd('recipetool appendfile %s /usr/share/selftest-replaceme-orig %s' % (exttemplayerdir, self.testfile)) 300 self.assertNotIn('Traceback', result.output) 301 createdfiles = [] 302 for root, _, files in os.walk(exttemplayerdir): 303 for f in files: 304 createdfiles.append(os.path.relpath(os.path.join(root, f), exttemplayerdir)) 305 createdfiles.remove('conf/layer.conf') 306 expectedfiles = ['metadata/recipes/recipes-test/selftest-recipetool-appendfile/selftest-recipetool-appendfile.bbappend', 307 'metadata/recipes/recipes-test/selftest-recipetool-appendfile/selftest-recipetool-appendfile/selftest-replaceme-orig'] 308 self.assertEqual(sorted(createdfiles), sorted(expectedfiles)) 309 310 def test_recipetool_appendfile_wildcard(self): 311 312 def try_appendfile_wc(options): 313 result = runCmd('recipetool appendfile %s /etc/profile %s %s' % (self.templayerdir, self.testfile, options)) 314 self.assertNotIn('Traceback', result.output) 315 bbappendfile = None 316 for root, _, files in os.walk(self.templayerdir): 317 for f in files: 318 if f.endswith('.bbappend'): 319 bbappendfile = f 320 break 321 if not bbappendfile: 322 self.fail('No bbappend file created') 323 runCmd('rm -rf %s/recipes-*' % self.templayerdir) 324 return bbappendfile 325 326 # Check without wildcard option 327 recipefn = os.path.basename(get_bb_var('FILE', 'base-files')) 328 filename = try_appendfile_wc('') 329 self.assertEqual(filename, recipefn.replace('.bb', '.bbappend')) 330 # Now check with wildcard option 331 filename = try_appendfile_wc('-w') 332 self.assertEqual(filename, recipefn.split('_')[0] + '_%.bbappend') 333 334 335class RecipetoolCreateTests(RecipetoolBase): 336 337 def test_recipetool_create(self): 338 # Try adding a recipe 339 tempsrc = os.path.join(self.tempdir, 'srctree') 340 os.makedirs(tempsrc) 341 recipefile = os.path.join(self.tempdir, 'logrotate_3.12.3.bb') 342 srcuri = 'https://github.com/logrotate/logrotate/releases/download/3.12.3/logrotate-3.12.3.tar.xz' 343 result = runCmd('recipetool create -o %s %s -x %s' % (recipefile, srcuri, tempsrc)) 344 self.assertTrue(os.path.isfile(recipefile)) 345 checkvars = {} 346 checkvars['LICENSE'] = 'GPL-2.0-only' 347 checkvars['LIC_FILES_CHKSUM'] = 'file://COPYING;md5=b234ee4d69f5fce4486a80fdaf4a4263' 348 checkvars['SRC_URI'] = 'https://github.com/logrotate/logrotate/releases/download/${PV}/logrotate-${PV}.tar.xz' 349 checkvars['SRC_URI[md5sum]'] = 'a560c57fac87c45b2fc17406cdf79288' 350 checkvars['SRC_URI[sha256sum]'] = '2e6a401cac9024db2288297e3be1a8ab60e7401ba8e91225218aaf4a27e82a07' 351 self._test_recipe_contents(recipefile, checkvars, []) 352 353 def test_recipetool_create_autotools(self): 354 if 'x11' not in get_bb_var('DISTRO_FEATURES'): 355 self.skipTest('Test requires x11 as distro feature') 356 # Ensure we have the right data in shlibs/pkgdata 357 bitbake('libpng pango libx11 libxext jpeg libcheck') 358 # Try adding a recipe 359 tempsrc = os.path.join(self.tempdir, 'srctree') 360 os.makedirs(tempsrc) 361 recipefile = os.path.join(self.tempdir, 'libmatchbox.bb') 362 srcuri = 'git://git.yoctoproject.org/libmatchbox' 363 result = runCmd(['recipetool', 'create', '-o', recipefile, srcuri + ";rev=9f7cf8895ae2d39c465c04cc78e918c157420269", '-x', tempsrc]) 364 self.assertTrue(os.path.isfile(recipefile), 'recipetool did not create recipe file; output:\n%s' % result.output) 365 checkvars = {} 366 checkvars['LICENSE'] = 'LGPL-2.1-only' 367 checkvars['LIC_FILES_CHKSUM'] = 'file://COPYING;md5=7fbc338309ac38fefcd64b04bb903e34' 368 checkvars['S'] = '${WORKDIR}/git' 369 checkvars['PV'] = '1.11+git${SRCPV}' 370 checkvars['SRC_URI'] = srcuri + ';branch=master' 371 checkvars['DEPENDS'] = set(['libcheck', 'libjpeg-turbo', 'libpng', 'libx11', 'libxext', 'pango']) 372 inherits = ['autotools', 'pkgconfig'] 373 self._test_recipe_contents(recipefile, checkvars, inherits) 374 375 def test_recipetool_create_simple(self): 376 # Try adding a recipe 377 temprecipe = os.path.join(self.tempdir, 'recipe') 378 os.makedirs(temprecipe) 379 pv = '1.7.4.1' 380 srcuri = 'http://www.dest-unreach.org/socat/download/Archive/socat-%s.tar.bz2' % pv 381 result = runCmd('recipetool create %s -o %s' % (srcuri, temprecipe)) 382 dirlist = os.listdir(temprecipe) 383 if len(dirlist) > 1: 384 self.fail('recipetool created more than just one file; output:\n%s\ndirlist:\n%s' % (result.output, str(dirlist))) 385 if len(dirlist) < 1 or not os.path.isfile(os.path.join(temprecipe, dirlist[0])): 386 self.fail('recipetool did not create recipe file; output:\n%s\ndirlist:\n%s' % (result.output, str(dirlist))) 387 self.assertEqual(dirlist[0], 'socat_%s.bb' % pv, 'Recipe file incorrectly named') 388 checkvars = {} 389 checkvars['LICENSE'] = set(['Unknown', 'GPL-2.0-only']) 390 checkvars['LIC_FILES_CHKSUM'] = set(['file://COPYING.OpenSSL;md5=5c9bccc77f67a8328ef4ebaf468116f4', 'file://COPYING;md5=b234ee4d69f5fce4486a80fdaf4a4263']) 391 # We don't check DEPENDS since they are variable for this recipe depending on what's in the sysroot 392 checkvars['S'] = None 393 checkvars['SRC_URI'] = srcuri.replace(pv, '${PV}') 394 inherits = ['autotools'] 395 self._test_recipe_contents(os.path.join(temprecipe, dirlist[0]), checkvars, inherits) 396 397 def test_recipetool_create_cmake(self): 398 temprecipe = os.path.join(self.tempdir, 'recipe') 399 os.makedirs(temprecipe) 400 recipefile = os.path.join(temprecipe, 'taglib_1.11.1.bb') 401 srcuri = 'http://taglib.github.io/releases/taglib-1.11.1.tar.gz' 402 result = runCmd('recipetool create -o %s %s' % (temprecipe, srcuri)) 403 self.assertTrue(os.path.isfile(recipefile)) 404 checkvars = {} 405 checkvars['LICENSE'] = set(['LGPL-2.1-only', 'MPL-1.1-only']) 406 checkvars['SRC_URI'] = 'http://taglib.github.io/releases/taglib-${PV}.tar.gz' 407 checkvars['SRC_URI[md5sum]'] = 'cee7be0ccfc892fa433d6c837df9522a' 408 checkvars['SRC_URI[sha256sum]'] = 'b6d1a5a610aae6ff39d93de5efd0fdc787aa9e9dc1e7026fa4c961b26563526b' 409 checkvars['DEPENDS'] = set(['boost', 'zlib']) 410 inherits = ['cmake'] 411 self._test_recipe_contents(recipefile, checkvars, inherits) 412 413 def test_recipetool_create_npm(self): 414 collections = get_bb_var('BBFILE_COLLECTIONS').split() 415 if "openembedded-layer" not in collections: 416 self.skipTest("Test needs meta-oe for nodejs") 417 418 temprecipe = os.path.join(self.tempdir, 'recipe') 419 os.makedirs(temprecipe) 420 recipefile = os.path.join(temprecipe, 'savoirfairelinux-node-server-example_1.0.0.bb') 421 shrinkwrap = os.path.join(temprecipe, 'savoirfairelinux-node-server-example', 'npm-shrinkwrap.json') 422 srcuri = 'npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example;version=1.0.0' 423 result = runCmd('recipetool create -o %s \'%s\'' % (temprecipe, srcuri)) 424 self.assertTrue(os.path.isfile(recipefile)) 425 self.assertTrue(os.path.isfile(shrinkwrap)) 426 checkvars = {} 427 checkvars['SUMMARY'] = 'Node Server Example' 428 checkvars['HOMEPAGE'] = 'https://github.com/savoirfairelinux/node-server-example#readme' 429 checkvars['LICENSE'] = 'BSD-3-Clause & ISC & MIT & Unknown' 430 urls = [] 431 urls.append('npm://registry.npmjs.org/;package=@savoirfairelinux/node-server-example;version=${PV}') 432 urls.append('npmsw://${THISDIR}/${BPN}/npm-shrinkwrap.json') 433 checkvars['SRC_URI'] = set(urls) 434 checkvars['S'] = '${WORKDIR}/npm' 435 checkvars['LICENSE:${PN}'] = 'MIT' 436 checkvars['LICENSE:${PN}-base64'] = 'Unknown' 437 checkvars['LICENSE:${PN}-accepts'] = 'MIT' 438 checkvars['LICENSE:${PN}-inherits'] = 'ISC' 439 inherits = ['npm'] 440 self._test_recipe_contents(recipefile, checkvars, inherits) 441 442 def test_recipetool_create_github(self): 443 # Basic test to see if github URL mangling works 444 temprecipe = os.path.join(self.tempdir, 'recipe') 445 os.makedirs(temprecipe) 446 recipefile = os.path.join(temprecipe, 'meson_git.bb') 447 srcuri = 'https://github.com/mesonbuild/meson;rev=0.32.0' 448 result = runCmd(['recipetool', 'create', '-o', temprecipe, srcuri]) 449 self.assertTrue(os.path.isfile(recipefile)) 450 checkvars = {} 451 checkvars['LICENSE'] = set(['Apache-2.0']) 452 checkvars['SRC_URI'] = 'git://github.com/mesonbuild/meson;protocol=https;branch=master' 453 inherits = ['setuptools3'] 454 self._test_recipe_contents(recipefile, checkvars, inherits) 455 456 def test_recipetool_create_python3_setuptools(self): 457 # Test creating python3 package from tarball (using setuptools3 class) 458 temprecipe = os.path.join(self.tempdir, 'recipe') 459 os.makedirs(temprecipe) 460 pn = 'python-magic' 461 pv = '0.4.15' 462 recipefile = os.path.join(temprecipe, '%s_%s.bb' % (pn, pv)) 463 srcuri = 'https://files.pythonhosted.org/packages/84/30/80932401906eaf787f2e9bd86dc458f1d2e75b064b4c187341f29516945c/python-magic-%s.tar.gz' % pv 464 result = runCmd('recipetool create -o %s %s' % (temprecipe, srcuri)) 465 self.assertTrue(os.path.isfile(recipefile)) 466 checkvars = {} 467 checkvars['LICENSE'] = set(['MIT']) 468 checkvars['LIC_FILES_CHKSUM'] = 'file://LICENSE;md5=16a934f165e8c3245f241e77d401bb88' 469 checkvars['SRC_URI'] = 'https://files.pythonhosted.org/packages/84/30/80932401906eaf787f2e9bd86dc458f1d2e75b064b4c187341f29516945c/python-magic-${PV}.tar.gz' 470 checkvars['SRC_URI[md5sum]'] = 'e384c95a47218f66c6501cd6dd45ff59' 471 checkvars['SRC_URI[sha256sum]'] = 'f3765c0f582d2dfc72c15f3b5a82aecfae9498bd29ca840d72f37d7bd38bfcd5' 472 inherits = ['setuptools3'] 473 self._test_recipe_contents(recipefile, checkvars, inherits) 474 475 def test_recipetool_create_github_tarball(self): 476 # Basic test to ensure github URL mangling doesn't apply to release tarballs 477 temprecipe = os.path.join(self.tempdir, 'recipe') 478 os.makedirs(temprecipe) 479 pv = '0.32.0' 480 recipefile = os.path.join(temprecipe, 'meson_%s.bb' % pv) 481 srcuri = 'https://github.com/mesonbuild/meson/releases/download/%s/meson-%s.tar.gz' % (pv, pv) 482 result = runCmd('recipetool create -o %s %s' % (temprecipe, srcuri)) 483 self.assertTrue(os.path.isfile(recipefile)) 484 checkvars = {} 485 checkvars['LICENSE'] = set(['Apache-2.0']) 486 checkvars['SRC_URI'] = 'https://github.com/mesonbuild/meson/releases/download/${PV}/meson-${PV}.tar.gz' 487 inherits = ['setuptools3'] 488 self._test_recipe_contents(recipefile, checkvars, inherits) 489 490 def _test_recipetool_create_git(self, srcuri, branch=None): 491 # Basic test to check http git URL mangling works 492 temprecipe = os.path.join(self.tempdir, 'recipe') 493 os.makedirs(temprecipe) 494 name = srcuri.split(';')[0].split('/')[-1] 495 recipefile = os.path.join(temprecipe, name + '_git.bb') 496 options = ' -B %s' % branch if branch else '' 497 result = runCmd('recipetool create -o %s%s "%s"' % (temprecipe, options, srcuri)) 498 self.assertTrue(os.path.isfile(recipefile)) 499 checkvars = {} 500 checkvars['SRC_URI'] = srcuri 501 for scheme in ['http', 'https']: 502 if srcuri.startswith(scheme + ":"): 503 checkvars['SRC_URI'] = 'git%s;protocol=%s' % (srcuri[len(scheme):], scheme) 504 if ';branch=' not in srcuri: 505 checkvars['SRC_URI'] += ';branch=' + (branch or 'master') 506 self._test_recipe_contents(recipefile, checkvars, []) 507 508 def test_recipetool_create_git_http(self): 509 self._test_recipetool_create_git('http://git.yoctoproject.org/git/matchbox-keyboard') 510 511 def test_recipetool_create_git_srcuri_master(self): 512 self._test_recipetool_create_git('git://git.yoctoproject.org/matchbox-keyboard;branch=master') 513 514 def test_recipetool_create_git_srcuri_branch(self): 515 self._test_recipetool_create_git('git://git.yoctoproject.org/matchbox-keyboard;branch=matchbox-keyboard-0-1') 516 517 def test_recipetool_create_git_srcbranch(self): 518 self._test_recipetool_create_git('git://git.yoctoproject.org/matchbox-keyboard', 'matchbox-keyboard-0-1') 519 520 521class RecipetoolTests(RecipetoolBase): 522 523 @classmethod 524 def setUpClass(cls): 525 import sys 526 527 super(RecipetoolTests, cls).setUpClass() 528 bb_vars = get_bb_vars(['BBPATH']) 529 cls.bbpath = bb_vars['BBPATH'] 530 libpath = os.path.join(get_bb_var('COREBASE'), 'scripts', 'lib', 'recipetool') 531 sys.path.insert(0, libpath) 532 533 def _copy_file_with_cleanup(self, srcfile, basedstdir, *paths): 534 dstdir = basedstdir 535 self.assertTrue(os.path.exists(dstdir)) 536 for p in paths: 537 dstdir = os.path.join(dstdir, p) 538 if not os.path.exists(dstdir): 539 os.makedirs(dstdir) 540 if p == "lib": 541 # Can race with other tests 542 self.add_command_to_tearDown('rmdir --ignore-fail-on-non-empty %s' % dstdir) 543 else: 544 self.track_for_cleanup(dstdir) 545 dstfile = os.path.join(dstdir, os.path.basename(srcfile)) 546 if srcfile != dstfile: 547 shutil.copy(srcfile, dstfile) 548 self.track_for_cleanup(dstfile) 549 550 def test_recipetool_load_plugin(self): 551 """Test that recipetool loads only the first found plugin in BBPATH.""" 552 553 recipetool = runCmd("which recipetool") 554 fromname = runCmd("recipetool --quiet pluginfile") 555 srcfile = fromname.output 556 searchpath = self.bbpath.split(':') + [os.path.dirname(recipetool.output)] 557 plugincontent = [] 558 with open(srcfile) as fh: 559 plugincontent = fh.readlines() 560 try: 561 self.assertIn('meta-selftest', srcfile, 'wrong bbpath plugin found') 562 for path in searchpath: 563 self._copy_file_with_cleanup(srcfile, path, 'lib', 'recipetool') 564 result = runCmd("recipetool --quiet count") 565 self.assertEqual(result.output, '1') 566 result = runCmd("recipetool --quiet multiloaded") 567 self.assertEqual(result.output, "no") 568 for path in searchpath: 569 result = runCmd("recipetool --quiet bbdir") 570 self.assertEqual(result.output, path) 571 os.unlink(os.path.join(result.output, 'lib', 'recipetool', 'bbpath.py')) 572 finally: 573 with open(srcfile, 'w') as fh: 574 fh.writelines(plugincontent) 575 576 def test_recipetool_handle_license_vars(self): 577 from create import handle_license_vars 578 from unittest.mock import Mock 579 580 commonlicdir = get_bb_var('COMMON_LICENSE_DIR') 581 582 class DataConnectorCopy(bb.tinfoil.TinfoilDataStoreConnector): 583 pass 584 585 d = DataConnectorCopy 586 d.getVar = Mock(return_value=commonlicdir) 587 588 srctree = tempfile.mkdtemp(prefix='recipetoolqa') 589 self.track_for_cleanup(srctree) 590 591 # Multiple licenses 592 licenses = ['MIT', 'ISC', 'BSD-3-Clause', 'Apache-2.0'] 593 for licence in licenses: 594 shutil.copy(os.path.join(commonlicdir, licence), os.path.join(srctree, 'LICENSE.' + licence)) 595 # Duplicate license 596 shutil.copy(os.path.join(commonlicdir, 'MIT'), os.path.join(srctree, 'LICENSE')) 597 598 extravalues = { 599 # Duplicate and missing licenses 600 'LICENSE': 'Zlib & BSD-2-Clause & Zlib', 601 'LIC_FILES_CHKSUM': [ 602 'file://README.md;md5=0123456789abcdef0123456789abcd' 603 ] 604 } 605 lines_before = [] 606 handled = [] 607 licvalues = handle_license_vars(srctree, lines_before, handled, extravalues, d) 608 expected_lines_before = [ 609 '# WARNING: the following LICENSE and LIC_FILES_CHKSUM values are best guesses - it is', 610 '# your responsibility to verify that the values are complete and correct.', 611 '# NOTE: Original package / source metadata indicates license is: BSD-2-Clause & Zlib', 612 '#', 613 '# NOTE: multiple licenses have been detected; they have been separated with &', 614 '# in the LICENSE value for now since it is a reasonable assumption that all', 615 '# of the licenses apply. If instead there is a choice between the multiple', 616 '# licenses then you should change the value to separate the licenses with |', 617 '# instead of &. If there is any doubt, check the accompanying documentation', 618 '# to determine which situation is applicable.', 619 'LICENSE = "Apache-2.0 & BSD-2-Clause & BSD-3-Clause & ISC & MIT & Zlib"', 620 'LIC_FILES_CHKSUM = "file://LICENSE;md5=0835ade698e0bcf8506ecda2f7b4f302 \\\n' 621 ' file://LICENSE.Apache-2.0;md5=89aea4e17d99a7cacdbeed46a0096b10 \\\n' 622 ' file://LICENSE.BSD-3-Clause;md5=550794465ba0ec5312d6919e203a55f9 \\\n' 623 ' file://LICENSE.ISC;md5=f3b90e78ea0cffb20bf5cca7947a896d \\\n' 624 ' file://LICENSE.MIT;md5=0835ade698e0bcf8506ecda2f7b4f302 \\\n' 625 ' file://README.md;md5=0123456789abcdef0123456789abcd"', 626 '' 627 ] 628 self.assertEqual(lines_before, expected_lines_before) 629 expected_licvalues = [ 630 ('MIT', 'LICENSE', '0835ade698e0bcf8506ecda2f7b4f302'), 631 ('Apache-2.0', 'LICENSE.Apache-2.0', '89aea4e17d99a7cacdbeed46a0096b10'), 632 ('BSD-3-Clause', 'LICENSE.BSD-3-Clause', '550794465ba0ec5312d6919e203a55f9'), 633 ('ISC', 'LICENSE.ISC', 'f3b90e78ea0cffb20bf5cca7947a896d'), 634 ('MIT', 'LICENSE.MIT', '0835ade698e0bcf8506ecda2f7b4f302') 635 ] 636 self.assertEqual(handled, [('license', expected_licvalues)]) 637 self.assertEqual(extravalues, {}) 638 self.assertEqual(licvalues, expected_licvalues) 639 640 641 def test_recipetool_split_pkg_licenses(self): 642 from create import split_pkg_licenses 643 licvalues = [ 644 # Duplicate licenses 645 ('BSD-2-Clause', 'x/COPYING', None), 646 ('BSD-2-Clause', 'x/LICENSE', None), 647 # Multiple licenses 648 ('MIT', 'x/a/LICENSE.MIT', None), 649 ('ISC', 'x/a/LICENSE.ISC', None), 650 # Alternative licenses 651 ('(MIT | ISC)', 'x/b/LICENSE', None), 652 # Alternative licenses without brackets 653 ('MIT | BSD-2-Clause', 'x/c/LICENSE', None), 654 # Multi licenses with alternatives 655 ('MIT', 'x/d/COPYING', None), 656 ('MIT | BSD-2-Clause', 'x/d/LICENSE', None), 657 # Multi licenses with alternatives and brackets 658 ('Apache-2.0 & ((MIT | ISC) & BSD-3-Clause)', 'x/e/LICENSE', None) 659 ] 660 packages = { 661 '${PN}': '', 662 'a': 'x/a', 663 'b': 'x/b', 664 'c': 'x/c', 665 'd': 'x/d', 666 'e': 'x/e', 667 'f': 'x/f', 668 'g': 'x/g', 669 } 670 fallback_licenses = { 671 # Ignored 672 'a': 'BSD-3-Clause', 673 # Used 674 'f': 'BSD-3-Clause' 675 } 676 outlines = [] 677 outlicenses = split_pkg_licenses(licvalues, packages, outlines, fallback_licenses) 678 expected_outlicenses = { 679 '${PN}': ['BSD-2-Clause'], 680 'a': ['ISC', 'MIT'], 681 'b': ['(ISC | MIT)'], 682 'c': ['(BSD-2-Clause | MIT)'], 683 'd': ['(BSD-2-Clause | MIT)', 'MIT'], 684 'e': ['(ISC | MIT)', 'Apache-2.0', 'BSD-3-Clause'], 685 'f': ['BSD-3-Clause'], 686 'g': ['Unknown'] 687 } 688 self.assertEqual(outlicenses, expected_outlicenses) 689 expected_outlines = [ 690 'LICENSE:${PN} = "BSD-2-Clause"', 691 'LICENSE:a = "ISC & MIT"', 692 'LICENSE:b = "(ISC | MIT)"', 693 'LICENSE:c = "(BSD-2-Clause | MIT)"', 694 'LICENSE:d = "(BSD-2-Clause | MIT) & MIT"', 695 'LICENSE:e = "(ISC | MIT) & Apache-2.0 & BSD-3-Clause"', 696 'LICENSE:f = "BSD-3-Clause"', 697 'LICENSE:g = "Unknown"' 698 ] 699 self.assertEqual(outlines, expected_outlines) 700 701 702class RecipetoolAppendsrcBase(RecipetoolBase): 703 def _try_recipetool_appendsrcfile(self, testrecipe, newfile, destfile, options, expectedlines, expectedfiles): 704 cmd = 'recipetool appendsrcfile %s %s %s %s %s' % (options, self.templayerdir, testrecipe, newfile, destfile) 705 return self._try_recipetool_appendcmd(cmd, testrecipe, expectedfiles, expectedlines) 706 707 def _try_recipetool_appendsrcfiles(self, testrecipe, newfiles, expectedlines=None, expectedfiles=None, destdir=None, options=''): 708 709 if destdir: 710 options += ' -D %s' % destdir 711 712 if expectedfiles is None: 713 expectedfiles = [os.path.basename(f) for f in newfiles] 714 715 cmd = 'recipetool appendsrcfiles %s %s %s %s' % (options, self.templayerdir, testrecipe, ' '.join(newfiles)) 716 return self._try_recipetool_appendcmd(cmd, testrecipe, expectedfiles, expectedlines) 717 718 def _try_recipetool_appendsrcfile_fail(self, testrecipe, newfile, destfile, checkerror): 719 cmd = 'recipetool appendsrcfile %s %s %s %s' % (self.templayerdir, testrecipe, newfile, destfile or '') 720 result = runCmd(cmd, ignore_status=True) 721 self.assertNotEqual(result.status, 0, 'Command "%s" should have failed but didn\'t' % cmd) 722 self.assertNotIn('Traceback', result.output) 723 for errorstr in checkerror: 724 self.assertIn(errorstr, result.output) 725 726 @staticmethod 727 def _get_first_file_uri(recipe): 728 '''Return the first file:// in SRC_URI for the specified recipe.''' 729 src_uri = get_bb_var('SRC_URI', recipe).split() 730 for uri in src_uri: 731 p = urllib.parse.urlparse(uri) 732 if p.scheme == 'file': 733 return p.netloc + p.path 734 735 def _test_appendsrcfile(self, testrecipe, filename=None, destdir=None, has_src_uri=True, srcdir=None, newfile=None, options=''): 736 if newfile is None: 737 newfile = self.testfile 738 739 if srcdir: 740 if destdir: 741 expected_subdir = os.path.join(srcdir, destdir) 742 else: 743 expected_subdir = srcdir 744 else: 745 options += " -W" 746 expected_subdir = destdir 747 748 if filename: 749 if destdir: 750 destpath = os.path.join(destdir, filename) 751 else: 752 destpath = filename 753 else: 754 filename = os.path.basename(newfile) 755 if destdir: 756 destpath = destdir + os.sep 757 else: 758 destpath = '.' + os.sep 759 760 expectedlines = ['FILESEXTRAPATHS:prepend := "${THISDIR}/${PN}:"\n', 761 '\n'] 762 if has_src_uri: 763 uri = 'file://%s' % filename 764 if expected_subdir: 765 uri += ';subdir=%s' % expected_subdir 766 expectedlines[0:0] = ['SRC_URI += "%s"\n' % uri, 767 '\n'] 768 769 return self._try_recipetool_appendsrcfile(testrecipe, newfile, destpath, options, expectedlines, [filename]) 770 771 def _test_appendsrcfiles(self, testrecipe, newfiles, expectedfiles=None, destdir=None, options=''): 772 if expectedfiles is None: 773 expectedfiles = [os.path.basename(n) for n in newfiles] 774 775 self._try_recipetool_appendsrcfiles(testrecipe, newfiles, expectedfiles=expectedfiles, destdir=destdir, options=options) 776 777 bb_vars = get_bb_vars(['SRC_URI', 'FILE', 'FILESEXTRAPATHS'], testrecipe) 778 src_uri = bb_vars['SRC_URI'].split() 779 for f in expectedfiles: 780 if destdir: 781 self.assertIn('file://%s;subdir=%s' % (f, destdir), src_uri) 782 else: 783 self.assertIn('file://%s' % f, src_uri) 784 785 recipefile = bb_vars['FILE'] 786 bbappendfile = self._check_bbappend(testrecipe, recipefile, self.templayerdir) 787 filesdir = os.path.join(os.path.dirname(bbappendfile), testrecipe) 788 filesextrapaths = bb_vars['FILESEXTRAPATHS'].split(':') 789 self.assertIn(filesdir, filesextrapaths) 790 791 792 793 794class RecipetoolAppendsrcTests(RecipetoolAppendsrcBase): 795 796 def test_recipetool_appendsrcfile_basic(self): 797 self._test_appendsrcfile('base-files', 'a-file') 798 799 def test_recipetool_appendsrcfile_basic_wildcard(self): 800 testrecipe = 'base-files' 801 self._test_appendsrcfile(testrecipe, 'a-file', options='-w') 802 recipefile = get_bb_var('FILE', testrecipe) 803 bbappendfile = self._check_bbappend(testrecipe, recipefile, self.templayerdir) 804 self.assertEqual(os.path.basename(bbappendfile), '%s_%%.bbappend' % testrecipe) 805 806 def test_recipetool_appendsrcfile_subdir_basic(self): 807 self._test_appendsrcfile('base-files', 'a-file', 'tmp') 808 809 def test_recipetool_appendsrcfile_subdir_basic_dirdest(self): 810 self._test_appendsrcfile('base-files', destdir='tmp') 811 812 def test_recipetool_appendsrcfile_srcdir_basic(self): 813 testrecipe = 'bash' 814 bb_vars = get_bb_vars(['S', 'WORKDIR'], testrecipe) 815 srcdir = bb_vars['S'] 816 workdir = bb_vars['WORKDIR'] 817 subdir = os.path.relpath(srcdir, workdir) 818 self._test_appendsrcfile(testrecipe, 'a-file', srcdir=subdir) 819 820 def test_recipetool_appendsrcfile_existing_in_src_uri(self): 821 testrecipe = 'base-files' 822 filepath = self._get_first_file_uri(testrecipe) 823 self.assertTrue(filepath, 'Unable to test, no file:// uri found in SRC_URI for %s' % testrecipe) 824 self._test_appendsrcfile(testrecipe, filepath, has_src_uri=False) 825 826 def test_recipetool_appendsrcfile_existing_in_src_uri_diff_params(self): 827 testrecipe = 'base-files' 828 subdir = 'tmp' 829 filepath = self._get_first_file_uri(testrecipe) 830 self.assertTrue(filepath, 'Unable to test, no file:// uri found in SRC_URI for %s' % testrecipe) 831 832 output = self._test_appendsrcfile(testrecipe, filepath, subdir, has_src_uri=False) 833 self.assertTrue(any('with different parameters' in l for l in output)) 834 835 def test_recipetool_appendsrcfile_replace_file_srcdir(self): 836 testrecipe = 'bash' 837 filepath = 'Makefile.in' 838 bb_vars = get_bb_vars(['S', 'WORKDIR'], testrecipe) 839 srcdir = bb_vars['S'] 840 workdir = bb_vars['WORKDIR'] 841 subdir = os.path.relpath(srcdir, workdir) 842 843 self._test_appendsrcfile(testrecipe, filepath, srcdir=subdir) 844 bitbake('%s:do_unpack' % testrecipe) 845 with open(self.testfile, 'r') as testfile: 846 with open(os.path.join(srcdir, filepath), 'r') as makefilein: 847 self.assertEqual(testfile.read(), makefilein.read()) 848 849 def test_recipetool_appendsrcfiles_basic(self, destdir=None): 850 newfiles = [self.testfile] 851 for i in range(1, 5): 852 testfile = os.path.join(self.tempdir, 'testfile%d' % i) 853 with open(testfile, 'w') as f: 854 f.write('Test file %d\n' % i) 855 newfiles.append(testfile) 856 self._test_appendsrcfiles('gcc', newfiles, destdir=destdir, options='-W') 857 858 def test_recipetool_appendsrcfiles_basic_subdir(self): 859 self.test_recipetool_appendsrcfiles_basic(destdir='testdir') 860