1# 2# BitBake Tests for the Fetcher (fetch2/) 3# 4# Copyright (C) 2012 Richard Purdie 5# 6# SPDX-License-Identifier: GPL-2.0-only 7# 8 9import unittest 10import hashlib 11import tempfile 12import collections 13import os 14import tarfile 15from bb.fetch2 import URI 16from bb.fetch2 import FetchMethod 17import bb 18from bb.tests.support.httpserver import HTTPService 19 20def skipIfNoNetwork(): 21 if os.environ.get("BB_SKIP_NETTESTS") == "yes": 22 return unittest.skip("network test") 23 return lambda f: f 24 25class URITest(unittest.TestCase): 26 test_uris = { 27 "http://www.google.com/index.html" : { 28 'uri': 'http://www.google.com/index.html', 29 'scheme': 'http', 30 'hostname': 'www.google.com', 31 'port': None, 32 'hostport': 'www.google.com', 33 'path': '/index.html', 34 'userinfo': '', 35 'username': '', 36 'password': '', 37 'params': {}, 38 'query': {}, 39 'relative': False 40 }, 41 "http://www.google.com/index.html;param1=value1" : { 42 'uri': 'http://www.google.com/index.html;param1=value1', 43 'scheme': 'http', 44 'hostname': 'www.google.com', 45 'port': None, 46 'hostport': 'www.google.com', 47 'path': '/index.html', 48 'userinfo': '', 49 'username': '', 50 'password': '', 51 'params': { 52 'param1': 'value1' 53 }, 54 'query': {}, 55 'relative': False 56 }, 57 "http://www.example.org/index.html?param1=value1" : { 58 'uri': 'http://www.example.org/index.html?param1=value1', 59 'scheme': 'http', 60 'hostname': 'www.example.org', 61 'port': None, 62 'hostport': 'www.example.org', 63 'path': '/index.html', 64 'userinfo': '', 65 'username': '', 66 'password': '', 67 'params': {}, 68 'query': { 69 'param1': 'value1' 70 }, 71 'relative': False 72 }, 73 "http://www.example.org/index.html?qparam1=qvalue1;param2=value2" : { 74 'uri': 'http://www.example.org/index.html?qparam1=qvalue1;param2=value2', 75 'scheme': 'http', 76 'hostname': 'www.example.org', 77 'port': None, 78 'hostport': 'www.example.org', 79 'path': '/index.html', 80 'userinfo': '', 81 'username': '', 82 'password': '', 83 'params': { 84 'param2': 'value2' 85 }, 86 'query': { 87 'qparam1': 'qvalue1' 88 }, 89 'relative': False 90 }, 91 # Check that trailing semicolons are handled correctly 92 "http://www.example.org/index.html?qparam1=qvalue1;param2=value2;" : { 93 'uri': 'http://www.example.org/index.html?qparam1=qvalue1;param2=value2', 94 'scheme': 'http', 95 'hostname': 'www.example.org', 96 'port': None, 97 'hostport': 'www.example.org', 98 'path': '/index.html', 99 'userinfo': '', 100 'username': '', 101 'password': '', 102 'params': { 103 'param2': 'value2' 104 }, 105 'query': { 106 'qparam1': 'qvalue1' 107 }, 108 'relative': False 109 }, 110 "http://www.example.com:8080/index.html" : { 111 'uri': 'http://www.example.com:8080/index.html', 112 'scheme': 'http', 113 'hostname': 'www.example.com', 114 'port': 8080, 115 'hostport': 'www.example.com:8080', 116 'path': '/index.html', 117 'userinfo': '', 118 'username': '', 119 'password': '', 120 'params': {}, 121 'query': {}, 122 'relative': False 123 }, 124 "cvs://anoncvs@cvs.handhelds.org/cvs;module=familiar/dist/ipkg" : { 125 'uri': 'cvs://anoncvs@cvs.handhelds.org/cvs;module=familiar/dist/ipkg', 126 'scheme': 'cvs', 127 'hostname': 'cvs.handhelds.org', 128 'port': None, 129 'hostport': 'cvs.handhelds.org', 130 'path': '/cvs', 131 'userinfo': 'anoncvs', 132 'username': 'anoncvs', 133 'password': '', 134 'params': { 135 'module': 'familiar/dist/ipkg' 136 }, 137 'query': {}, 138 'relative': False 139 }, 140 "cvs://anoncvs:anonymous@cvs.handhelds.org/cvs;tag=V0-99-81;module=familiar/dist/ipkg": { 141 'uri': 'cvs://anoncvs:anonymous@cvs.handhelds.org/cvs;tag=V0-99-81;module=familiar/dist/ipkg', 142 'scheme': 'cvs', 143 'hostname': 'cvs.handhelds.org', 144 'port': None, 145 'hostport': 'cvs.handhelds.org', 146 'path': '/cvs', 147 'userinfo': 'anoncvs:anonymous', 148 'username': 'anoncvs', 149 'password': 'anonymous', 150 'params': collections.OrderedDict([ 151 ('tag', 'V0-99-81'), 152 ('module', 'familiar/dist/ipkg') 153 ]), 154 'query': {}, 155 'relative': False 156 }, 157 "file://example.diff": { # NOTE: Not RFC compliant! 158 'uri': 'file:example.diff', 159 'scheme': 'file', 160 'hostname': '', 161 'port': None, 162 'hostport': '', 163 'path': 'example.diff', 164 'userinfo': '', 165 'username': '', 166 'password': '', 167 'params': {}, 168 'query': {}, 169 'relative': True 170 }, 171 "file:example.diff": { # NOTE: RFC compliant version of the former 172 'uri': 'file:example.diff', 173 'scheme': 'file', 174 'hostname': '', 175 'port': None, 176 'hostport': '', 177 'path': 'example.diff', 178 'userinfo': '', 179 'userinfo': '', 180 'username': '', 181 'password': '', 182 'params': {}, 183 'query': {}, 184 'relative': True 185 }, 186 "file:///tmp/example.diff": { 187 'uri': 'file:///tmp/example.diff', 188 'scheme': 'file', 189 'hostname': '', 190 'port': None, 191 'hostport': '', 192 'path': '/tmp/example.diff', 193 'userinfo': '', 194 'userinfo': '', 195 'username': '', 196 'password': '', 197 'params': {}, 198 'query': {}, 199 'relative': False 200 }, 201 "git:///path/example.git": { 202 'uri': 'git:///path/example.git', 203 'scheme': 'git', 204 'hostname': '', 205 'port': None, 206 'hostport': '', 207 'path': '/path/example.git', 208 'userinfo': '', 209 'userinfo': '', 210 'username': '', 211 'password': '', 212 'params': {}, 213 'query': {}, 214 'relative': False 215 }, 216 "git:path/example.git": { 217 'uri': 'git:path/example.git', 218 'scheme': 'git', 219 'hostname': '', 220 'port': None, 221 'hostport': '', 222 'path': 'path/example.git', 223 'userinfo': '', 224 'userinfo': '', 225 'username': '', 226 'password': '', 227 'params': {}, 228 'query': {}, 229 'relative': True 230 }, 231 "git://example.net/path/example.git": { 232 'uri': 'git://example.net/path/example.git', 233 'scheme': 'git', 234 'hostname': 'example.net', 235 'port': None, 236 'hostport': 'example.net', 237 'path': '/path/example.git', 238 'userinfo': '', 239 'userinfo': '', 240 'username': '', 241 'password': '', 242 'params': {}, 243 'query': {}, 244 'relative': False 245 }, 246 "git://tfs-example.org:22/tfs/example%20path/example.git": { 247 'uri': 'git://tfs-example.org:22/tfs/example%20path/example.git', 248 'scheme': 'git', 249 'hostname': 'tfs-example.org', 250 'port': 22, 251 'hostport': 'tfs-example.org:22', 252 'path': '/tfs/example path/example.git', 253 'userinfo': '', 254 'userinfo': '', 255 'username': '', 256 'password': '', 257 'params': {}, 258 'query': {}, 259 'relative': False 260 }, 261 "http://somesite.net;someparam=1": { 262 'uri': 'http://somesite.net;someparam=1', 263 'scheme': 'http', 264 'hostname': 'somesite.net', 265 'port': None, 266 'hostport': 'somesite.net', 267 'path': '', 268 'userinfo': '', 269 'userinfo': '', 270 'username': '', 271 'password': '', 272 'params': {"someparam" : "1"}, 273 'query': {}, 274 'relative': False 275 }, 276 "file://somelocation;someparam=1": { 277 'uri': 'file:somelocation;someparam=1', 278 'scheme': 'file', 279 'hostname': '', 280 'port': None, 281 'hostport': '', 282 'path': 'somelocation', 283 'userinfo': '', 284 'userinfo': '', 285 'username': '', 286 'password': '', 287 'params': {"someparam" : "1"}, 288 'query': {}, 289 'relative': True 290 } 291 292 } 293 294 def test_uri(self): 295 for test_uri, ref in self.test_uris.items(): 296 uri = URI(test_uri) 297 298 self.assertEqual(str(uri), ref['uri']) 299 300 # expected attributes 301 self.assertEqual(uri.scheme, ref['scheme']) 302 303 self.assertEqual(uri.userinfo, ref['userinfo']) 304 self.assertEqual(uri.username, ref['username']) 305 self.assertEqual(uri.password, ref['password']) 306 307 self.assertEqual(uri.hostname, ref['hostname']) 308 self.assertEqual(uri.port, ref['port']) 309 self.assertEqual(uri.hostport, ref['hostport']) 310 311 self.assertEqual(uri.path, ref['path']) 312 self.assertEqual(uri.params, ref['params']) 313 314 self.assertEqual(uri.relative, ref['relative']) 315 316 def test_dict(self): 317 for test in self.test_uris.values(): 318 uri = URI() 319 320 self.assertEqual(uri.scheme, '') 321 self.assertEqual(uri.userinfo, '') 322 self.assertEqual(uri.username, '') 323 self.assertEqual(uri.password, '') 324 self.assertEqual(uri.hostname, '') 325 self.assertEqual(uri.port, None) 326 self.assertEqual(uri.path, '') 327 self.assertEqual(uri.params, {}) 328 329 330 uri.scheme = test['scheme'] 331 self.assertEqual(uri.scheme, test['scheme']) 332 333 uri.userinfo = test['userinfo'] 334 self.assertEqual(uri.userinfo, test['userinfo']) 335 self.assertEqual(uri.username, test['username']) 336 self.assertEqual(uri.password, test['password']) 337 338 # make sure changing the values doesn't do anything unexpected 339 uri.username = 'changeme' 340 self.assertEqual(uri.username, 'changeme') 341 self.assertEqual(uri.password, test['password']) 342 uri.password = 'insecure' 343 self.assertEqual(uri.username, 'changeme') 344 self.assertEqual(uri.password, 'insecure') 345 346 # reset back after our trickery 347 uri.userinfo = test['userinfo'] 348 self.assertEqual(uri.userinfo, test['userinfo']) 349 self.assertEqual(uri.username, test['username']) 350 self.assertEqual(uri.password, test['password']) 351 352 uri.hostname = test['hostname'] 353 self.assertEqual(uri.hostname, test['hostname']) 354 self.assertEqual(uri.hostport, test['hostname']) 355 356 uri.port = test['port'] 357 self.assertEqual(uri.port, test['port']) 358 self.assertEqual(uri.hostport, test['hostport']) 359 360 uri.path = test['path'] 361 self.assertEqual(uri.path, test['path']) 362 363 uri.params = test['params'] 364 self.assertEqual(uri.params, test['params']) 365 366 uri.query = test['query'] 367 self.assertEqual(uri.query, test['query']) 368 369 self.assertEqual(str(uri), test['uri']) 370 371 uri.params = {} 372 self.assertEqual(uri.params, {}) 373 self.assertEqual(str(uri), (str(uri).split(";"))[0]) 374 375class FetcherTest(unittest.TestCase): 376 377 def setUp(self): 378 self.origdir = os.getcwd() 379 self.d = bb.data.init() 380 self.tempdir = tempfile.mkdtemp(prefix="bitbake-fetch-") 381 self.dldir = os.path.join(self.tempdir, "download") 382 os.mkdir(self.dldir) 383 self.d.setVar("DL_DIR", self.dldir) 384 self.unpackdir = os.path.join(self.tempdir, "unpacked") 385 os.mkdir(self.unpackdir) 386 persistdir = os.path.join(self.tempdir, "persistdata") 387 self.d.setVar("PERSISTENT_DIR", persistdir) 388 389 def tearDown(self): 390 os.chdir(self.origdir) 391 if os.environ.get("BB_TMPDIR_NOCLEAN") == "yes": 392 print("Not cleaning up %s. Please remove manually." % self.tempdir) 393 else: 394 bb.process.run('chmod u+rw -R %s' % self.tempdir) 395 bb.utils.prunedir(self.tempdir) 396 397 def git(self, cmd, cwd=None): 398 if isinstance(cmd, str): 399 cmd = 'git ' + cmd 400 else: 401 cmd = ['git'] + cmd 402 if cwd is None: 403 cwd = self.gitdir 404 return bb.process.run(cmd, cwd=cwd)[0] 405 406 def git_init(self, cwd=None): 407 self.git('init', cwd=cwd) 408 if not self.git(['config', 'user.email'], cwd=cwd): 409 self.git(['config', 'user.email', 'you@example.com'], cwd=cwd) 410 if not self.git(['config', 'user.name'], cwd=cwd): 411 self.git(['config', 'user.name', 'Your Name'], cwd=cwd) 412 413class MirrorUriTest(FetcherTest): 414 415 replaceuris = { 416 ("git://git.invalid.infradead.org/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/.*", "http://somewhere.org/somedir/") 417 : "http://somewhere.org/somedir/git2_git.invalid.infradead.org.mtd-utils.git.tar.gz", 418 ("git://git.invalid.infradead.org/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/([^/]+/)*([^/]*)", "git://somewhere.org/somedir/\\2;protocol=http") 419 : "git://somewhere.org/somedir/mtd-utils.git;tag=1234567890123456789012345678901234567890;protocol=http", 420 ("git://git.invalid.infradead.org/foo/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/([^/]+/)*([^/]*)", "git://somewhere.org/somedir/\\2;protocol=http") 421 : "git://somewhere.org/somedir/mtd-utils.git;tag=1234567890123456789012345678901234567890;protocol=http", 422 ("git://git.invalid.infradead.org/foo/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/([^/]+/)*([^/]*)", "git://somewhere.org/\\2;protocol=http") 423 : "git://somewhere.org/mtd-utils.git;tag=1234567890123456789012345678901234567890;protocol=http", 424 ("git://someserver.org/bitbake;tag=1234567890123456789012345678901234567890", "git://someserver.org/bitbake", "git://git.openembedded.org/bitbake") 425 : "git://git.openembedded.org/bitbake;tag=1234567890123456789012345678901234567890", 426 ("file://sstate-xyz.tgz", "file://.*", "file:///somewhere/1234/sstate-cache") 427 : "file:///somewhere/1234/sstate-cache/sstate-xyz.tgz", 428 ("file://sstate-xyz.tgz", "file://.*", "file:///somewhere/1234/sstate-cache/") 429 : "file:///somewhere/1234/sstate-cache/sstate-xyz.tgz", 430 ("http://somewhere.org/somedir1/somedir2/somefile_1.2.3.tar.gz", "http://.*/.*", "http://somewhere2.org/somedir3") 431 : "http://somewhere2.org/somedir3/somefile_1.2.3.tar.gz", 432 ("http://somewhere.org/somedir1/somefile_1.2.3.tar.gz", "http://somewhere.org/somedir1/somefile_1.2.3.tar.gz", "http://somewhere2.org/somedir3/somefile_1.2.3.tar.gz") 433 : "http://somewhere2.org/somedir3/somefile_1.2.3.tar.gz", 434 ("http://www.apache.org/dist/subversion/subversion-1.7.1.tar.bz2", "http://www.apache.org/dist", "http://archive.apache.org/dist") 435 : "http://archive.apache.org/dist/subversion/subversion-1.7.1.tar.bz2", 436 ("http://www.apache.org/dist/subversion/subversion-1.7.1.tar.bz2", "http://.*/.*", "file:///somepath/downloads/") 437 : "file:///somepath/downloads/subversion-1.7.1.tar.bz2", 438 ("git://git.invalid.infradead.org/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/.*", "git://somewhere.org/somedir/BASENAME;protocol=http") 439 : "git://somewhere.org/somedir/mtd-utils.git;tag=1234567890123456789012345678901234567890;protocol=http", 440 ("git://git.invalid.infradead.org/foo/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/.*", "git://somewhere.org/somedir/BASENAME;protocol=http") 441 : "git://somewhere.org/somedir/mtd-utils.git;tag=1234567890123456789012345678901234567890;protocol=http", 442 ("git://git.invalid.infradead.org/foo/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/.*", "git://somewhere.org/somedir/MIRRORNAME;protocol=http") 443 : "git://somewhere.org/somedir/git.invalid.infradead.org.foo.mtd-utils.git;tag=1234567890123456789012345678901234567890;protocol=http", 444 ("http://somewhere.org/somedir1/somedir2/somefile_1.2.3.tar.gz", "http://.*/.*", "http://somewhere2.org") 445 : "http://somewhere2.org/somefile_1.2.3.tar.gz", 446 ("http://somewhere.org/somedir1/somedir2/somefile_1.2.3.tar.gz", "http://.*/.*", "http://somewhere2.org/") 447 : "http://somewhere2.org/somefile_1.2.3.tar.gz", 448 ("git://someserver.org/bitbake;tag=1234567890123456789012345678901234567890;branch=master", "git://someserver.org/bitbake;branch=master", "git://git.openembedded.org/bitbake;protocol=http") 449 : "git://git.openembedded.org/bitbake;tag=1234567890123456789012345678901234567890;branch=master;protocol=http", 450 ("git://user1@someserver.org/bitbake;tag=1234567890123456789012345678901234567890;branch=master", "git://someserver.org/bitbake;branch=master", "git://user2@git.openembedded.org/bitbake;protocol=http") 451 : "git://user2@git.openembedded.org/bitbake;tag=1234567890123456789012345678901234567890;branch=master;protocol=http", 452 ("git://someserver.org/bitbake;tag=1234567890123456789012345678901234567890;protocol=git;branch=master", "git://someserver.org/bitbake", "git://someotherserver.org/bitbake;protocol=https") 453 : "git://someotherserver.org/bitbake;tag=1234567890123456789012345678901234567890;protocol=https;branch=master", 454 ("gitsm://git.qemu.org/git/seabios.git/;protocol=https;name=roms/seabios;subpath=roms/seabios;bareclone=1;nobranch=1;rev=1234567890123456789012345678901234567890", "gitsm://.*/.*", "http://petalinux.xilinx.com/sswreleases/rel-v${XILINX_VER_MAIN}/downloads") : "http://petalinux.xilinx.com/sswreleases/rel-v%24%7BXILINX_VER_MAIN%7D/downloads/git2_git.qemu.org.git.seabios.git..tar.gz", 455 ("https://somewhere.org/example/1.0.0/example;downloadfilename=some-example-1.0.0.tgz", "https://.*/.*", "file:///mirror/PATH") 456 : "file:///mirror/example/1.0.0/some-example-1.0.0.tgz;downloadfilename=some-example-1.0.0.tgz", 457 ("https://somewhere.org/example-1.0.0.tgz;downloadfilename=some-example-1.0.0.tgz", "https://.*/.*", "file:///mirror/some-example-1.0.0.tgz") 458 : "file:///mirror/some-example-1.0.0.tgz;downloadfilename=some-example-1.0.0.tgz", 459 460 #Renaming files doesn't work 461 #("http://somewhere.org/somedir1/somefile_1.2.3.tar.gz", "http://somewhere.org/somedir1/somefile_1.2.3.tar.gz", "http://somewhere2.org/somedir3/somefile_2.3.4.tar.gz") : "http://somewhere2.org/somedir3/somefile_2.3.4.tar.gz" 462 #("file://sstate-xyz.tgz", "file://.*/.*", "file:///somewhere/1234/sstate-cache") : "file:///somewhere/1234/sstate-cache/sstate-xyz.tgz", 463 } 464 465 mirrorvar = "http://.*/.* file:///somepath/downloads/ " \ 466 "git://someserver.org/bitbake git://git.openembedded.org/bitbake " \ 467 "https://.*/.* file:///someotherpath/downloads/ " \ 468 "http://.*/.* file:///someotherpath/downloads/" 469 470 def test_urireplace(self): 471 for k, v in self.replaceuris.items(): 472 ud = bb.fetch.FetchData(k[0], self.d) 473 ud.setup_localpath(self.d) 474 mirrors = bb.fetch2.mirror_from_string("%s %s" % (k[1], k[2])) 475 newuris, uds = bb.fetch2.build_mirroruris(ud, mirrors, self.d) 476 self.assertEqual([v], newuris) 477 478 def test_urilist1(self): 479 fetcher = bb.fetch.FetchData("http://downloads.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz", self.d) 480 mirrors = bb.fetch2.mirror_from_string(self.mirrorvar) 481 uris, uds = bb.fetch2.build_mirroruris(fetcher, mirrors, self.d) 482 self.assertEqual(uris, ['file:///somepath/downloads/bitbake-1.0.tar.gz', 'file:///someotherpath/downloads/bitbake-1.0.tar.gz']) 483 484 def test_urilist2(self): 485 # Catch https:// -> files:// bug 486 fetcher = bb.fetch.FetchData("https://downloads.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz", self.d) 487 mirrors = bb.fetch2.mirror_from_string(self.mirrorvar) 488 uris, uds = bb.fetch2.build_mirroruris(fetcher, mirrors, self.d) 489 self.assertEqual(uris, ['file:///someotherpath/downloads/bitbake-1.0.tar.gz']) 490 491 def test_mirror_of_mirror(self): 492 # Test if mirror of a mirror works 493 mirrorvar = self.mirrorvar + " http://.*/.* http://otherdownloads.yoctoproject.org/downloads/" 494 mirrorvar = mirrorvar + " http://otherdownloads.yoctoproject.org/.* http://downloads2.yoctoproject.org/downloads/" 495 fetcher = bb.fetch.FetchData("http://downloads.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz", self.d) 496 mirrors = bb.fetch2.mirror_from_string(mirrorvar) 497 uris, uds = bb.fetch2.build_mirroruris(fetcher, mirrors, self.d) 498 self.assertEqual(uris, ['file:///somepath/downloads/bitbake-1.0.tar.gz', 499 'file:///someotherpath/downloads/bitbake-1.0.tar.gz', 500 'http://otherdownloads.yoctoproject.org/downloads/bitbake-1.0.tar.gz', 501 'http://downloads2.yoctoproject.org/downloads/bitbake-1.0.tar.gz']) 502 503 recmirrorvar = "https://.*/[^/]* http://AAAA/A/A/A/ " \ 504 "https://.*/[^/]* https://BBBB/B/B/B/" 505 506 def test_recursive(self): 507 fetcher = bb.fetch.FetchData("https://downloads.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz", self.d) 508 mirrors = bb.fetch2.mirror_from_string(self.recmirrorvar) 509 uris, uds = bb.fetch2.build_mirroruris(fetcher, mirrors, self.d) 510 self.assertEqual(uris, ['http://AAAA/A/A/A/bitbake/bitbake-1.0.tar.gz', 511 'https://BBBB/B/B/B/bitbake/bitbake-1.0.tar.gz', 512 'http://AAAA/A/A/A/B/B/bitbake/bitbake-1.0.tar.gz']) 513 514 515class GitDownloadDirectoryNamingTest(FetcherTest): 516 def setUp(self): 517 super(GitDownloadDirectoryNamingTest, self).setUp() 518 self.recipe_url = "git://git.openembedded.org/bitbake;branch=master" 519 self.recipe_dir = "git.openembedded.org.bitbake" 520 self.mirror_url = "git://github.com/openembedded/bitbake.git;protocol=https;branch=master" 521 self.mirror_dir = "github.com.openembedded.bitbake.git" 522 523 self.d.setVar('SRCREV', '82ea737a0b42a8b53e11c9cde141e9e9c0bd8c40') 524 525 def setup_mirror_rewrite(self): 526 self.d.setVar("PREMIRRORS", self.recipe_url + " " + self.mirror_url) 527 528 @skipIfNoNetwork() 529 def test_that_directory_is_named_after_recipe_url_when_no_mirroring_is_used(self): 530 self.setup_mirror_rewrite() 531 fetcher = bb.fetch.Fetch([self.recipe_url], self.d) 532 533 fetcher.download() 534 535 dir = os.listdir(self.dldir + "/git2") 536 self.assertIn(self.recipe_dir, dir) 537 538 @skipIfNoNetwork() 539 def test_that_directory_exists_for_mirrored_url_and_recipe_url_when_mirroring_is_used(self): 540 self.setup_mirror_rewrite() 541 fetcher = bb.fetch.Fetch([self.recipe_url], self.d) 542 543 fetcher.download() 544 545 dir = os.listdir(self.dldir + "/git2") 546 self.assertIn(self.mirror_dir, dir) 547 self.assertIn(self.recipe_dir, dir) 548 549 @skipIfNoNetwork() 550 def test_that_recipe_directory_and_mirrored_directory_exists_when_mirroring_is_used_and_the_mirrored_directory_already_exists(self): 551 self.setup_mirror_rewrite() 552 fetcher = bb.fetch.Fetch([self.mirror_url], self.d) 553 fetcher.download() 554 fetcher = bb.fetch.Fetch([self.recipe_url], self.d) 555 556 fetcher.download() 557 558 dir = os.listdir(self.dldir + "/git2") 559 self.assertIn(self.mirror_dir, dir) 560 self.assertIn(self.recipe_dir, dir) 561 562 563class TarballNamingTest(FetcherTest): 564 def setUp(self): 565 super(TarballNamingTest, self).setUp() 566 self.recipe_url = "git://git.openembedded.org/bitbake;branch=master" 567 self.recipe_tarball = "git2_git.openembedded.org.bitbake.tar.gz" 568 self.mirror_url = "git://github.com/openembedded/bitbake.git;protocol=https;branch=master" 569 self.mirror_tarball = "git2_github.com.openembedded.bitbake.git.tar.gz" 570 571 self.d.setVar('BB_GENERATE_MIRROR_TARBALLS', '1') 572 self.d.setVar('SRCREV', '82ea737a0b42a8b53e11c9cde141e9e9c0bd8c40') 573 574 def setup_mirror_rewrite(self): 575 self.d.setVar("PREMIRRORS", self.recipe_url + " " + self.mirror_url) 576 577 @skipIfNoNetwork() 578 def test_that_the_recipe_tarball_is_created_when_no_mirroring_is_used(self): 579 fetcher = bb.fetch.Fetch([self.recipe_url], self.d) 580 581 fetcher.download() 582 583 dir = os.listdir(self.dldir) 584 self.assertIn(self.recipe_tarball, dir) 585 586 @skipIfNoNetwork() 587 def test_that_the_mirror_tarball_is_created_when_mirroring_is_used(self): 588 self.setup_mirror_rewrite() 589 fetcher = bb.fetch.Fetch([self.recipe_url], self.d) 590 591 fetcher.download() 592 593 dir = os.listdir(self.dldir) 594 self.assertIn(self.mirror_tarball, dir) 595 596 597class GitShallowTarballNamingTest(FetcherTest): 598 def setUp(self): 599 super(GitShallowTarballNamingTest, self).setUp() 600 self.recipe_url = "git://git.openembedded.org/bitbake;branch=master" 601 self.recipe_tarball = "gitshallow_git.openembedded.org.bitbake_82ea737-1_master.tar.gz" 602 self.mirror_url = "git://github.com/openembedded/bitbake.git;protocol=https;branch=master" 603 self.mirror_tarball = "gitshallow_github.com.openembedded.bitbake.git_82ea737-1_master.tar.gz" 604 605 self.d.setVar('BB_GIT_SHALLOW', '1') 606 self.d.setVar('BB_GENERATE_SHALLOW_TARBALLS', '1') 607 self.d.setVar('SRCREV', '82ea737a0b42a8b53e11c9cde141e9e9c0bd8c40') 608 609 def setup_mirror_rewrite(self): 610 self.d.setVar("PREMIRRORS", self.recipe_url + " " + self.mirror_url) 611 612 @skipIfNoNetwork() 613 def test_that_the_tarball_is_named_after_recipe_url_when_no_mirroring_is_used(self): 614 fetcher = bb.fetch.Fetch([self.recipe_url], self.d) 615 616 fetcher.download() 617 618 dir = os.listdir(self.dldir) 619 self.assertIn(self.recipe_tarball, dir) 620 621 @skipIfNoNetwork() 622 def test_that_the_mirror_tarball_is_created_when_mirroring_is_used(self): 623 self.setup_mirror_rewrite() 624 fetcher = bb.fetch.Fetch([self.recipe_url], self.d) 625 626 fetcher.download() 627 628 dir = os.listdir(self.dldir) 629 self.assertIn(self.mirror_tarball, dir) 630 631 632class CleanTarballTest(FetcherTest): 633 def setUp(self): 634 super(CleanTarballTest, self).setUp() 635 self.recipe_url = "git://git.openembedded.org/bitbake" 636 self.recipe_tarball = "git2_git.openembedded.org.bitbake.tar.gz" 637 638 self.d.setVar('BB_GENERATE_MIRROR_TARBALLS', '1') 639 self.d.setVar('SRCREV', '82ea737a0b42a8b53e11c9cde141e9e9c0bd8c40') 640 641 @skipIfNoNetwork() 642 def test_that_the_tarball_contents_does_not_leak_info(self): 643 fetcher = bb.fetch.Fetch([self.recipe_url], self.d) 644 645 fetcher.download() 646 647 fetcher.unpack(self.unpackdir) 648 mtime = bb.process.run('git log --all -1 --format=%ct', 649 cwd=os.path.join(self.unpackdir, 'git')) 650 self.assertEqual(len(mtime), 2) 651 mtime = int(mtime[0]) 652 653 archive = tarfile.open(os.path.join(self.dldir, self.recipe_tarball)) 654 self.assertNotEqual(len(archive.members), 0) 655 for member in archive.members: 656 self.assertEqual(member.uname, 'oe') 657 self.assertEqual(member.uid, 0) 658 self.assertEqual(member.gname, 'oe') 659 self.assertEqual(member.gid, 0) 660 self.assertEqual(member.mtime, mtime) 661 662 663class FetcherLocalTest(FetcherTest): 664 def setUp(self): 665 def touch(fn): 666 with open(fn, 'a'): 667 os.utime(fn, None) 668 669 super(FetcherLocalTest, self).setUp() 670 self.localsrcdir = os.path.join(self.tempdir, 'localsrc') 671 os.makedirs(self.localsrcdir) 672 touch(os.path.join(self.localsrcdir, 'a')) 673 touch(os.path.join(self.localsrcdir, 'b')) 674 os.makedirs(os.path.join(self.localsrcdir, 'dir')) 675 touch(os.path.join(self.localsrcdir, 'dir', 'c')) 676 touch(os.path.join(self.localsrcdir, 'dir', 'd')) 677 os.makedirs(os.path.join(self.localsrcdir, 'dir', 'subdir')) 678 touch(os.path.join(self.localsrcdir, 'dir', 'subdir', 'e')) 679 touch(os.path.join(self.localsrcdir, r'backslash\x2dsystemd-unit.device')) 680 bb.process.run('tar cf archive.tar -C dir .', cwd=self.localsrcdir) 681 bb.process.run('tar czf archive.tar.gz -C dir .', cwd=self.localsrcdir) 682 bb.process.run('tar cjf archive.tar.bz2 -C dir .', cwd=self.localsrcdir) 683 self.d.setVar("FILESPATH", self.localsrcdir) 684 685 def fetchUnpack(self, uris): 686 fetcher = bb.fetch.Fetch(uris, self.d) 687 fetcher.download() 688 fetcher.unpack(self.unpackdir) 689 flst = [] 690 for root, dirs, files in os.walk(self.unpackdir): 691 for f in files: 692 flst.append(os.path.relpath(os.path.join(root, f), self.unpackdir)) 693 flst.sort() 694 return flst 695 696 def test_local(self): 697 tree = self.fetchUnpack(['file://a', 'file://dir/c']) 698 self.assertEqual(tree, ['a', 'dir/c']) 699 700 def test_local_backslash(self): 701 tree = self.fetchUnpack([r'file://backslash\x2dsystemd-unit.device']) 702 self.assertEqual(tree, [r'backslash\x2dsystemd-unit.device']) 703 704 def test_local_wildcard(self): 705 with self.assertRaises(bb.fetch2.ParameterError): 706 tree = self.fetchUnpack(['file://a', 'file://dir/*']) 707 708 def test_local_dir(self): 709 tree = self.fetchUnpack(['file://a', 'file://dir']) 710 self.assertEqual(tree, ['a', 'dir/c', 'dir/d', 'dir/subdir/e']) 711 712 def test_local_subdir(self): 713 tree = self.fetchUnpack(['file://dir/subdir']) 714 self.assertEqual(tree, ['dir/subdir/e']) 715 716 def test_local_subdir_file(self): 717 tree = self.fetchUnpack(['file://dir/subdir/e']) 718 self.assertEqual(tree, ['dir/subdir/e']) 719 720 def test_local_subdirparam(self): 721 tree = self.fetchUnpack(['file://a;subdir=bar', 'file://dir;subdir=foo/moo']) 722 self.assertEqual(tree, ['bar/a', 'foo/moo/dir/c', 'foo/moo/dir/d', 'foo/moo/dir/subdir/e']) 723 724 def test_local_deepsubdirparam(self): 725 tree = self.fetchUnpack(['file://dir/subdir/e;subdir=bar']) 726 self.assertEqual(tree, ['bar/dir/subdir/e']) 727 728 def test_local_absolutedir(self): 729 # Unpacking to an absolute path that is a subdirectory of the root 730 # should work 731 tree = self.fetchUnpack(['file://a;subdir=%s' % os.path.join(self.unpackdir, 'bar')]) 732 733 # Unpacking to an absolute path outside of the root should fail 734 with self.assertRaises(bb.fetch2.UnpackError): 735 self.fetchUnpack(['file://a;subdir=/bin/sh']) 736 737 def test_local_striplevel(self): 738 tree = self.fetchUnpack(['file://archive.tar;subdir=bar;striplevel=1']) 739 self.assertEqual(tree, ['bar/c', 'bar/d', 'bar/subdir/e']) 740 741 def test_local_striplevel_gzip(self): 742 tree = self.fetchUnpack(['file://archive.tar.gz;subdir=bar;striplevel=1']) 743 self.assertEqual(tree, ['bar/c', 'bar/d', 'bar/subdir/e']) 744 745 def test_local_striplevel_bzip2(self): 746 tree = self.fetchUnpack(['file://archive.tar.bz2;subdir=bar;striplevel=1']) 747 self.assertEqual(tree, ['bar/c', 'bar/d', 'bar/subdir/e']) 748 749 def dummyGitTest(self, suffix): 750 # Create dummy local Git repo 751 src_dir = tempfile.mkdtemp(dir=self.tempdir, 752 prefix='gitfetch_localusehead_') 753 self.gitdir = os.path.abspath(src_dir) 754 self.git_init() 755 self.git(['commit', '--allow-empty', '-m', 'Dummy commit']) 756 # Use other branch than master 757 self.git(['checkout', '-b', 'my-devel']) 758 self.git(['commit', '--allow-empty', '-m', 'Dummy commit 2']) 759 orig_rev = self.git(['rev-parse', 'HEAD']).strip() 760 761 # Fetch and check revision 762 self.d.setVar("SRCREV", "AUTOINC") 763 self.d.setVar("__BBSEENSRCREV", "1") 764 url = "git://" + self.gitdir + ";branch=master;protocol=file;" + suffix 765 fetcher = bb.fetch.Fetch([url], self.d) 766 fetcher.download() 767 fetcher.unpack(self.unpackdir) 768 unpack_rev = self.git(['rev-parse', 'HEAD'], 769 cwd=os.path.join(self.unpackdir, 'git')).strip() 770 self.assertEqual(orig_rev, unpack_rev) 771 772 def test_local_gitfetch_usehead(self): 773 self.dummyGitTest("usehead=1") 774 775 def test_local_gitfetch_usehead_withname(self): 776 self.dummyGitTest("usehead=1;name=newName") 777 778 def test_local_gitfetch_shared(self): 779 self.dummyGitTest("usehead=1;name=sharedName") 780 alt = os.path.join(self.unpackdir, 'git/.git/objects/info/alternates') 781 self.assertTrue(os.path.exists(alt)) 782 783 def test_local_gitfetch_noshared(self): 784 self.d.setVar('BB_GIT_NOSHARED', '1') 785 self.unpackdir += '_noshared' 786 self.dummyGitTest("usehead=1;name=noSharedName") 787 alt = os.path.join(self.unpackdir, 'git/.git/objects/info/alternates') 788 self.assertFalse(os.path.exists(alt)) 789 790class FetcherNoNetworkTest(FetcherTest): 791 def setUp(self): 792 super().setUp() 793 # all test cases are based on not having network 794 self.d.setVar("BB_NO_NETWORK", "1") 795 796 def test_missing(self): 797 string = "this is a test file\n".encode("utf-8") 798 self.d.setVarFlag("SRC_URI", "md5sum", hashlib.md5(string).hexdigest()) 799 self.d.setVarFlag("SRC_URI", "sha256sum", hashlib.sha256(string).hexdigest()) 800 801 self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz"))) 802 self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done"))) 803 fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/test-file.tar.gz"], self.d) 804 with self.assertRaises(bb.fetch2.NetworkAccess): 805 fetcher.download() 806 807 def test_valid_missing_donestamp(self): 808 # create the file in the download directory with correct hash 809 string = "this is a test file\n".encode("utf-8") 810 with open(os.path.join(self.dldir, "test-file.tar.gz"), "wb") as f: 811 f.write(string) 812 813 self.d.setVarFlag("SRC_URI", "md5sum", hashlib.md5(string).hexdigest()) 814 self.d.setVarFlag("SRC_URI", "sha256sum", hashlib.sha256(string).hexdigest()) 815 816 self.assertTrue(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz"))) 817 self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done"))) 818 fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/test-file.tar.gz"], self.d) 819 fetcher.download() 820 self.assertTrue(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done"))) 821 822 def test_invalid_missing_donestamp(self): 823 # create an invalid file in the download directory with incorrect hash 824 string = "this is a test file\n".encode("utf-8") 825 with open(os.path.join(self.dldir, "test-file.tar.gz"), "wb"): 826 pass 827 828 self.d.setVarFlag("SRC_URI", "md5sum", hashlib.md5(string).hexdigest()) 829 self.d.setVarFlag("SRC_URI", "sha256sum", hashlib.sha256(string).hexdigest()) 830 831 self.assertTrue(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz"))) 832 self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done"))) 833 fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/test-file.tar.gz"], self.d) 834 with self.assertRaises(bb.fetch2.NetworkAccess): 835 fetcher.download() 836 # the existing file should not exist or should have be moved to "bad-checksum" 837 self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz"))) 838 839 def test_nochecksums_missing(self): 840 self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz"))) 841 self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done"))) 842 # ssh fetch does not support checksums 843 fetcher = bb.fetch.Fetch(["ssh://invalid@invalid.yoctoproject.org/test-file.tar.gz"], self.d) 844 # attempts to download with missing donestamp 845 with self.assertRaises(bb.fetch2.NetworkAccess): 846 fetcher.download() 847 848 def test_nochecksums_missing_donestamp(self): 849 # create a file in the download directory 850 with open(os.path.join(self.dldir, "test-file.tar.gz"), "wb"): 851 pass 852 853 self.assertTrue(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz"))) 854 self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done"))) 855 # ssh fetch does not support checksums 856 fetcher = bb.fetch.Fetch(["ssh://invalid@invalid.yoctoproject.org/test-file.tar.gz"], self.d) 857 # attempts to download with missing donestamp 858 with self.assertRaises(bb.fetch2.NetworkAccess): 859 fetcher.download() 860 861 def test_nochecksums_has_donestamp(self): 862 # create a file in the download directory with the donestamp 863 with open(os.path.join(self.dldir, "test-file.tar.gz"), "wb"): 864 pass 865 with open(os.path.join(self.dldir, "test-file.tar.gz.done"), "wb"): 866 pass 867 868 self.assertTrue(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz"))) 869 self.assertTrue(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done"))) 870 # ssh fetch does not support checksums 871 fetcher = bb.fetch.Fetch(["ssh://invalid@invalid.yoctoproject.org/test-file.tar.gz"], self.d) 872 # should not fetch 873 fetcher.download() 874 # both files should still exist 875 self.assertTrue(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz"))) 876 self.assertTrue(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done"))) 877 878 def test_nochecksums_missing_has_donestamp(self): 879 # create a file in the download directory with the donestamp 880 with open(os.path.join(self.dldir, "test-file.tar.gz.done"), "wb"): 881 pass 882 883 self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz"))) 884 self.assertTrue(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done"))) 885 # ssh fetch does not support checksums 886 fetcher = bb.fetch.Fetch(["ssh://invalid@invalid.yoctoproject.org/test-file.tar.gz"], self.d) 887 with self.assertRaises(bb.fetch2.NetworkAccess): 888 fetcher.download() 889 # both files should still exist 890 self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz"))) 891 self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done"))) 892 893class FetcherNetworkTest(FetcherTest): 894 @skipIfNoNetwork() 895 def test_fetch(self): 896 fetcher = bb.fetch.Fetch(["https://downloads.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz", "https://downloads.yoctoproject.org/releases/bitbake/bitbake-1.1.tar.gz"], self.d) 897 fetcher.download() 898 self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.0.tar.gz"), 57749) 899 self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.1.tar.gz"), 57892) 900 self.d.setVar("BB_NO_NETWORK", "1") 901 fetcher = bb.fetch.Fetch(["https://downloads.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz", "https://downloads.yoctoproject.org/releases/bitbake/bitbake-1.1.tar.gz"], self.d) 902 fetcher.download() 903 fetcher.unpack(self.unpackdir) 904 self.assertEqual(len(os.listdir(self.unpackdir + "/bitbake-1.0/")), 9) 905 self.assertEqual(len(os.listdir(self.unpackdir + "/bitbake-1.1/")), 9) 906 907 @skipIfNoNetwork() 908 def test_fetch_mirror(self): 909 self.d.setVar("MIRRORS", "http://.*/.* https://downloads.yoctoproject.org/releases/bitbake") 910 fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz"], self.d) 911 fetcher.download() 912 self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.0.tar.gz"), 57749) 913 914 @skipIfNoNetwork() 915 def test_fetch_mirror_of_mirror(self): 916 self.d.setVar("MIRRORS", "http://.*/.* http://invalid2.yoctoproject.org/ http://invalid2.yoctoproject.org/.* https://downloads.yoctoproject.org/releases/bitbake") 917 fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz"], self.d) 918 fetcher.download() 919 self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.0.tar.gz"), 57749) 920 921 @skipIfNoNetwork() 922 def test_fetch_file_mirror_of_mirror(self): 923 self.d.setVar("MIRRORS", "http://.*/.* file:///some1where/ file:///some1where/.* file://some2where/ file://some2where/.* https://downloads.yoctoproject.org/releases/bitbake") 924 fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz"], self.d) 925 os.mkdir(self.dldir + "/some2where") 926 fetcher.download() 927 self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.0.tar.gz"), 57749) 928 929 @skipIfNoNetwork() 930 def test_fetch_premirror(self): 931 self.d.setVar("PREMIRRORS", "http://.*/.* https://downloads.yoctoproject.org/releases/bitbake") 932 fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz"], self.d) 933 fetcher.download() 934 self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.0.tar.gz"), 57749) 935 936 @skipIfNoNetwork() 937 def test_fetch_specify_downloadfilename(self): 938 fetcher = bb.fetch.Fetch(["https://downloads.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz;downloadfilename=bitbake-v1.0.0.tar.gz"], self.d) 939 fetcher.download() 940 self.assertEqual(os.path.getsize(self.dldir + "/bitbake-v1.0.0.tar.gz"), 57749) 941 942 @skipIfNoNetwork() 943 def test_fetch_premirror_specify_downloadfilename_regex_uri(self): 944 self.d.setVar("PREMIRRORS", "http://.*/.* https://downloads.yoctoproject.org/releases/bitbake/") 945 fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/releases/bitbake/1.0.tar.gz;downloadfilename=bitbake-1.0.tar.gz"], self.d) 946 fetcher.download() 947 self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.0.tar.gz"), 57749) 948 949 @skipIfNoNetwork() 950 # BZ13039 951 def test_fetch_premirror_specify_downloadfilename_specific_uri(self): 952 self.d.setVar("PREMIRRORS", "http://invalid.yoctoproject.org/releases/bitbake https://downloads.yoctoproject.org/releases/bitbake") 953 fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/releases/bitbake/1.0.tar.gz;downloadfilename=bitbake-1.0.tar.gz"], self.d) 954 fetcher.download() 955 self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.0.tar.gz"), 57749) 956 957 @skipIfNoNetwork() 958 def test_fetch_premirror_use_downloadfilename_to_fetch(self): 959 # Ensure downloadfilename is used when fetching from premirror. 960 self.d.setVar("PREMIRRORS", "http://.*/.* https://downloads.yoctoproject.org/releases/bitbake") 961 fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/releases/bitbake/bitbake-1.1.tar.gz;downloadfilename=bitbake-1.0.tar.gz"], self.d) 962 fetcher.download() 963 self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.0.tar.gz"), 57749) 964 965 @skipIfNoNetwork() 966 def gitfetcher(self, url1, url2): 967 def checkrevision(self, fetcher): 968 fetcher.unpack(self.unpackdir) 969 revision = self.git(['rev-parse', 'HEAD'], 970 cwd=os.path.join(self.unpackdir, 'git')).strip() 971 self.assertEqual(revision, "270a05b0b4ba0959fe0624d2a4885d7b70426da5") 972 973 self.d.setVar("BB_GENERATE_MIRROR_TARBALLS", "1") 974 self.d.setVar("SRCREV", "270a05b0b4ba0959fe0624d2a4885d7b70426da5") 975 fetcher = bb.fetch.Fetch([url1], self.d) 976 fetcher.download() 977 checkrevision(self, fetcher) 978 # Wipe out the dldir clone and the unpacked source, turn off the network and check mirror tarball works 979 bb.utils.prunedir(self.dldir + "/git2/") 980 bb.utils.prunedir(self.unpackdir) 981 self.d.setVar("BB_NO_NETWORK", "1") 982 fetcher = bb.fetch.Fetch([url2], self.d) 983 fetcher.download() 984 checkrevision(self, fetcher) 985 986 @skipIfNoNetwork() 987 def test_gitfetch(self): 988 url1 = url2 = "git://git.openembedded.org/bitbake;branch=master" 989 self.gitfetcher(url1, url2) 990 991 @skipIfNoNetwork() 992 def test_gitfetch_goodsrcrev(self): 993 # SRCREV is set but matches rev= parameter 994 url1 = url2 = "git://git.openembedded.org/bitbake;rev=270a05b0b4ba0959fe0624d2a4885d7b70426da5;branch=master" 995 self.gitfetcher(url1, url2) 996 997 @skipIfNoNetwork() 998 def test_gitfetch_badsrcrev(self): 999 # SRCREV is set but does not match rev= parameter 1000 url1 = url2 = "git://git.openembedded.org/bitbake;rev=dead05b0b4ba0959fe0624d2a4885d7b70426da5;branch=master" 1001 self.assertRaises(bb.fetch.FetchError, self.gitfetcher, url1, url2) 1002 1003 @skipIfNoNetwork() 1004 def test_gitfetch_tagandrev(self): 1005 # SRCREV is set but does not match rev= parameter 1006 url1 = url2 = "git://git.openembedded.org/bitbake;rev=270a05b0b4ba0959fe0624d2a4885d7b70426da5;tag=270a05b0b4ba0959fe0624d2a4885d7b70426da5" 1007 self.assertRaises(bb.fetch.FetchError, self.gitfetcher, url1, url2) 1008 1009 @skipIfNoNetwork() 1010 def test_gitfetch_usehead(self): 1011 # Since self.gitfetcher() sets SRCREV we expect this to override 1012 # `usehead=1' and instead fetch the specified SRCREV. See 1013 # test_local_gitfetch_usehead() for a positive use of the usehead 1014 # feature. 1015 url = "git://git.openembedded.org/bitbake;usehead=1;branch=master" 1016 self.assertRaises(bb.fetch.ParameterError, self.gitfetcher, url, url) 1017 1018 @skipIfNoNetwork() 1019 def test_gitfetch_usehead_withname(self): 1020 # Since self.gitfetcher() sets SRCREV we expect this to override 1021 # `usehead=1' and instead fetch the specified SRCREV. See 1022 # test_local_gitfetch_usehead() for a positive use of the usehead 1023 # feature. 1024 url = "git://git.openembedded.org/bitbake;usehead=1;name=newName;branch=master" 1025 self.assertRaises(bb.fetch.ParameterError, self.gitfetcher, url, url) 1026 1027 @skipIfNoNetwork() 1028 def test_gitfetch_finds_local_tarball_for_mirrored_url_when_previous_downloaded_by_the_recipe_url(self): 1029 recipeurl = "git://git.openembedded.org/bitbake;branch=master" 1030 mirrorurl = "git://someserver.org/bitbake;branch=master" 1031 self.d.setVar("PREMIRRORS", "git://someserver.org/bitbake git://git.openembedded.org/bitbake") 1032 self.gitfetcher(recipeurl, mirrorurl) 1033 1034 @skipIfNoNetwork() 1035 def test_gitfetch_finds_local_tarball_when_previous_downloaded_from_a_premirror(self): 1036 recipeurl = "git://someserver.org/bitbake;branch=master" 1037 self.d.setVar("PREMIRRORS", "git://someserver.org/bitbake git://git.openembedded.org/bitbake") 1038 self.gitfetcher(recipeurl, recipeurl) 1039 1040 @skipIfNoNetwork() 1041 def test_gitfetch_finds_local_repository_when_premirror_rewrites_the_recipe_url(self): 1042 realurl = "git://git.openembedded.org/bitbake" 1043 recipeurl = "git://someserver.org/bitbake" 1044 self.sourcedir = self.unpackdir.replace("unpacked", "sourcemirror.git") 1045 os.chdir(self.tempdir) 1046 self.git(['clone', realurl, self.sourcedir], cwd=self.tempdir) 1047 self.d.setVar("PREMIRRORS", "%s git://%s;protocol=file" % (recipeurl, self.sourcedir)) 1048 self.gitfetcher(recipeurl, recipeurl) 1049 1050 @skipIfNoNetwork() 1051 def test_git_submodule(self): 1052 # URL with ssh submodules 1053 url = "gitsm://git.yoctoproject.org/git-submodule-test;branch=ssh-gitsm-tests;rev=049da4a6cb198d7c0302e9e8b243a1443cb809a7;branch=master" 1054 # Original URL (comment this if you have ssh access to git.yoctoproject.org) 1055 url = "gitsm://git.yoctoproject.org/git-submodule-test;branch=master;rev=a2885dd7d25380d23627e7544b7bbb55014b16ee;branch=master" 1056 fetcher = bb.fetch.Fetch([url], self.d) 1057 fetcher.download() 1058 # Previous cwd has been deleted 1059 os.chdir(os.path.dirname(self.unpackdir)) 1060 fetcher.unpack(self.unpackdir) 1061 1062 repo_path = os.path.join(self.tempdir, 'unpacked', 'git') 1063 self.assertTrue(os.path.exists(repo_path), msg='Unpacked repository missing') 1064 self.assertTrue(os.path.exists(os.path.join(repo_path, 'bitbake')), msg='bitbake submodule missing') 1065 self.assertFalse(os.path.exists(os.path.join(repo_path, 'na')), msg='uninitialized submodule present') 1066 1067 # Only when we're running the extended test with a submodule's submodule, can we check this. 1068 if os.path.exists(os.path.join(repo_path, 'bitbake-gitsm-test1')): 1069 self.assertTrue(os.path.exists(os.path.join(repo_path, 'bitbake-gitsm-test1', 'bitbake')), msg='submodule of submodule missing') 1070 1071 @skipIfNoNetwork() 1072 def test_git_submodule_dbus_broker(self): 1073 # The following external repositories have show failures in fetch and unpack operations 1074 # We want to avoid regressions! 1075 url = "gitsm://github.com/bus1/dbus-broker;protocol=https;rev=fc874afa0992d0c75ec25acb43d344679f0ee7d2;branch=main" 1076 fetcher = bb.fetch.Fetch([url], self.d) 1077 fetcher.download() 1078 # Previous cwd has been deleted 1079 os.chdir(os.path.dirname(self.unpackdir)) 1080 fetcher.unpack(self.unpackdir) 1081 1082 repo_path = os.path.join(self.tempdir, 'unpacked', 'git') 1083 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/subprojects/c-dvar/config')), msg='Missing submodule config "subprojects/c-dvar"') 1084 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/subprojects/c-list/config')), msg='Missing submodule config "subprojects/c-list"') 1085 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/subprojects/c-rbtree/config')), msg='Missing submodule config "subprojects/c-rbtree"') 1086 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/subprojects/c-sundry/config')), msg='Missing submodule config "subprojects/c-sundry"') 1087 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/subprojects/c-utf8/config')), msg='Missing submodule config "subprojects/c-utf8"') 1088 1089 @skipIfNoNetwork() 1090 def test_git_submodule_CLI11(self): 1091 url = "gitsm://github.com/CLIUtils/CLI11;protocol=https;rev=bd4dc911847d0cde7a6b41dfa626a85aab213baf;branch=main" 1092 fetcher = bb.fetch.Fetch([url], self.d) 1093 fetcher.download() 1094 # Previous cwd has been deleted 1095 os.chdir(os.path.dirname(self.unpackdir)) 1096 fetcher.unpack(self.unpackdir) 1097 1098 repo_path = os.path.join(self.tempdir, 'unpacked', 'git') 1099 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/extern/googletest/config')), msg='Missing submodule config "extern/googletest"') 1100 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/extern/json/config')), msg='Missing submodule config "extern/json"') 1101 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/extern/sanitizers/config')), msg='Missing submodule config "extern/sanitizers"') 1102 1103 @skipIfNoNetwork() 1104 def test_git_submodule_update_CLI11(self): 1105 """ Prevent regression on update detection not finding missing submodule, or modules without needed commits """ 1106 url = "gitsm://github.com/CLIUtils/CLI11;protocol=https;rev=cf6a99fa69aaefe477cc52e3ef4a7d2d7fa40714;branch=main" 1107 fetcher = bb.fetch.Fetch([url], self.d) 1108 fetcher.download() 1109 1110 # CLI11 that pulls in a newer nlohmann-json 1111 url = "gitsm://github.com/CLIUtils/CLI11;protocol=https;rev=49ac989a9527ee9bb496de9ded7b4872c2e0e5ca;branch=main" 1112 fetcher = bb.fetch.Fetch([url], self.d) 1113 fetcher.download() 1114 # Previous cwd has been deleted 1115 os.chdir(os.path.dirname(self.unpackdir)) 1116 fetcher.unpack(self.unpackdir) 1117 1118 repo_path = os.path.join(self.tempdir, 'unpacked', 'git') 1119 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/extern/googletest/config')), msg='Missing submodule config "extern/googletest"') 1120 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/extern/json/config')), msg='Missing submodule config "extern/json"') 1121 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/extern/sanitizers/config')), msg='Missing submodule config "extern/sanitizers"') 1122 1123 @skipIfNoNetwork() 1124 def test_git_submodule_aktualizr(self): 1125 url = "gitsm://github.com/advancedtelematic/aktualizr;branch=master;protocol=https;rev=d00d1a04cc2366d1a5f143b84b9f507f8bd32c44" 1126 fetcher = bb.fetch.Fetch([url], self.d) 1127 fetcher.download() 1128 # Previous cwd has been deleted 1129 os.chdir(os.path.dirname(self.unpackdir)) 1130 fetcher.unpack(self.unpackdir) 1131 1132 repo_path = os.path.join(self.tempdir, 'unpacked', 'git') 1133 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/partial/extern/isotp-c/config')), msg='Missing submodule config "partial/extern/isotp-c/config"') 1134 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/partial/extern/isotp-c/modules/deps/bitfield-c/config')), msg='Missing submodule config "partial/extern/isotp-c/modules/deps/bitfield-c/config"') 1135 self.assertTrue(os.path.exists(os.path.join(repo_path, 'partial/extern/isotp-c/deps/bitfield-c/.git')), msg="Submodule of submodule isotp-c did not unpack properly") 1136 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/tests/tuf-test-vectors/config')), msg='Missing submodule config "tests/tuf-test-vectors/config"') 1137 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/third_party/googletest/config')), msg='Missing submodule config "third_party/googletest/config"') 1138 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/third_party/HdrHistogram_c/config')), msg='Missing submodule config "third_party/HdrHistogram_c/config"') 1139 1140 @skipIfNoNetwork() 1141 def test_git_submodule_iotedge(self): 1142 """ Prevent regression on deeply nested submodules not being checked out properly, even though they were fetched. """ 1143 1144 # This repository also has submodules where the module (name), path and url do not align 1145 url = "gitsm://github.com/azure/iotedge.git;protocol=https;rev=d76e0316c6f324345d77c48a83ce836d09392699;branch=main" 1146 fetcher = bb.fetch.Fetch([url], self.d) 1147 fetcher.download() 1148 # Previous cwd has been deleted 1149 os.chdir(os.path.dirname(self.unpackdir)) 1150 fetcher.unpack(self.unpackdir) 1151 1152 repo_path = os.path.join(self.tempdir, 'unpacked', 'git') 1153 1154 self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/c-shared/README.md')), msg='Missing submodule checkout') 1155 self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/c-shared/testtools/ctest/README.md')), msg='Missing submodule checkout') 1156 self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/c-shared/testtools/testrunner/readme.md')), msg='Missing submodule checkout') 1157 self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/c-shared/testtools/umock-c/readme.md')), msg='Missing submodule checkout') 1158 self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/c-shared/testtools/umock-c/deps/ctest/README.md')), msg='Missing submodule checkout') 1159 self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/c-shared/testtools/umock-c/deps/testrunner/readme.md')), msg='Missing submodule checkout') 1160 self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/utpm/README.md')), msg='Missing submodule checkout') 1161 self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/utpm/deps/c-utility/README.md')), msg='Missing submodule checkout') 1162 self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/utpm/deps/c-utility/testtools/ctest/README.md')), msg='Missing submodule checkout') 1163 self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/utpm/deps/c-utility/testtools/testrunner/readme.md')), msg='Missing submodule checkout') 1164 self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/utpm/deps/c-utility/testtools/umock-c/readme.md')), msg='Missing submodule checkout') 1165 self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/utpm/deps/c-utility/testtools/umock-c/deps/ctest/README.md')), msg='Missing submodule checkout') 1166 self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/utpm/deps/c-utility/testtools/umock-c/deps/testrunner/readme.md')), msg='Missing submodule checkout') 1167 1168class SVNTest(FetcherTest): 1169 def skipIfNoSvn(): 1170 import shutil 1171 if not shutil.which("svn"): 1172 return unittest.skip("svn not installed, tests being skipped") 1173 1174 if not shutil.which("svnadmin"): 1175 return unittest.skip("svnadmin not installed, tests being skipped") 1176 1177 return lambda f: f 1178 1179 @skipIfNoSvn() 1180 def setUp(self): 1181 """ Create a local repository """ 1182 1183 super(SVNTest, self).setUp() 1184 1185 # Create something we can fetch 1186 src_dir = tempfile.mkdtemp(dir=self.tempdir, 1187 prefix='svnfetch_srcdir_') 1188 src_dir = os.path.abspath(src_dir) 1189 bb.process.run("echo readme > README.md", cwd=src_dir) 1190 1191 # Store it in a local SVN repository 1192 repo_dir = tempfile.mkdtemp(dir=self.tempdir, 1193 prefix='svnfetch_localrepo_') 1194 repo_dir = os.path.abspath(repo_dir) 1195 bb.process.run("svnadmin create project", cwd=repo_dir) 1196 1197 self.repo_url = "file://%s/project" % repo_dir 1198 bb.process.run("svn import --non-interactive -m 'Initial import' %s %s/trunk" % (src_dir, self.repo_url), 1199 cwd=repo_dir) 1200 1201 bb.process.run("svn co %s svnfetch_co" % self.repo_url, cwd=self.tempdir) 1202 # Github will emulate SVN. Use this to check if we're downloding... 1203 bb.process.run("svn propset svn:externals 'bitbake https://github.com/PhilipHazel/pcre2.git' .", 1204 cwd=os.path.join(self.tempdir, 'svnfetch_co', 'trunk')) 1205 bb.process.run("svn commit --non-interactive -m 'Add external'", 1206 cwd=os.path.join(self.tempdir, 'svnfetch_co', 'trunk')) 1207 1208 self.src_dir = src_dir 1209 self.repo_dir = repo_dir 1210 1211 @skipIfNoSvn() 1212 def tearDown(self): 1213 os.chdir(self.origdir) 1214 if os.environ.get("BB_TMPDIR_NOCLEAN") == "yes": 1215 print("Not cleaning up %s. Please remove manually." % self.tempdir) 1216 else: 1217 bb.utils.prunedir(self.tempdir) 1218 1219 @skipIfNoSvn() 1220 @skipIfNoNetwork() 1221 def test_noexternal_svn(self): 1222 # Always match the rev count from setUp (currently rev 2) 1223 url = "svn://%s;module=trunk;protocol=file;rev=2" % self.repo_url.replace('file://', '') 1224 fetcher = bb.fetch.Fetch([url], self.d) 1225 fetcher.download() 1226 os.chdir(os.path.dirname(self.unpackdir)) 1227 fetcher.unpack(self.unpackdir) 1228 1229 self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'trunk')), msg="Missing trunk") 1230 self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'trunk', 'README.md')), msg="Missing contents") 1231 self.assertFalse(os.path.exists(os.path.join(self.unpackdir, 'trunk/bitbake/trunk')), msg="External dir should NOT exist") 1232 self.assertFalse(os.path.exists(os.path.join(self.unpackdir, 'trunk/bitbake/trunk', 'README')), msg="External README should NOT exit") 1233 1234 @skipIfNoSvn() 1235 def test_external_svn(self): 1236 # Always match the rev count from setUp (currently rev 2) 1237 url = "svn://%s;module=trunk;protocol=file;externals=allowed;rev=2" % self.repo_url.replace('file://', '') 1238 fetcher = bb.fetch.Fetch([url], self.d) 1239 fetcher.download() 1240 os.chdir(os.path.dirname(self.unpackdir)) 1241 fetcher.unpack(self.unpackdir) 1242 1243 self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'trunk')), msg="Missing trunk") 1244 self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'trunk', 'README.md')), msg="Missing contents") 1245 self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'trunk/bitbake/trunk')), msg="External dir should exist") 1246 self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'trunk/bitbake/trunk', 'README')), msg="External README should exit") 1247 1248class TrustedNetworksTest(FetcherTest): 1249 def test_trusted_network(self): 1250 # Ensure trusted_network returns False when the host IS in the list. 1251 url = "git://Someserver.org/foo;rev=1;branch=master" 1252 self.d.setVar("BB_ALLOWED_NETWORKS", "server1.org someserver.org server2.org server3.org") 1253 self.assertTrue(bb.fetch.trusted_network(self.d, url)) 1254 1255 def test_wild_trusted_network(self): 1256 # Ensure trusted_network returns true when the *.host IS in the list. 1257 url = "git://Someserver.org/foo;rev=1;branch=master" 1258 self.d.setVar("BB_ALLOWED_NETWORKS", "server1.org *.someserver.org server2.org server3.org") 1259 self.assertTrue(bb.fetch.trusted_network(self.d, url)) 1260 1261 def test_prefix_wild_trusted_network(self): 1262 # Ensure trusted_network returns true when the prefix matches *.host. 1263 url = "git://git.Someserver.org/foo;rev=1;branch=master" 1264 self.d.setVar("BB_ALLOWED_NETWORKS", "server1.org *.someserver.org server2.org server3.org") 1265 self.assertTrue(bb.fetch.trusted_network(self.d, url)) 1266 1267 def test_two_prefix_wild_trusted_network(self): 1268 # Ensure trusted_network returns true when the prefix matches *.host. 1269 url = "git://something.git.Someserver.org/foo;rev=1;branch=master" 1270 self.d.setVar("BB_ALLOWED_NETWORKS", "server1.org *.someserver.org server2.org server3.org") 1271 self.assertTrue(bb.fetch.trusted_network(self.d, url)) 1272 1273 def test_port_trusted_network(self): 1274 # Ensure trusted_network returns True, even if the url specifies a port. 1275 url = "git://someserver.org:8080/foo;rev=1;branch=master" 1276 self.d.setVar("BB_ALLOWED_NETWORKS", "someserver.org") 1277 self.assertTrue(bb.fetch.trusted_network(self.d, url)) 1278 1279 def test_untrusted_network(self): 1280 # Ensure trusted_network returns False when the host is NOT in the list. 1281 url = "git://someserver.org/foo;rev=1;branch=master" 1282 self.d.setVar("BB_ALLOWED_NETWORKS", "server1.org server2.org server3.org") 1283 self.assertFalse(bb.fetch.trusted_network(self.d, url)) 1284 1285 def test_wild_untrusted_network(self): 1286 # Ensure trusted_network returns False when the host is NOT in the list. 1287 url = "git://*.someserver.org/foo;rev=1;branch=master" 1288 self.d.setVar("BB_ALLOWED_NETWORKS", "server1.org server2.org server3.org") 1289 self.assertFalse(bb.fetch.trusted_network(self.d, url)) 1290 1291class URLHandle(unittest.TestCase): 1292 1293 datatable = { 1294 "http://www.google.com/index.html" : ('http', 'www.google.com', '/index.html', '', '', {}), 1295 "cvs://anoncvs@cvs.handhelds.org/cvs;module=familiar/dist/ipkg" : ('cvs', 'cvs.handhelds.org', '/cvs', 'anoncvs', '', {'module': 'familiar/dist/ipkg'}), 1296 "cvs://anoncvs:anonymous@cvs.handhelds.org/cvs;tag=V0-99-81;module=familiar/dist/ipkg" : ('cvs', 'cvs.handhelds.org', '/cvs', 'anoncvs', 'anonymous', collections.OrderedDict([('tag', 'V0-99-81'), ('module', 'familiar/dist/ipkg')])), 1297 "git://git.openembedded.org/bitbake;branch=@foo" : ('git', 'git.openembedded.org', '/bitbake', '', '', {'branch': '@foo'}), 1298 "file://somelocation;someparam=1": ('file', '', 'somelocation', '', '', {'someparam': '1'}), 1299 } 1300 # we require a pathname to encodeurl but users can still pass such urls to 1301 # decodeurl and we need to handle them 1302 decodedata = datatable.copy() 1303 decodedata.update({ 1304 "http://somesite.net;someparam=1": ('http', 'somesite.net', '/', '', '', {'someparam': '1'}), 1305 }) 1306 1307 def test_decodeurl(self): 1308 for k, v in self.decodedata.items(): 1309 result = bb.fetch.decodeurl(k) 1310 self.assertEqual(result, v) 1311 1312 def test_encodeurl(self): 1313 for k, v in self.datatable.items(): 1314 result = bb.fetch.encodeurl(v) 1315 self.assertEqual(result, k) 1316 1317class FetchLatestVersionTest(FetcherTest): 1318 1319 test_git_uris = { 1320 # version pattern "X.Y.Z" 1321 ("mx-1.0", "git://github.com/clutter-project/mx.git;branch=mx-1.4;protocol=https", "9b1db6b8060bd00b121a692f942404a24ae2960f", "") 1322 : "1.99.4", 1323 # version pattern "vX.Y" 1324 # mirror of git.infradead.org since network issues interfered with testing 1325 ("mtd-utils", "git://git.yoctoproject.org/mtd-utils.git;branch=master", "ca39eb1d98e736109c64ff9c1aa2a6ecca222d8f", "") 1326 : "1.5.0", 1327 # version pattern "pkg_name-X.Y" 1328 # mirror of git://anongit.freedesktop.org/git/xorg/proto/presentproto since network issues interfered with testing 1329 ("presentproto", "git://git.yoctoproject.org/bbfetchtests-presentproto;branch=master", "24f3a56e541b0a9e6c6ee76081f441221a120ef9", "") 1330 : "1.0", 1331 # version pattern "pkg_name-vX.Y.Z" 1332 ("dtc", "git://git.yoctoproject.org/bbfetchtests-dtc.git;branch=master", "65cc4d2748a2c2e6f27f1cf39e07a5dbabd80ebf", "") 1333 : "1.4.0", 1334 # combination version pattern 1335 ("sysprof", "git://gitlab.gnome.org/GNOME/sysprof.git;protocol=https;branch=master", "cd44ee6644c3641507fb53b8a2a69137f2971219", "") 1336 : "1.2.0", 1337 ("u-boot-mkimage", "git://git.denx.de/u-boot.git;branch=master;protocol=git", "62c175fbb8a0f9a926c88294ea9f7e88eb898f6c", "") 1338 : "2014.01", 1339 # version pattern "yyyymmdd" 1340 ("mobile-broadband-provider-info", "git://gitlab.gnome.org/GNOME/mobile-broadband-provider-info.git;protocol=https;branch=master", "4ed19e11c2975105b71b956440acdb25d46a347d", "") 1341 : "20120614", 1342 # packages with a valid UPSTREAM_CHECK_GITTAGREGEX 1343 # mirror of git://anongit.freedesktop.org/xorg/driver/xf86-video-omap since network issues interfered with testing 1344 ("xf86-video-omap", "git://git.yoctoproject.org/bbfetchtests-xf86-video-omap;branch=master", "ae0394e687f1a77e966cf72f895da91840dffb8f", r"(?P<pver>(\d+\.(\d\.?)*))") 1345 : "0.4.3", 1346 ("build-appliance-image", "git://git.yoctoproject.org/poky;branch=master", "b37dd451a52622d5b570183a81583cc34c2ff555", r"(?P<pver>(([0-9][\.|_]?)+[0-9]))") 1347 : "11.0.0", 1348 ("chkconfig-alternatives-native", "git://github.com/kergoth/chkconfig;branch=sysroot;protocol=https", "cd437ecbd8986c894442f8fce1e0061e20f04dee", r"chkconfig\-(?P<pver>((\d+[\.\-_]*)+))") 1349 : "1.3.59", 1350 ("remake", "git://github.com/rocky/remake.git;protocol=https;branch=master", "f05508e521987c8494c92d9c2871aec46307d51d", r"(?P<pver>(\d+\.(\d+\.)*\d*(\+dbg\d+(\.\d+)*)*))") 1351 : "3.82+dbg0.9", 1352 } 1353 1354 test_wget_uris = { 1355 # 1356 # packages with versions inside directory name 1357 # 1358 # http://kernel.org/pub/linux/utils/util-linux/v2.23/util-linux-2.24.2.tar.bz2 1359 ("util-linux", "/pub/linux/utils/util-linux/v2.23/util-linux-2.24.2.tar.bz2", "", "") 1360 : "2.24.2", 1361 # http://www.abisource.com/downloads/enchant/1.6.0/enchant-1.6.0.tar.gz 1362 ("enchant", "/downloads/enchant/1.6.0/enchant-1.6.0.tar.gz", "", "") 1363 : "1.6.0", 1364 # http://www.cmake.org/files/v2.8/cmake-2.8.12.1.tar.gz 1365 ("cmake", "/files/v2.8/cmake-2.8.12.1.tar.gz", "", "") 1366 : "2.8.12.1", 1367 # 1368 # packages with versions only in current directory 1369 # 1370 # https://downloads.yoctoproject.org/releases/eglibc/eglibc-2.18-svnr23787.tar.bz2 1371 ("eglic", "/releases/eglibc/eglibc-2.18-svnr23787.tar.bz2", "", "") 1372 : "2.19", 1373 # https://downloads.yoctoproject.org/releases/gnu-config/gnu-config-20120814.tar.bz2 1374 ("gnu-config", "/releases/gnu-config/gnu-config-20120814.tar.bz2", "", "") 1375 : "20120814", 1376 # 1377 # packages with "99" in the name of possible version 1378 # 1379 # http://freedesktop.org/software/pulseaudio/releases/pulseaudio-4.0.tar.xz 1380 ("pulseaudio", "/software/pulseaudio/releases/pulseaudio-4.0.tar.xz", "", "") 1381 : "5.0", 1382 # http://xorg.freedesktop.org/releases/individual/xserver/xorg-server-1.15.1.tar.bz2 1383 ("xserver-xorg", "/releases/individual/xserver/xorg-server-1.15.1.tar.bz2", "", "") 1384 : "1.15.1", 1385 # 1386 # packages with valid UPSTREAM_CHECK_URI and UPSTREAM_CHECK_REGEX 1387 # 1388 # http://www.cups.org/software/1.7.2/cups-1.7.2-source.tar.bz2 1389 # https://github.com/apple/cups/releases 1390 ("cups", "/software/1.7.2/cups-1.7.2-source.tar.bz2", "/apple/cups/releases", r"(?P<name>cups\-)(?P<pver>((\d+[\.\-_]*)+))\-source\.tar\.gz") 1391 : "2.0.0", 1392 # http://download.oracle.com/berkeley-db/db-5.3.21.tar.gz 1393 # http://ftp.debian.org/debian/pool/main/d/db5.3/ 1394 ("db", "/berkeley-db/db-5.3.21.tar.gz", "/debian/pool/main/d/db5.3/", r"(?P<name>db5\.3_)(?P<pver>\d+(\.\d+)+).+\.orig\.tar\.xz") 1395 : "5.3.10", 1396 # 1397 # packages where the tarball compression changed in the new version 1398 # 1399 # http://ftp.debian.org/debian/pool/main/m/minicom/minicom_2.7.1.orig.tar.gz 1400 ("minicom", "/debian/pool/main/m/minicom/minicom_2.7.1.orig.tar.gz", "", "") 1401 : "2.8", 1402 } 1403 1404 @skipIfNoNetwork() 1405 def test_git_latest_versionstring(self): 1406 for k, v in self.test_git_uris.items(): 1407 self.d.setVar("PN", k[0]) 1408 self.d.setVar("SRCREV", k[2]) 1409 self.d.setVar("UPSTREAM_CHECK_GITTAGREGEX", k[3]) 1410 ud = bb.fetch2.FetchData(k[1], self.d) 1411 pupver= ud.method.latest_versionstring(ud, self.d) 1412 verstring = pupver[0] 1413 self.assertTrue(verstring, msg="Could not find upstream version for %s" % k[0]) 1414 r = bb.utils.vercmp_string(v, verstring) 1415 self.assertTrue(r == -1 or r == 0, msg="Package %s, version: %s <= %s" % (k[0], v, verstring)) 1416 1417 def test_wget_latest_versionstring(self): 1418 testdata = os.path.dirname(os.path.abspath(__file__)) + "/fetch-testdata" 1419 server = HTTPService(testdata) 1420 server.start() 1421 port = server.port 1422 try: 1423 for k, v in self.test_wget_uris.items(): 1424 self.d.setVar("PN", k[0]) 1425 checkuri = "" 1426 if k[2]: 1427 checkuri = "http://localhost:%s/" % port + k[2] 1428 self.d.setVar("UPSTREAM_CHECK_URI", checkuri) 1429 self.d.setVar("UPSTREAM_CHECK_REGEX", k[3]) 1430 url = "http://localhost:%s/" % port + k[1] 1431 ud = bb.fetch2.FetchData(url, self.d) 1432 pupver = ud.method.latest_versionstring(ud, self.d) 1433 verstring = pupver[0] 1434 self.assertTrue(verstring, msg="Could not find upstream version for %s" % k[0]) 1435 r = bb.utils.vercmp_string(v, verstring) 1436 self.assertTrue(r == -1 or r == 0, msg="Package %s, version: %s <= %s" % (k[0], v, verstring)) 1437 finally: 1438 server.stop() 1439 1440 1441class FetchCheckStatusTest(FetcherTest): 1442 test_wget_uris = ["https://downloads.yoctoproject.org/releases/sato/sato-engine-0.1.tar.gz", 1443 "https://downloads.yoctoproject.org/releases/sato/sato-engine-0.2.tar.gz", 1444 "https://downloads.yoctoproject.org/releases/sato/sato-engine-0.3.tar.gz", 1445 "https://yoctoproject.org/", 1446 "https://docs.yoctoproject.org", 1447 "https://downloads.yoctoproject.org/releases/opkg/opkg-0.1.7.tar.gz", 1448 "https://downloads.yoctoproject.org/releases/opkg/opkg-0.3.0.tar.gz", 1449 "ftp://sourceware.org/pub/libffi/libffi-1.20.tar.gz", 1450 # GitHub releases are hosted on Amazon S3, which doesn't support HEAD 1451 "https://github.com/kergoth/tslib/releases/download/1.1/tslib-1.1.tar.xz" 1452 ] 1453 1454 @skipIfNoNetwork() 1455 def test_wget_checkstatus(self): 1456 fetch = bb.fetch2.Fetch(self.test_wget_uris, self.d) 1457 for u in self.test_wget_uris: 1458 with self.subTest(url=u): 1459 ud = fetch.ud[u] 1460 m = ud.method 1461 ret = m.checkstatus(fetch, ud, self.d) 1462 self.assertTrue(ret, msg="URI %s, can't check status" % (u)) 1463 1464 @skipIfNoNetwork() 1465 def test_wget_checkstatus_connection_cache(self): 1466 from bb.fetch2 import FetchConnectionCache 1467 1468 connection_cache = FetchConnectionCache() 1469 fetch = bb.fetch2.Fetch(self.test_wget_uris, self.d, 1470 connection_cache = connection_cache) 1471 1472 for u in self.test_wget_uris: 1473 with self.subTest(url=u): 1474 ud = fetch.ud[u] 1475 m = ud.method 1476 ret = m.checkstatus(fetch, ud, self.d) 1477 self.assertTrue(ret, msg="URI %s, can't check status" % (u)) 1478 1479 connection_cache.close_connections() 1480 1481 1482class GitMakeShallowTest(FetcherTest): 1483 def setUp(self): 1484 FetcherTest.setUp(self) 1485 self.gitdir = os.path.join(self.tempdir, 'gitshallow') 1486 bb.utils.mkdirhier(self.gitdir) 1487 self.git_init() 1488 1489 def assertRefs(self, expected_refs): 1490 actual_refs = self.git(['for-each-ref', '--format=%(refname)']).splitlines() 1491 full_expected = self.git(['rev-parse', '--symbolic-full-name'] + expected_refs).splitlines() 1492 self.assertEqual(sorted(full_expected), sorted(actual_refs)) 1493 1494 def assertRevCount(self, expected_count, args=None): 1495 if args is None: 1496 args = ['HEAD'] 1497 revs = self.git(['rev-list'] + args) 1498 actual_count = len(revs.splitlines()) 1499 self.assertEqual(expected_count, actual_count, msg='Object count `%d` is not the expected `%d`' % (actual_count, expected_count)) 1500 1501 def make_shallow(self, args=None): 1502 if args is None: 1503 args = ['HEAD'] 1504 return bb.process.run([bb.fetch2.git.Git.make_shallow_path] + args, cwd=self.gitdir) 1505 1506 def add_empty_file(self, path, msg=None): 1507 if msg is None: 1508 msg = path 1509 open(os.path.join(self.gitdir, path), 'w').close() 1510 self.git(['add', path]) 1511 self.git(['commit', '-m', msg, path]) 1512 1513 def test_make_shallow_single_branch_no_merge(self): 1514 self.add_empty_file('a') 1515 self.add_empty_file('b') 1516 self.assertRevCount(2) 1517 self.make_shallow() 1518 self.assertRevCount(1) 1519 1520 def test_make_shallow_single_branch_one_merge(self): 1521 self.add_empty_file('a') 1522 self.add_empty_file('b') 1523 self.git('checkout -b a_branch') 1524 self.add_empty_file('c') 1525 self.git('checkout master') 1526 self.add_empty_file('d') 1527 self.git('merge --no-ff --no-edit a_branch') 1528 self.git('branch -d a_branch') 1529 self.add_empty_file('e') 1530 self.assertRevCount(6) 1531 self.make_shallow(['HEAD~2']) 1532 self.assertRevCount(5) 1533 1534 def test_make_shallow_at_merge(self): 1535 self.add_empty_file('a') 1536 self.git('checkout -b a_branch') 1537 self.add_empty_file('b') 1538 self.git('checkout master') 1539 self.git('merge --no-ff --no-edit a_branch') 1540 self.git('branch -d a_branch') 1541 self.assertRevCount(3) 1542 self.make_shallow() 1543 self.assertRevCount(1) 1544 1545 def test_make_shallow_annotated_tag(self): 1546 self.add_empty_file('a') 1547 self.add_empty_file('b') 1548 self.git('tag -a -m a_tag a_tag') 1549 self.assertRevCount(2) 1550 self.make_shallow(['a_tag']) 1551 self.assertRevCount(1) 1552 1553 def test_make_shallow_multi_ref(self): 1554 self.add_empty_file('a') 1555 self.add_empty_file('b') 1556 self.git('checkout -b a_branch') 1557 self.add_empty_file('c') 1558 self.git('checkout master') 1559 self.add_empty_file('d') 1560 self.git('checkout -b a_branch_2') 1561 self.add_empty_file('a_tag') 1562 self.git('tag a_tag') 1563 self.git('checkout master') 1564 self.git('branch -D a_branch_2') 1565 self.add_empty_file('e') 1566 self.assertRevCount(6, ['--all']) 1567 self.make_shallow() 1568 self.assertRevCount(5, ['--all']) 1569 1570 def test_make_shallow_multi_ref_trim(self): 1571 self.add_empty_file('a') 1572 self.git('checkout -b a_branch') 1573 self.add_empty_file('c') 1574 self.git('checkout master') 1575 self.assertRevCount(1) 1576 self.assertRevCount(2, ['--all']) 1577 self.assertRefs(['master', 'a_branch']) 1578 self.make_shallow(['-r', 'master', 'HEAD']) 1579 self.assertRevCount(1, ['--all']) 1580 self.assertRefs(['master']) 1581 1582 def test_make_shallow_noop(self): 1583 self.add_empty_file('a') 1584 self.assertRevCount(1) 1585 self.make_shallow() 1586 self.assertRevCount(1) 1587 1588 @skipIfNoNetwork() 1589 def test_make_shallow_bitbake(self): 1590 self.git('remote add origin https://github.com/openembedded/bitbake') 1591 self.git('fetch --tags origin') 1592 orig_revs = len(self.git('rev-list --all').splitlines()) 1593 self.make_shallow(['refs/tags/1.10.0']) 1594 self.assertRevCount(orig_revs - 1746, ['--all']) 1595 1596class GitShallowTest(FetcherTest): 1597 def setUp(self): 1598 FetcherTest.setUp(self) 1599 self.gitdir = os.path.join(self.tempdir, 'git') 1600 self.srcdir = os.path.join(self.tempdir, 'gitsource') 1601 1602 bb.utils.mkdirhier(self.srcdir) 1603 self.git_init(cwd=self.srcdir) 1604 self.d.setVar('WORKDIR', self.tempdir) 1605 self.d.setVar('S', self.gitdir) 1606 self.d.delVar('PREMIRRORS') 1607 self.d.delVar('MIRRORS') 1608 1609 uri = 'git://%s;protocol=file;subdir=${S};branch=master' % self.srcdir 1610 self.d.setVar('SRC_URI', uri) 1611 self.d.setVar('SRCREV', '${AUTOREV}') 1612 self.d.setVar('AUTOREV', '${@bb.fetch2.get_autorev(d)}') 1613 1614 self.d.setVar('BB_GIT_SHALLOW', '1') 1615 self.d.setVar('BB_GENERATE_MIRROR_TARBALLS', '0') 1616 self.d.setVar('BB_GENERATE_SHALLOW_TARBALLS', '1') 1617 self.d.setVar("__BBSEENSRCREV", "1") 1618 1619 def assertRefs(self, expected_refs, cwd=None): 1620 if cwd is None: 1621 cwd = self.gitdir 1622 actual_refs = self.git(['for-each-ref', '--format=%(refname)'], cwd=cwd).splitlines() 1623 full_expected = self.git(['rev-parse', '--symbolic-full-name'] + expected_refs, cwd=cwd).splitlines() 1624 self.assertEqual(sorted(set(full_expected)), sorted(set(actual_refs))) 1625 1626 def assertRevCount(self, expected_count, args=None, cwd=None): 1627 if args is None: 1628 args = ['HEAD'] 1629 if cwd is None: 1630 cwd = self.gitdir 1631 revs = self.git(['rev-list'] + args, cwd=cwd) 1632 actual_count = len(revs.splitlines()) 1633 self.assertEqual(expected_count, actual_count, msg='Object count `%d` is not the expected `%d`' % (actual_count, expected_count)) 1634 1635 def add_empty_file(self, path, cwd=None, msg=None): 1636 if msg is None: 1637 msg = path 1638 if cwd is None: 1639 cwd = self.srcdir 1640 open(os.path.join(cwd, path), 'w').close() 1641 self.git(['add', path], cwd) 1642 self.git(['commit', '-m', msg, path], cwd) 1643 1644 def fetch(self, uri=None): 1645 if uri is None: 1646 uris = self.d.getVar('SRC_URI').split() 1647 uri = uris[0] 1648 d = self.d 1649 else: 1650 d = self.d.createCopy() 1651 d.setVar('SRC_URI', uri) 1652 uri = d.expand(uri) 1653 uris = [uri] 1654 1655 fetcher = bb.fetch2.Fetch(uris, d) 1656 fetcher.download() 1657 ud = fetcher.ud[uri] 1658 return fetcher, ud 1659 1660 def fetch_and_unpack(self, uri=None): 1661 fetcher, ud = self.fetch(uri) 1662 fetcher.unpack(self.d.getVar('WORKDIR')) 1663 assert os.path.exists(self.d.getVar('S')) 1664 return fetcher, ud 1665 1666 def fetch_shallow(self, uri=None, disabled=False, keepclone=False): 1667 """Fetch a uri, generating a shallow tarball, then unpack using it""" 1668 fetcher, ud = self.fetch_and_unpack(uri) 1669 assert os.path.exists(ud.clonedir), 'Git clone in DLDIR (%s) does not exist for uri %s' % (ud.clonedir, uri) 1670 1671 # Confirm that the unpacked repo is unshallow 1672 if not disabled: 1673 assert os.path.exists(os.path.join(self.dldir, ud.mirrortarballs[0])) 1674 1675 # fetch and unpack, from the shallow tarball 1676 bb.utils.remove(self.gitdir, recurse=True) 1677 bb.process.run('chmod u+w -R "%s"' % ud.clonedir) 1678 bb.utils.remove(ud.clonedir, recurse=True) 1679 bb.utils.remove(ud.clonedir.replace('gitsource', 'gitsubmodule'), recurse=True) 1680 1681 # confirm that the unpacked repo is used when no git clone or git 1682 # mirror tarball is available 1683 fetcher, ud = self.fetch_and_unpack(uri) 1684 if not disabled: 1685 assert os.path.exists(os.path.join(self.gitdir, '.git', 'shallow')), 'Unpacked git repository at %s is not shallow' % self.gitdir 1686 else: 1687 assert not os.path.exists(os.path.join(self.gitdir, '.git', 'shallow')), 'Unpacked git repository at %s is shallow' % self.gitdir 1688 return fetcher, ud 1689 1690 def test_shallow_disabled(self): 1691 self.add_empty_file('a') 1692 self.add_empty_file('b') 1693 self.assertRevCount(2, cwd=self.srcdir) 1694 1695 self.d.setVar('BB_GIT_SHALLOW', '0') 1696 self.fetch_shallow(disabled=True) 1697 self.assertRevCount(2) 1698 1699 def test_shallow_nobranch(self): 1700 self.add_empty_file('a') 1701 self.add_empty_file('b') 1702 self.assertRevCount(2, cwd=self.srcdir) 1703 1704 srcrev = self.git('rev-parse HEAD', cwd=self.srcdir).strip() 1705 self.d.setVar('SRCREV', srcrev) 1706 uri = self.d.getVar('SRC_URI').split()[0] 1707 uri = '%s;nobranch=1;bare=1' % uri 1708 1709 self.fetch_shallow(uri) 1710 self.assertRevCount(1) 1711 1712 # shallow refs are used to ensure the srcrev sticks around when we 1713 # have no other branches referencing it 1714 self.assertRefs(['refs/shallow/default']) 1715 1716 def test_shallow_default_depth_1(self): 1717 # Create initial git repo 1718 self.add_empty_file('a') 1719 self.add_empty_file('b') 1720 self.assertRevCount(2, cwd=self.srcdir) 1721 1722 self.fetch_shallow() 1723 self.assertRevCount(1) 1724 1725 def test_shallow_depth_0_disables(self): 1726 self.add_empty_file('a') 1727 self.add_empty_file('b') 1728 self.assertRevCount(2, cwd=self.srcdir) 1729 1730 self.d.setVar('BB_GIT_SHALLOW_DEPTH', '0') 1731 self.fetch_shallow(disabled=True) 1732 self.assertRevCount(2) 1733 1734 def test_shallow_depth_default_override(self): 1735 self.add_empty_file('a') 1736 self.add_empty_file('b') 1737 self.assertRevCount(2, cwd=self.srcdir) 1738 1739 self.d.setVar('BB_GIT_SHALLOW_DEPTH', '2') 1740 self.d.setVar('BB_GIT_SHALLOW_DEPTH_default', '1') 1741 self.fetch_shallow() 1742 self.assertRevCount(1) 1743 1744 def test_shallow_depth_default_override_disable(self): 1745 self.add_empty_file('a') 1746 self.add_empty_file('b') 1747 self.add_empty_file('c') 1748 self.assertRevCount(3, cwd=self.srcdir) 1749 1750 self.d.setVar('BB_GIT_SHALLOW_DEPTH', '0') 1751 self.d.setVar('BB_GIT_SHALLOW_DEPTH_default', '2') 1752 self.fetch_shallow() 1753 self.assertRevCount(2) 1754 1755 def test_current_shallow_out_of_date_clone(self): 1756 # Create initial git repo 1757 self.add_empty_file('a') 1758 self.add_empty_file('b') 1759 self.add_empty_file('c') 1760 self.assertRevCount(3, cwd=self.srcdir) 1761 1762 # Clone and generate mirror tarball 1763 fetcher, ud = self.fetch() 1764 1765 # Ensure we have a current mirror tarball, but an out of date clone 1766 self.git('update-ref refs/heads/master refs/heads/master~1', cwd=ud.clonedir) 1767 self.assertRevCount(2, cwd=ud.clonedir) 1768 1769 # Fetch and unpack, from the current tarball, not the out of date clone 1770 bb.utils.remove(self.gitdir, recurse=True) 1771 fetcher, ud = self.fetch() 1772 fetcher.unpack(self.d.getVar('WORKDIR')) 1773 self.assertRevCount(1) 1774 1775 def test_shallow_single_branch_no_merge(self): 1776 self.add_empty_file('a') 1777 self.add_empty_file('b') 1778 self.assertRevCount(2, cwd=self.srcdir) 1779 1780 self.fetch_shallow() 1781 self.assertRevCount(1) 1782 assert os.path.exists(os.path.join(self.gitdir, 'a')) 1783 assert os.path.exists(os.path.join(self.gitdir, 'b')) 1784 1785 def test_shallow_no_dangling(self): 1786 self.add_empty_file('a') 1787 self.add_empty_file('b') 1788 self.assertRevCount(2, cwd=self.srcdir) 1789 1790 self.fetch_shallow() 1791 self.assertRevCount(1) 1792 assert not self.git('fsck --dangling') 1793 1794 def test_shallow_srcrev_branch_truncation(self): 1795 self.add_empty_file('a') 1796 self.add_empty_file('b') 1797 b_commit = self.git('rev-parse HEAD', cwd=self.srcdir).rstrip() 1798 self.add_empty_file('c') 1799 self.assertRevCount(3, cwd=self.srcdir) 1800 1801 self.d.setVar('SRCREV', b_commit) 1802 self.fetch_shallow() 1803 1804 # The 'c' commit was removed entirely, and 'a' was removed from history 1805 self.assertRevCount(1, ['--all']) 1806 self.assertEqual(self.git('rev-parse HEAD').strip(), b_commit) 1807 assert os.path.exists(os.path.join(self.gitdir, 'a')) 1808 assert os.path.exists(os.path.join(self.gitdir, 'b')) 1809 assert not os.path.exists(os.path.join(self.gitdir, 'c')) 1810 1811 def test_shallow_ref_pruning(self): 1812 self.add_empty_file('a') 1813 self.add_empty_file('b') 1814 self.git('branch a_branch', cwd=self.srcdir) 1815 self.assertRefs(['master', 'a_branch'], cwd=self.srcdir) 1816 self.assertRevCount(2, cwd=self.srcdir) 1817 1818 self.fetch_shallow() 1819 1820 self.assertRefs(['master', 'origin/master']) 1821 self.assertRevCount(1) 1822 1823 def test_shallow_submodules(self): 1824 self.add_empty_file('a') 1825 self.add_empty_file('b') 1826 1827 smdir = os.path.join(self.tempdir, 'gitsubmodule') 1828 bb.utils.mkdirhier(smdir) 1829 self.git_init(cwd=smdir) 1830 # Make this look like it was cloned from a remote... 1831 self.git('config --add remote.origin.url "%s"' % smdir, cwd=smdir) 1832 self.git('config --add remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*"', cwd=smdir) 1833 self.add_empty_file('asub', cwd=smdir) 1834 self.add_empty_file('bsub', cwd=smdir) 1835 1836 self.git('submodule init', cwd=self.srcdir) 1837 self.git('-c protocol.file.allow=always submodule add file://%s' % smdir, cwd=self.srcdir) 1838 self.git('submodule update', cwd=self.srcdir) 1839 self.git('commit -m submodule -a', cwd=self.srcdir) 1840 1841 uri = 'gitsm://%s;protocol=file;subdir=${S};branch=master' % self.srcdir 1842 fetcher, ud = self.fetch_shallow(uri) 1843 1844 # Verify the main repository is shallow 1845 self.assertRevCount(1) 1846 1847 # Verify the gitsubmodule directory is present 1848 assert os.listdir(os.path.join(self.gitdir, 'gitsubmodule')) 1849 1850 # Verify the submodule is also shallow 1851 self.assertRevCount(1, cwd=os.path.join(self.gitdir, 'gitsubmodule')) 1852 1853 def test_shallow_submodule_mirrors(self): 1854 self.add_empty_file('a') 1855 self.add_empty_file('b') 1856 1857 smdir = os.path.join(self.tempdir, 'gitsubmodule') 1858 bb.utils.mkdirhier(smdir) 1859 self.git_init(cwd=smdir) 1860 # Make this look like it was cloned from a remote... 1861 self.git('config --add remote.origin.url "%s"' % smdir, cwd=smdir) 1862 self.git('config --add remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*"', cwd=smdir) 1863 self.add_empty_file('asub', cwd=smdir) 1864 self.add_empty_file('bsub', cwd=smdir) 1865 1866 self.git('submodule init', cwd=self.srcdir) 1867 self.git('-c protocol.file.allow=always submodule add file://%s' % smdir, cwd=self.srcdir) 1868 self.git('submodule update', cwd=self.srcdir) 1869 self.git('commit -m submodule -a', cwd=self.srcdir) 1870 1871 uri = 'gitsm://%s;protocol=file;subdir=${S}' % self.srcdir 1872 1873 # Fetch once to generate the shallow tarball 1874 fetcher, ud = self.fetch(uri) 1875 1876 # Set up the mirror 1877 mirrordir = os.path.join(self.tempdir, 'mirror') 1878 bb.utils.rename(self.dldir, mirrordir) 1879 self.d.setVar('PREMIRRORS', 'gitsm://.*/.* file://%s/' % mirrordir) 1880 1881 # Fetch from the mirror 1882 bb.utils.remove(self.dldir, recurse=True) 1883 bb.utils.remove(self.gitdir, recurse=True) 1884 self.fetch_and_unpack(uri) 1885 1886 # Verify the main repository is shallow 1887 self.assertRevCount(1) 1888 1889 # Verify the gitsubmodule directory is present 1890 assert os.listdir(os.path.join(self.gitdir, 'gitsubmodule')) 1891 1892 # Verify the submodule is also shallow 1893 self.assertRevCount(1, cwd=os.path.join(self.gitdir, 'gitsubmodule')) 1894 1895 if any(os.path.exists(os.path.join(p, 'git-annex')) for p in os.environ.get('PATH').split(':')): 1896 def test_shallow_annex(self): 1897 self.add_empty_file('a') 1898 self.add_empty_file('b') 1899 self.git('annex init', cwd=self.srcdir) 1900 open(os.path.join(self.srcdir, 'c'), 'w').close() 1901 self.git('annex add c', cwd=self.srcdir) 1902 self.git('commit --author "Foo Bar <foo@bar>" -m annex-c -a', cwd=self.srcdir) 1903 bb.process.run('chmod u+w -R %s' % self.srcdir) 1904 1905 uri = 'gitannex://%s;protocol=file;subdir=${S};branch=master' % self.srcdir 1906 fetcher, ud = self.fetch_shallow(uri) 1907 1908 self.assertRevCount(1) 1909 assert './.git/annex/' in bb.process.run('tar -tzf %s' % os.path.join(self.dldir, ud.mirrortarballs[0]))[0] 1910 assert os.path.exists(os.path.join(self.gitdir, 'c')) 1911 1912 def test_shallow_multi_one_uri(self): 1913 # Create initial git repo 1914 self.add_empty_file('a') 1915 self.add_empty_file('b') 1916 self.git('checkout -b a_branch', cwd=self.srcdir) 1917 self.add_empty_file('c') 1918 self.add_empty_file('d') 1919 self.git('checkout master', cwd=self.srcdir) 1920 self.git('tag v0.0 a_branch', cwd=self.srcdir) 1921 self.add_empty_file('e') 1922 self.git('merge --no-ff --no-edit a_branch', cwd=self.srcdir) 1923 self.add_empty_file('f') 1924 self.assertRevCount(7, cwd=self.srcdir) 1925 1926 uri = self.d.getVar('SRC_URI').split()[0] 1927 uri = '%s;branch=master,a_branch;name=master,a_branch' % uri 1928 1929 self.d.setVar('BB_GIT_SHALLOW_DEPTH', '0') 1930 self.d.setVar('BB_GIT_SHALLOW_REVS', 'v0.0') 1931 self.d.setVar('SRCREV_master', '${AUTOREV}') 1932 self.d.setVar('SRCREV_a_branch', '${AUTOREV}') 1933 1934 self.fetch_shallow(uri) 1935 1936 self.assertRevCount(5) 1937 self.assertRefs(['master', 'origin/master', 'origin/a_branch']) 1938 1939 def test_shallow_multi_one_uri_depths(self): 1940 # Create initial git repo 1941 self.add_empty_file('a') 1942 self.add_empty_file('b') 1943 self.git('checkout -b a_branch', cwd=self.srcdir) 1944 self.add_empty_file('c') 1945 self.add_empty_file('d') 1946 self.git('checkout master', cwd=self.srcdir) 1947 self.add_empty_file('e') 1948 self.git('merge --no-ff --no-edit a_branch', cwd=self.srcdir) 1949 self.add_empty_file('f') 1950 self.assertRevCount(7, cwd=self.srcdir) 1951 1952 uri = self.d.getVar('SRC_URI').split()[0] 1953 uri = '%s;branch=master,a_branch;name=master,a_branch' % uri 1954 1955 self.d.setVar('BB_GIT_SHALLOW_DEPTH', '0') 1956 self.d.setVar('BB_GIT_SHALLOW_DEPTH_master', '3') 1957 self.d.setVar('BB_GIT_SHALLOW_DEPTH_a_branch', '1') 1958 self.d.setVar('SRCREV_master', '${AUTOREV}') 1959 self.d.setVar('SRCREV_a_branch', '${AUTOREV}') 1960 1961 self.fetch_shallow(uri) 1962 1963 self.assertRevCount(4, ['--all']) 1964 self.assertRefs(['master', 'origin/master', 'origin/a_branch']) 1965 1966 def test_shallow_clone_preferred_over_shallow(self): 1967 self.add_empty_file('a') 1968 self.add_empty_file('b') 1969 1970 # Fetch once to generate the shallow tarball 1971 fetcher, ud = self.fetch() 1972 assert os.path.exists(os.path.join(self.dldir, ud.mirrortarballs[0])) 1973 1974 # Fetch and unpack with both the clonedir and shallow tarball available 1975 bb.utils.remove(self.gitdir, recurse=True) 1976 fetcher, ud = self.fetch_and_unpack() 1977 1978 # The unpacked tree should *not* be shallow 1979 self.assertRevCount(2) 1980 assert not os.path.exists(os.path.join(self.gitdir, '.git', 'shallow')) 1981 1982 def test_shallow_mirrors(self): 1983 self.add_empty_file('a') 1984 self.add_empty_file('b') 1985 1986 # Fetch once to generate the shallow tarball 1987 fetcher, ud = self.fetch() 1988 mirrortarball = ud.mirrortarballs[0] 1989 assert os.path.exists(os.path.join(self.dldir, mirrortarball)) 1990 1991 # Set up the mirror 1992 mirrordir = os.path.join(self.tempdir, 'mirror') 1993 bb.utils.mkdirhier(mirrordir) 1994 self.d.setVar('PREMIRRORS', 'git://.*/.* file://%s/' % mirrordir) 1995 1996 bb.utils.rename(os.path.join(self.dldir, mirrortarball), 1997 os.path.join(mirrordir, mirrortarball)) 1998 1999 # Fetch from the mirror 2000 bb.utils.remove(self.dldir, recurse=True) 2001 bb.utils.remove(self.gitdir, recurse=True) 2002 self.fetch_and_unpack() 2003 self.assertRevCount(1) 2004 2005 def test_shallow_invalid_depth(self): 2006 self.add_empty_file('a') 2007 self.add_empty_file('b') 2008 2009 self.d.setVar('BB_GIT_SHALLOW_DEPTH', '-12') 2010 with self.assertRaises(bb.fetch2.FetchError): 2011 self.fetch() 2012 2013 def test_shallow_invalid_depth_default(self): 2014 self.add_empty_file('a') 2015 self.add_empty_file('b') 2016 2017 self.d.setVar('BB_GIT_SHALLOW_DEPTH_default', '-12') 2018 with self.assertRaises(bb.fetch2.FetchError): 2019 self.fetch() 2020 2021 def test_shallow_extra_refs(self): 2022 self.add_empty_file('a') 2023 self.add_empty_file('b') 2024 self.git('branch a_branch', cwd=self.srcdir) 2025 self.assertRefs(['master', 'a_branch'], cwd=self.srcdir) 2026 self.assertRevCount(2, cwd=self.srcdir) 2027 2028 self.d.setVar('BB_GIT_SHALLOW_EXTRA_REFS', 'refs/heads/a_branch') 2029 self.fetch_shallow() 2030 2031 self.assertRefs(['master', 'origin/master', 'origin/a_branch']) 2032 self.assertRevCount(1) 2033 2034 def test_shallow_extra_refs_wildcard(self): 2035 self.add_empty_file('a') 2036 self.add_empty_file('b') 2037 self.git('branch a_branch', cwd=self.srcdir) 2038 self.git('tag v1.0', cwd=self.srcdir) 2039 self.assertRefs(['master', 'a_branch', 'v1.0'], cwd=self.srcdir) 2040 self.assertRevCount(2, cwd=self.srcdir) 2041 2042 self.d.setVar('BB_GIT_SHALLOW_EXTRA_REFS', 'refs/tags/*') 2043 self.fetch_shallow() 2044 2045 self.assertRefs(['master', 'origin/master', 'v1.0']) 2046 self.assertRevCount(1) 2047 2048 def test_shallow_missing_extra_refs(self): 2049 self.add_empty_file('a') 2050 self.add_empty_file('b') 2051 2052 self.d.setVar('BB_GIT_SHALLOW_EXTRA_REFS', 'refs/heads/foo') 2053 with self.assertRaises(bb.fetch2.FetchError): 2054 self.fetch() 2055 2056 def test_shallow_missing_extra_refs_wildcard(self): 2057 self.add_empty_file('a') 2058 self.add_empty_file('b') 2059 2060 self.d.setVar('BB_GIT_SHALLOW_EXTRA_REFS', 'refs/tags/*') 2061 self.fetch() 2062 2063 def test_shallow_remove_revs(self): 2064 # Create initial git repo 2065 self.add_empty_file('a') 2066 self.add_empty_file('b') 2067 self.git('checkout -b a_branch', cwd=self.srcdir) 2068 self.add_empty_file('c') 2069 self.add_empty_file('d') 2070 self.git('checkout master', cwd=self.srcdir) 2071 self.git('tag v0.0 a_branch', cwd=self.srcdir) 2072 self.add_empty_file('e') 2073 self.git('merge --no-ff --no-edit a_branch', cwd=self.srcdir) 2074 self.git('branch -d a_branch', cwd=self.srcdir) 2075 self.add_empty_file('f') 2076 self.assertRevCount(7, cwd=self.srcdir) 2077 2078 self.d.setVar('BB_GIT_SHALLOW_DEPTH', '0') 2079 self.d.setVar('BB_GIT_SHALLOW_REVS', 'v0.0') 2080 2081 self.fetch_shallow() 2082 2083 self.assertRevCount(5) 2084 2085 def test_shallow_invalid_revs(self): 2086 self.add_empty_file('a') 2087 self.add_empty_file('b') 2088 2089 self.d.setVar('BB_GIT_SHALLOW_DEPTH', '0') 2090 self.d.setVar('BB_GIT_SHALLOW_REVS', 'v0.0') 2091 2092 with self.assertRaises(bb.fetch2.FetchError): 2093 self.fetch() 2094 2095 def test_shallow_fetch_missing_revs(self): 2096 self.add_empty_file('a') 2097 self.add_empty_file('b') 2098 fetcher, ud = self.fetch(self.d.getVar('SRC_URI')) 2099 self.git('tag v0.0 master', cwd=self.srcdir) 2100 self.d.setVar('BB_GIT_SHALLOW_DEPTH', '0') 2101 self.d.setVar('BB_GIT_SHALLOW_REVS', 'v0.0') 2102 self.fetch_shallow() 2103 2104 def test_shallow_fetch_missing_revs_fails(self): 2105 self.add_empty_file('a') 2106 self.add_empty_file('b') 2107 fetcher, ud = self.fetch(self.d.getVar('SRC_URI')) 2108 self.d.setVar('BB_GIT_SHALLOW_DEPTH', '0') 2109 self.d.setVar('BB_GIT_SHALLOW_REVS', 'v0.0') 2110 2111 with self.assertRaises(bb.fetch2.FetchError), self.assertLogs("BitBake.Fetcher", level="ERROR") as cm: 2112 self.fetch_shallow() 2113 self.assertIn("Unable to find revision v0.0 even from upstream", cm.output[0]) 2114 2115 @skipIfNoNetwork() 2116 def test_bitbake(self): 2117 self.git('remote add --mirror=fetch origin https://github.com/openembedded/bitbake', cwd=self.srcdir) 2118 self.git('config core.bare true', cwd=self.srcdir) 2119 self.git('fetch', cwd=self.srcdir) 2120 2121 self.d.setVar('BB_GIT_SHALLOW_DEPTH', '0') 2122 # Note that the 1.10.0 tag is annotated, so this also tests 2123 # reference of an annotated vs unannotated tag 2124 self.d.setVar('BB_GIT_SHALLOW_REVS', '1.10.0') 2125 2126 self.fetch_shallow() 2127 2128 # Confirm that the history of 1.10.0 was removed 2129 orig_revs = len(self.git('rev-list master', cwd=self.srcdir).splitlines()) 2130 revs = len(self.git('rev-list master').splitlines()) 2131 self.assertNotEqual(orig_revs, revs) 2132 self.assertRefs(['master', 'origin/master']) 2133 self.assertRevCount(orig_revs - 1758) 2134 2135 def test_that_unpack_throws_an_error_when_the_git_clone_nor_shallow_tarball_exist(self): 2136 self.add_empty_file('a') 2137 fetcher, ud = self.fetch() 2138 bb.utils.remove(self.gitdir, recurse=True) 2139 bb.utils.remove(self.dldir, recurse=True) 2140 2141 with self.assertRaises(bb.fetch2.UnpackError) as context: 2142 fetcher.unpack(self.d.getVar('WORKDIR')) 2143 2144 self.assertIn("No up to date source found", context.exception.msg) 2145 self.assertIn("clone directory not available or not up to date", context.exception.msg) 2146 2147 @skipIfNoNetwork() 2148 def test_that_unpack_does_work_when_using_git_shallow_tarball_but_tarball_is_not_available(self): 2149 self.d.setVar('SRCREV', 'e5939ff608b95cdd4d0ab0e1935781ab9a276ac0') 2150 self.d.setVar('BB_GIT_SHALLOW', '1') 2151 self.d.setVar('BB_GENERATE_SHALLOW_TARBALLS', '1') 2152 fetcher = bb.fetch.Fetch(["git://git.yoctoproject.org/fstests;branch=master"], self.d) 2153 fetcher.download() 2154 2155 bb.utils.remove(self.dldir + "/*.tar.gz") 2156 fetcher.unpack(self.unpackdir) 2157 2158 dir = os.listdir(self.unpackdir + "/git/") 2159 self.assertIn("fstests.doap", dir) 2160 2161class GitLfsTest(FetcherTest): 2162 def setUp(self): 2163 FetcherTest.setUp(self) 2164 2165 self.gitdir = os.path.join(self.tempdir, 'git') 2166 self.srcdir = os.path.join(self.tempdir, 'gitsource') 2167 2168 self.d.setVar('WORKDIR', self.tempdir) 2169 self.d.setVar('S', self.gitdir) 2170 self.d.delVar('PREMIRRORS') 2171 self.d.delVar('MIRRORS') 2172 2173 self.d.setVar('SRCREV', '${AUTOREV}') 2174 self.d.setVar('AUTOREV', '${@bb.fetch2.get_autorev(d)}') 2175 self.d.setVar("__BBSEENSRCREV", "1") 2176 2177 bb.utils.mkdirhier(self.srcdir) 2178 self.git_init(cwd=self.srcdir) 2179 with open(os.path.join(self.srcdir, '.gitattributes'), 'wt') as attrs: 2180 attrs.write('*.mp3 filter=lfs -text') 2181 self.git(['add', '.gitattributes'], cwd=self.srcdir) 2182 self.git(['commit', '-m', "attributes", '.gitattributes'], cwd=self.srcdir) 2183 2184 def fetch(self, uri=None, download=True): 2185 uris = self.d.getVar('SRC_URI').split() 2186 uri = uris[0] 2187 d = self.d 2188 2189 fetcher = bb.fetch2.Fetch(uris, d) 2190 if download: 2191 fetcher.download() 2192 ud = fetcher.ud[uri] 2193 return fetcher, ud 2194 2195 def test_lfs_enabled(self): 2196 import shutil 2197 2198 uri = 'git://%s;protocol=file;lfs=1;branch=master' % self.srcdir 2199 self.d.setVar('SRC_URI', uri) 2200 2201 # Careful: suppress initial attempt at downloading until 2202 # we know whether git-lfs is installed. 2203 fetcher, ud = self.fetch(uri=None, download=False) 2204 self.assertIsNotNone(ud.method._find_git_lfs) 2205 2206 # If git-lfs can be found, the unpack should be successful. Only 2207 # attempt this with the real live copy of git-lfs installed. 2208 if ud.method._find_git_lfs(self.d): 2209 fetcher.download() 2210 shutil.rmtree(self.gitdir, ignore_errors=True) 2211 fetcher.unpack(self.d.getVar('WORKDIR')) 2212 2213 # If git-lfs cannot be found, the unpack should throw an error 2214 with self.assertRaises(bb.fetch2.FetchError): 2215 fetcher.download() 2216 ud.method._find_git_lfs = lambda d: False 2217 shutil.rmtree(self.gitdir, ignore_errors=True) 2218 fetcher.unpack(self.d.getVar('WORKDIR')) 2219 2220 def test_lfs_disabled(self): 2221 import shutil 2222 2223 uri = 'git://%s;protocol=file;lfs=0;branch=master' % self.srcdir 2224 self.d.setVar('SRC_URI', uri) 2225 2226 # In contrast to test_lfs_enabled(), allow the implicit download 2227 # done by self.fetch() to occur here. The point of this test case 2228 # is to verify that the fetcher can survive even if the source 2229 # repository has Git LFS usage configured. 2230 fetcher, ud = self.fetch() 2231 self.assertIsNotNone(ud.method._find_git_lfs) 2232 2233 # If git-lfs can be found, the unpack should be successful. A 2234 # live copy of git-lfs is not required for this case, so 2235 # unconditionally forge its presence. 2236 ud.method._find_git_lfs = lambda d: True 2237 shutil.rmtree(self.gitdir, ignore_errors=True) 2238 fetcher.unpack(self.d.getVar('WORKDIR')) 2239 2240 # If git-lfs cannot be found, the unpack should be successful 2241 ud.method._find_git_lfs = lambda d: False 2242 shutil.rmtree(self.gitdir, ignore_errors=True) 2243 fetcher.unpack(self.d.getVar('WORKDIR')) 2244 2245class GitURLWithSpacesTest(FetcherTest): 2246 test_git_urls = { 2247 "git://tfs-example.org:22/tfs/example%20path/example.git;branch=master" : { 2248 'url': 'git://tfs-example.org:22/tfs/example%20path/example.git;branch=master', 2249 'gitsrcname': 'tfs-example.org.22.tfs.example_path.example.git', 2250 'path': '/tfs/example path/example.git' 2251 }, 2252 "git://tfs-example.org:22/tfs/example%20path/example%20repo.git;branch=master" : { 2253 'url': 'git://tfs-example.org:22/tfs/example%20path/example%20repo.git;branch=master', 2254 'gitsrcname': 'tfs-example.org.22.tfs.example_path.example_repo.git', 2255 'path': '/tfs/example path/example repo.git' 2256 } 2257 } 2258 2259 def test_urls(self): 2260 2261 # Set fake SRCREV to stop git fetcher from trying to contact non-existent git repo 2262 self.d.setVar('SRCREV', '82ea737a0b42a8b53e11c9cde141e9e9c0bd8c40') 2263 2264 for test_git_url, ref in self.test_git_urls.items(): 2265 2266 fetcher = bb.fetch.Fetch([test_git_url], self.d) 2267 ud = fetcher.ud[fetcher.urls[0]] 2268 2269 self.assertEqual(ud.url, ref['url']) 2270 self.assertEqual(ud.path, ref['path']) 2271 self.assertEqual(ud.localfile, os.path.join(self.dldir, "git2", ref['gitsrcname'])) 2272 self.assertEqual(ud.localpath, os.path.join(self.dldir, "git2", ref['gitsrcname'])) 2273 self.assertEqual(ud.lockfile, os.path.join(self.dldir, "git2", ref['gitsrcname'] + '.lock')) 2274 self.assertEqual(ud.clonedir, os.path.join(self.dldir, "git2", ref['gitsrcname'])) 2275 self.assertEqual(ud.fullmirror, os.path.join(self.dldir, "git2_" + ref['gitsrcname'] + '.tar.gz')) 2276 2277class CrateTest(FetcherTest): 2278 @skipIfNoNetwork() 2279 def test_crate_url(self): 2280 2281 uri = "crate://crates.io/glob/0.2.11" 2282 self.d.setVar('SRC_URI', uri) 2283 2284 uris = self.d.getVar('SRC_URI').split() 2285 d = self.d 2286 2287 fetcher = bb.fetch2.Fetch(uris, self.d) 2288 fetcher.download() 2289 fetcher.unpack(self.tempdir) 2290 self.assertEqual(sorted(os.listdir(self.tempdir)), ['cargo_home', 'download' , 'unpacked']) 2291 self.assertEqual(sorted(os.listdir(self.tempdir + "/download")), ['glob-0.2.11.crate', 'glob-0.2.11.crate.done']) 2292 self.assertTrue(os.path.exists(self.tempdir + "/cargo_home/bitbake/glob-0.2.11/.cargo-checksum.json")) 2293 self.assertTrue(os.path.exists(self.tempdir + "/cargo_home/bitbake/glob-0.2.11/src/lib.rs")) 2294 2295 @skipIfNoNetwork() 2296 def test_crate_url_multi(self): 2297 2298 uri = "crate://crates.io/glob/0.2.11 crate://crates.io/time/0.1.35" 2299 self.d.setVar('SRC_URI', uri) 2300 2301 uris = self.d.getVar('SRC_URI').split() 2302 d = self.d 2303 2304 fetcher = bb.fetch2.Fetch(uris, self.d) 2305 fetcher.download() 2306 fetcher.unpack(self.tempdir) 2307 self.assertEqual(sorted(os.listdir(self.tempdir)), ['cargo_home', 'download' , 'unpacked']) 2308 self.assertEqual(sorted(os.listdir(self.tempdir + "/download")), ['glob-0.2.11.crate', 'glob-0.2.11.crate.done', 'time-0.1.35.crate', 'time-0.1.35.crate.done']) 2309 self.assertTrue(os.path.exists(self.tempdir + "/cargo_home/bitbake/glob-0.2.11/.cargo-checksum.json")) 2310 self.assertTrue(os.path.exists(self.tempdir + "/cargo_home/bitbake/glob-0.2.11/src/lib.rs")) 2311 self.assertTrue(os.path.exists(self.tempdir + "/cargo_home/bitbake/time-0.1.35/.cargo-checksum.json")) 2312 self.assertTrue(os.path.exists(self.tempdir + "/cargo_home/bitbake/time-0.1.35/src/lib.rs")) 2313 2314class NPMTest(FetcherTest): 2315 def skipIfNoNpm(): 2316 import shutil 2317 if not shutil.which('npm'): 2318 return unittest.skip('npm not installed') 2319 return lambda f: f 2320 2321 @skipIfNoNpm() 2322 @skipIfNoNetwork() 2323 def test_npm(self): 2324 url = 'npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example;version=1.0.0' 2325 fetcher = bb.fetch.Fetch([url], self.d) 2326 ud = fetcher.ud[fetcher.urls[0]] 2327 fetcher.download() 2328 self.assertTrue(os.path.exists(ud.localpath)) 2329 self.assertTrue(os.path.exists(ud.localpath + '.done')) 2330 self.assertTrue(os.path.exists(ud.resolvefile)) 2331 fetcher.unpack(self.unpackdir) 2332 unpackdir = os.path.join(self.unpackdir, 'npm') 2333 self.assertTrue(os.path.exists(os.path.join(unpackdir, 'package.json'))) 2334 2335 @skipIfNoNpm() 2336 @skipIfNoNetwork() 2337 def test_npm_bad_checksum(self): 2338 url = 'npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example;version=1.0.0' 2339 # Fetch once to get a tarball 2340 fetcher = bb.fetch.Fetch([url], self.d) 2341 ud = fetcher.ud[fetcher.urls[0]] 2342 fetcher.download() 2343 self.assertTrue(os.path.exists(ud.localpath)) 2344 # Modify the tarball 2345 bad = b'bad checksum' 2346 with open(ud.localpath, 'wb') as f: 2347 f.write(bad) 2348 # Verify that the tarball is fetched again 2349 fetcher.download() 2350 badsum = hashlib.sha512(bad).hexdigest() 2351 self.assertTrue(os.path.exists(ud.localpath + '_bad-checksum_' + badsum)) 2352 self.assertTrue(os.path.exists(ud.localpath)) 2353 2354 @skipIfNoNpm() 2355 @skipIfNoNetwork() 2356 def test_npm_premirrors(self): 2357 url = 'npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example;version=1.0.0' 2358 # Fetch once to get a tarball 2359 fetcher = bb.fetch.Fetch([url], self.d) 2360 ud = fetcher.ud[fetcher.urls[0]] 2361 fetcher.download() 2362 self.assertTrue(os.path.exists(ud.localpath)) 2363 2364 # Setup the mirror by renaming the download directory 2365 mirrordir = os.path.join(self.tempdir, 'mirror') 2366 bb.utils.rename(self.dldir, mirrordir) 2367 os.mkdir(self.dldir) 2368 2369 # Configure the premirror to be used 2370 self.d.setVar('PREMIRRORS', 'https?$://.*/.* file://%s/npm2' % mirrordir) 2371 self.d.setVar('BB_FETCH_PREMIRRORONLY', '1') 2372 2373 # Fetch again 2374 self.assertFalse(os.path.exists(ud.localpath)) 2375 # The npm fetcher doesn't handle that the .resolved file disappears 2376 # while the fetcher object exists, which it does when we rename the 2377 # download directory to "mirror" above. Thus we need a new fetcher to go 2378 # with the now empty download directory. 2379 fetcher = bb.fetch.Fetch([url], self.d) 2380 ud = fetcher.ud[fetcher.urls[0]] 2381 fetcher.download() 2382 self.assertTrue(os.path.exists(ud.localpath)) 2383 2384 @skipIfNoNpm() 2385 @skipIfNoNetwork() 2386 def test_npm_premirrors_with_specified_filename(self): 2387 url = 'npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example;version=1.0.0' 2388 # Fetch once to get a tarball 2389 fetcher = bb.fetch.Fetch([url], self.d) 2390 ud = fetcher.ud[fetcher.urls[0]] 2391 fetcher.download() 2392 self.assertTrue(os.path.exists(ud.localpath)) 2393 # Setup the mirror 2394 mirrordir = os.path.join(self.tempdir, 'mirror') 2395 bb.utils.mkdirhier(mirrordir) 2396 mirrorfilename = os.path.join(mirrordir, os.path.basename(ud.localpath)) 2397 os.replace(ud.localpath, mirrorfilename) 2398 self.d.setVar('PREMIRRORS', 'https?$://.*/.* file://%s' % mirrorfilename) 2399 self.d.setVar('BB_FETCH_PREMIRRORONLY', '1') 2400 # Fetch again 2401 self.assertFalse(os.path.exists(ud.localpath)) 2402 fetcher.download() 2403 self.assertTrue(os.path.exists(ud.localpath)) 2404 2405 @skipIfNoNpm() 2406 @skipIfNoNetwork() 2407 def test_npm_mirrors(self): 2408 # Fetch once to get a tarball 2409 url = 'npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example;version=1.0.0' 2410 fetcher = bb.fetch.Fetch([url], self.d) 2411 ud = fetcher.ud[fetcher.urls[0]] 2412 fetcher.download() 2413 self.assertTrue(os.path.exists(ud.localpath)) 2414 # Setup the mirror 2415 mirrordir = os.path.join(self.tempdir, 'mirror') 2416 bb.utils.mkdirhier(mirrordir) 2417 os.replace(ud.localpath, os.path.join(mirrordir, os.path.basename(ud.localpath))) 2418 self.d.setVar('MIRRORS', 'https?$://.*/.* file://%s/' % mirrordir) 2419 # Update the resolved url to an invalid url 2420 with open(ud.resolvefile, 'r') as f: 2421 url = f.read() 2422 uri = URI(url) 2423 uri.path = '/invalid' 2424 with open(ud.resolvefile, 'w') as f: 2425 f.write(str(uri)) 2426 # Fetch again 2427 self.assertFalse(os.path.exists(ud.localpath)) 2428 fetcher.download() 2429 self.assertTrue(os.path.exists(ud.localpath)) 2430 2431 @skipIfNoNpm() 2432 @skipIfNoNetwork() 2433 def test_npm_destsuffix_downloadfilename(self): 2434 url = 'npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example;version=1.0.0;destsuffix=foo/bar;downloadfilename=foo-bar.tgz' 2435 fetcher = bb.fetch.Fetch([url], self.d) 2436 fetcher.download() 2437 self.assertTrue(os.path.exists(os.path.join(self.dldir, 'npm2', 'foo-bar.tgz'))) 2438 fetcher.unpack(self.unpackdir) 2439 unpackdir = os.path.join(self.unpackdir, 'foo', 'bar') 2440 self.assertTrue(os.path.exists(os.path.join(unpackdir, 'package.json'))) 2441 2442 def test_npm_no_network_no_tarball(self): 2443 url = 'npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example;version=1.0.0' 2444 self.d.setVar('BB_NO_NETWORK', '1') 2445 fetcher = bb.fetch.Fetch([url], self.d) 2446 with self.assertRaises(bb.fetch2.NetworkAccess): 2447 fetcher.download() 2448 2449 @skipIfNoNpm() 2450 @skipIfNoNetwork() 2451 def test_npm_no_network_with_tarball(self): 2452 url = 'npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example;version=1.0.0' 2453 # Fetch once to get a tarball 2454 fetcher = bb.fetch.Fetch([url], self.d) 2455 fetcher.download() 2456 # Disable network access 2457 self.d.setVar('BB_NO_NETWORK', '1') 2458 # Fetch again 2459 fetcher.download() 2460 fetcher.unpack(self.unpackdir) 2461 unpackdir = os.path.join(self.unpackdir, 'npm') 2462 self.assertTrue(os.path.exists(os.path.join(unpackdir, 'package.json'))) 2463 2464 @skipIfNoNpm() 2465 @skipIfNoNetwork() 2466 def test_npm_registry_alternate(self): 2467 url = 'npm://skimdb.npmjs.com;package=@savoirfairelinux/node-server-example;version=1.0.0' 2468 fetcher = bb.fetch.Fetch([url], self.d) 2469 fetcher.download() 2470 fetcher.unpack(self.unpackdir) 2471 unpackdir = os.path.join(self.unpackdir, 'npm') 2472 self.assertTrue(os.path.exists(os.path.join(unpackdir, 'package.json'))) 2473 2474 @skipIfNoNpm() 2475 @skipIfNoNetwork() 2476 def test_npm_version_latest(self): 2477 url = 'npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example;version=latest' 2478 fetcher = bb.fetch.Fetch([url], self.d) 2479 fetcher.download() 2480 fetcher.unpack(self.unpackdir) 2481 unpackdir = os.path.join(self.unpackdir, 'npm') 2482 self.assertTrue(os.path.exists(os.path.join(unpackdir, 'package.json'))) 2483 2484 @skipIfNoNpm() 2485 @skipIfNoNetwork() 2486 def test_npm_registry_invalid(self): 2487 url = 'npm://registry.invalid.org;package=@savoirfairelinux/node-server-example;version=1.0.0' 2488 fetcher = bb.fetch.Fetch([url], self.d) 2489 with self.assertRaises(bb.fetch2.FetchError): 2490 fetcher.download() 2491 2492 @skipIfNoNpm() 2493 @skipIfNoNetwork() 2494 def test_npm_package_invalid(self): 2495 url = 'npm://registry.npmjs.org;package=@savoirfairelinux/invalid;version=1.0.0' 2496 fetcher = bb.fetch.Fetch([url], self.d) 2497 with self.assertRaises(bb.fetch2.FetchError): 2498 fetcher.download() 2499 2500 @skipIfNoNpm() 2501 @skipIfNoNetwork() 2502 def test_npm_version_invalid(self): 2503 url = 'npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example;version=invalid' 2504 with self.assertRaises(bb.fetch2.ParameterError): 2505 fetcher = bb.fetch.Fetch([url], self.d) 2506 2507 @skipIfNoNpm() 2508 @skipIfNoNetwork() 2509 def test_npm_registry_none(self): 2510 url = 'npm://;package=@savoirfairelinux/node-server-example;version=1.0.0' 2511 with self.assertRaises(bb.fetch2.MalformedUrl): 2512 fetcher = bb.fetch.Fetch([url], self.d) 2513 2514 @skipIfNoNpm() 2515 @skipIfNoNetwork() 2516 def test_npm_package_none(self): 2517 url = 'npm://registry.npmjs.org;version=1.0.0' 2518 with self.assertRaises(bb.fetch2.MissingParameterError): 2519 fetcher = bb.fetch.Fetch([url], self.d) 2520 2521 @skipIfNoNpm() 2522 @skipIfNoNetwork() 2523 def test_npm_version_none(self): 2524 url = 'npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example' 2525 with self.assertRaises(bb.fetch2.MissingParameterError): 2526 fetcher = bb.fetch.Fetch([url], self.d) 2527 2528 def create_shrinkwrap_file(self, data): 2529 import json 2530 datadir = os.path.join(self.tempdir, 'data') 2531 swfile = os.path.join(datadir, 'npm-shrinkwrap.json') 2532 bb.utils.mkdirhier(datadir) 2533 with open(swfile, 'w') as f: 2534 json.dump(data, f) 2535 # Also configure the S directory 2536 self.sdir = os.path.join(self.unpackdir, 'S') 2537 self.d.setVar('S', self.sdir) 2538 return swfile 2539 2540 @skipIfNoNpm() 2541 @skipIfNoNetwork() 2542 def test_npmsw(self): 2543 swfile = self.create_shrinkwrap_file({ 2544 'dependencies': { 2545 'array-flatten': { 2546 'version': '1.1.1', 2547 'resolved': 'https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz', 2548 'integrity': 'sha1-ml9pkFGx5wczKPKgCJaLZOopVdI=', 2549 'dependencies': { 2550 'content-type': { 2551 'version': 'https://registry.npmjs.org/content-type/-/content-type-1.0.4.tgz', 2552 'integrity': 'sha512-hIP3EEPs8tB9AT1L+NUqtwOAps4mk2Zob89MWXMHjHWg9milF/j4osnnQLXBCBFBk/tvIG/tUc9mOUJiPBhPXA==', 2553 'dependencies': { 2554 'cookie': { 2555 'version': 'git+https://github.com/jshttp/cookie.git#aec1177c7da67e3b3273df96cf476824dbc9ae09', 2556 'from': 'git+https://github.com/jshttp/cookie.git' 2557 } 2558 } 2559 } 2560 } 2561 } 2562 } 2563 }) 2564 fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d) 2565 fetcher.download() 2566 self.assertTrue(os.path.exists(os.path.join(self.dldir, 'npm2', 'array-flatten-1.1.1.tgz'))) 2567 self.assertTrue(os.path.exists(os.path.join(self.dldir, 'npm2', 'content-type-1.0.4.tgz'))) 2568 self.assertTrue(os.path.exists(os.path.join(self.dldir, 'git2', 'github.com.jshttp.cookie.git'))) 2569 fetcher.unpack(self.unpackdir) 2570 self.assertTrue(os.path.exists(os.path.join(self.sdir, 'npm-shrinkwrap.json'))) 2571 self.assertTrue(os.path.exists(os.path.join(self.sdir, 'node_modules', 'array-flatten', 'package.json'))) 2572 self.assertTrue(os.path.exists(os.path.join(self.sdir, 'node_modules', 'array-flatten', 'node_modules', 'content-type', 'package.json'))) 2573 self.assertTrue(os.path.exists(os.path.join(self.sdir, 'node_modules', 'array-flatten', 'node_modules', 'content-type', 'node_modules', 'cookie', 'package.json'))) 2574 2575 @skipIfNoNpm() 2576 @skipIfNoNetwork() 2577 def test_npmsw_dev(self): 2578 swfile = self.create_shrinkwrap_file({ 2579 'dependencies': { 2580 'array-flatten': { 2581 'version': '1.1.1', 2582 'resolved': 'https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz', 2583 'integrity': 'sha1-ml9pkFGx5wczKPKgCJaLZOopVdI=' 2584 }, 2585 'content-type': { 2586 'version': '1.0.4', 2587 'resolved': 'https://registry.npmjs.org/content-type/-/content-type-1.0.4.tgz', 2588 'integrity': 'sha512-hIP3EEPs8tB9AT1L+NUqtwOAps4mk2Zob89MWXMHjHWg9milF/j4osnnQLXBCBFBk/tvIG/tUc9mOUJiPBhPXA==', 2589 'dev': True 2590 } 2591 } 2592 }) 2593 # Fetch with dev disabled 2594 fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d) 2595 fetcher.download() 2596 self.assertTrue(os.path.exists(os.path.join(self.dldir, 'npm2', 'array-flatten-1.1.1.tgz'))) 2597 self.assertFalse(os.path.exists(os.path.join(self.dldir, 'npm2', 'content-type-1.0.4.tgz'))) 2598 # Fetch with dev enabled 2599 fetcher = bb.fetch.Fetch(['npmsw://' + swfile + ';dev=1'], self.d) 2600 fetcher.download() 2601 self.assertTrue(os.path.exists(os.path.join(self.dldir, 'npm2', 'array-flatten-1.1.1.tgz'))) 2602 self.assertTrue(os.path.exists(os.path.join(self.dldir, 'npm2', 'content-type-1.0.4.tgz'))) 2603 2604 @skipIfNoNpm() 2605 @skipIfNoNetwork() 2606 def test_npmsw_destsuffix(self): 2607 swfile = self.create_shrinkwrap_file({ 2608 'dependencies': { 2609 'array-flatten': { 2610 'version': '1.1.1', 2611 'resolved': 'https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz', 2612 'integrity': 'sha1-ml9pkFGx5wczKPKgCJaLZOopVdI=' 2613 } 2614 } 2615 }) 2616 fetcher = bb.fetch.Fetch(['npmsw://' + swfile + ';destsuffix=foo/bar'], self.d) 2617 fetcher.download() 2618 fetcher.unpack(self.unpackdir) 2619 self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'foo', 'bar', 'node_modules', 'array-flatten', 'package.json'))) 2620 2621 def test_npmsw_no_network_no_tarball(self): 2622 swfile = self.create_shrinkwrap_file({ 2623 'dependencies': { 2624 'array-flatten': { 2625 'version': '1.1.1', 2626 'resolved': 'https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz', 2627 'integrity': 'sha1-ml9pkFGx5wczKPKgCJaLZOopVdI=' 2628 } 2629 } 2630 }) 2631 self.d.setVar('BB_NO_NETWORK', '1') 2632 fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d) 2633 with self.assertRaises(bb.fetch2.NetworkAccess): 2634 fetcher.download() 2635 2636 @skipIfNoNpm() 2637 @skipIfNoNetwork() 2638 def test_npmsw_no_network_with_tarball(self): 2639 # Fetch once to get a tarball 2640 fetcher = bb.fetch.Fetch(['npm://registry.npmjs.org;package=array-flatten;version=1.1.1'], self.d) 2641 fetcher.download() 2642 # Disable network access 2643 self.d.setVar('BB_NO_NETWORK', '1') 2644 # Fetch again 2645 swfile = self.create_shrinkwrap_file({ 2646 'dependencies': { 2647 'array-flatten': { 2648 'version': '1.1.1', 2649 'resolved': 'https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz', 2650 'integrity': 'sha1-ml9pkFGx5wczKPKgCJaLZOopVdI=' 2651 } 2652 } 2653 }) 2654 fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d) 2655 fetcher.download() 2656 fetcher.unpack(self.unpackdir) 2657 self.assertTrue(os.path.exists(os.path.join(self.sdir, 'node_modules', 'array-flatten', 'package.json'))) 2658 2659 @skipIfNoNpm() 2660 @skipIfNoNetwork() 2661 def test_npmsw_npm_reusability(self): 2662 # Fetch once with npmsw 2663 swfile = self.create_shrinkwrap_file({ 2664 'dependencies': { 2665 'array-flatten': { 2666 'version': '1.1.1', 2667 'resolved': 'https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz', 2668 'integrity': 'sha1-ml9pkFGx5wczKPKgCJaLZOopVdI=' 2669 } 2670 } 2671 }) 2672 fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d) 2673 fetcher.download() 2674 # Disable network access 2675 self.d.setVar('BB_NO_NETWORK', '1') 2676 # Fetch again with npm 2677 fetcher = bb.fetch.Fetch(['npm://registry.npmjs.org;package=array-flatten;version=1.1.1'], self.d) 2678 fetcher.download() 2679 fetcher.unpack(self.unpackdir) 2680 self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'npm', 'package.json'))) 2681 2682 @skipIfNoNpm() 2683 @skipIfNoNetwork() 2684 def test_npmsw_bad_checksum(self): 2685 # Try to fetch with bad checksum 2686 swfile = self.create_shrinkwrap_file({ 2687 'dependencies': { 2688 'array-flatten': { 2689 'version': '1.1.1', 2690 'resolved': 'https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz', 2691 'integrity': 'sha1-gfNEp2hqgLTFKT6P3AsBYMgsBqg=' 2692 } 2693 } 2694 }) 2695 fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d) 2696 with self.assertRaises(bb.fetch2.FetchError): 2697 fetcher.download() 2698 # Fetch correctly to get a tarball 2699 swfile = self.create_shrinkwrap_file({ 2700 'dependencies': { 2701 'array-flatten': { 2702 'version': '1.1.1', 2703 'resolved': 'https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz', 2704 'integrity': 'sha1-ml9pkFGx5wczKPKgCJaLZOopVdI=' 2705 } 2706 } 2707 }) 2708 fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d) 2709 fetcher.download() 2710 localpath = os.path.join(self.dldir, 'npm2', 'array-flatten-1.1.1.tgz') 2711 self.assertTrue(os.path.exists(localpath)) 2712 # Modify the tarball 2713 bad = b'bad checksum' 2714 with open(localpath, 'wb') as f: 2715 f.write(bad) 2716 # Verify that the tarball is fetched again 2717 fetcher.download() 2718 badsum = hashlib.sha1(bad).hexdigest() 2719 self.assertTrue(os.path.exists(localpath + '_bad-checksum_' + badsum)) 2720 self.assertTrue(os.path.exists(localpath)) 2721 2722 @skipIfNoNpm() 2723 @skipIfNoNetwork() 2724 def test_npmsw_premirrors(self): 2725 # Fetch once to get a tarball 2726 fetcher = bb.fetch.Fetch(['npm://registry.npmjs.org;package=array-flatten;version=1.1.1'], self.d) 2727 ud = fetcher.ud[fetcher.urls[0]] 2728 fetcher.download() 2729 self.assertTrue(os.path.exists(ud.localpath)) 2730 # Setup the mirror 2731 mirrordir = os.path.join(self.tempdir, 'mirror') 2732 bb.utils.mkdirhier(mirrordir) 2733 os.replace(ud.localpath, os.path.join(mirrordir, os.path.basename(ud.localpath))) 2734 self.d.setVar('PREMIRRORS', 'https?$://.*/.* file://%s/' % mirrordir) 2735 self.d.setVar('BB_FETCH_PREMIRRORONLY', '1') 2736 # Fetch again 2737 self.assertFalse(os.path.exists(ud.localpath)) 2738 swfile = self.create_shrinkwrap_file({ 2739 'dependencies': { 2740 'array-flatten': { 2741 'version': '1.1.1', 2742 'resolved': 'https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz', 2743 'integrity': 'sha1-ml9pkFGx5wczKPKgCJaLZOopVdI=' 2744 } 2745 } 2746 }) 2747 fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d) 2748 fetcher.download() 2749 self.assertTrue(os.path.exists(ud.localpath)) 2750 2751 @skipIfNoNpm() 2752 @skipIfNoNetwork() 2753 def test_npmsw_mirrors(self): 2754 # Fetch once to get a tarball 2755 fetcher = bb.fetch.Fetch(['npm://registry.npmjs.org;package=array-flatten;version=1.1.1'], self.d) 2756 ud = fetcher.ud[fetcher.urls[0]] 2757 fetcher.download() 2758 self.assertTrue(os.path.exists(ud.localpath)) 2759 # Setup the mirror 2760 mirrordir = os.path.join(self.tempdir, 'mirror') 2761 bb.utils.mkdirhier(mirrordir) 2762 os.replace(ud.localpath, os.path.join(mirrordir, os.path.basename(ud.localpath))) 2763 self.d.setVar('MIRRORS', 'https?$://.*/.* file://%s/' % mirrordir) 2764 # Fetch again with invalid url 2765 self.assertFalse(os.path.exists(ud.localpath)) 2766 swfile = self.create_shrinkwrap_file({ 2767 'dependencies': { 2768 'array-flatten': { 2769 'version': '1.1.1', 2770 'resolved': 'https://invalid', 2771 'integrity': 'sha1-ml9pkFGx5wczKPKgCJaLZOopVdI=' 2772 } 2773 } 2774 }) 2775 fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d) 2776 fetcher.download() 2777 self.assertTrue(os.path.exists(ud.localpath)) 2778 2779class GitSharedTest(FetcherTest): 2780 def setUp(self): 2781 super(GitSharedTest, self).setUp() 2782 self.recipe_url = "git://git.openembedded.org/bitbake;branch=master" 2783 self.d.setVar('SRCREV', '82ea737a0b42a8b53e11c9cde141e9e9c0bd8c40') 2784 self.d.setVar("__BBSEENSRCREV", "1") 2785 2786 @skipIfNoNetwork() 2787 def test_shared_unpack(self): 2788 fetcher = bb.fetch.Fetch([self.recipe_url], self.d) 2789 2790 fetcher.download() 2791 fetcher.unpack(self.unpackdir) 2792 alt = os.path.join(self.unpackdir, 'git/.git/objects/info/alternates') 2793 self.assertTrue(os.path.exists(alt)) 2794 2795 @skipIfNoNetwork() 2796 def test_noshared_unpack(self): 2797 self.d.setVar('BB_GIT_NOSHARED', '1') 2798 self.unpackdir += '_noshared' 2799 fetcher = bb.fetch.Fetch([self.recipe_url], self.d) 2800 2801 fetcher.download() 2802 fetcher.unpack(self.unpackdir) 2803 alt = os.path.join(self.unpackdir, 'git/.git/objects/info/alternates') 2804 self.assertFalse(os.path.exists(alt)) 2805