xref: /OK3568_Linux_fs/yocto/scripts/gen-lockedsig-cache (revision 4882a59341e53eb6f0b4789bf948001014eff981) 
1#!/usr/bin/env python3
2#
3# SPDX-License-Identifier: GPL-2.0-only
4#
5
6import os
7import sys
8import shutil
9import errno
10import time
11
12def mkdir(d):
13    try:
14        os.makedirs(d)
15    except OSError as e:
16        if e.errno != errno.EEXIST:
17            raise e
18
19# extract the hash from past the last colon to last underscore
20def extract_sha(filename):
21    return filename.split(':')[7].split('_')[0]
22
23# get all files in a directory, extract hash and make
24# a map from hash to list of file with that hash
25def map_sha_to_files(dir_, prefix, sha_map):
26    sstate_prefix_path = dir_ + '/' + prefix + '/'
27    if not os.path.exists(sstate_prefix_path):
28        return
29    sstate_files = os.listdir(sstate_prefix_path)
30    for f in sstate_files:
31        try:
32            sha = extract_sha(f)
33            if sha not in sha_map:
34                sha_map[sha] = []
35            sha_map[sha].append(sstate_prefix_path + f)
36        except IndexError:
37            continue
38
39# given a prefix build a map of hash to list of files
40def build_sha_cache(prefix):
41    sha_map = {}
42
43    sstate_dir = sys.argv[2]
44    map_sha_to_files(sstate_dir, prefix, sha_map)
45
46    native_sstate_dir = sys.argv[2] + '/' + sys.argv[4]
47    map_sha_to_files(native_sstate_dir, prefix, sha_map)
48
49    return sha_map
50
51if len(sys.argv) < 5:
52    print("Incorrect number of arguments specified")
53    print("syntax: gen-lockedsig-cache <locked-sigs.inc> <input-cachedir> <output-cachedir> <nativelsbstring> [filterfile]")
54    sys.exit(1)
55
56filterlist = []
57if len(sys.argv) > 5:
58    print('Reading filter file %s' % sys.argv[5])
59    with open(sys.argv[5]) as f:
60        for l in f.readlines():
61            if ":" in l:
62                filterlist.append(l.rstrip())
63
64print('Reading %s' % sys.argv[1])
65sigs = []
66with open(sys.argv[1]) as f:
67    for l in f.readlines():
68        if ":" in l:
69            task, sig = l.split()[0].rsplit(':', 1)
70            if filterlist and not task in filterlist:
71                print('Filtering out %s' % task)
72            else:
73                sigs.append(sig)
74
75print('Gathering file list')
76start_time = time.perf_counter()
77files = set()
78sstate_content_cache = {}
79for s in sigs:
80    prefix = s[:2]
81    prefix2 = s[2:4]
82    if prefix not in sstate_content_cache:
83        sstate_content_cache[prefix] = {}
84    if prefix2 not in sstate_content_cache[prefix]:
85        sstate_content_cache[prefix][prefix2] = build_sha_cache(prefix + "/" + prefix2)
86
87    if s in sstate_content_cache[prefix][prefix2]:
88        for f in sstate_content_cache[prefix][prefix2][s]:
89            files.add(f)
90
91elapsed = time.perf_counter() - start_time
92print("Gathering file list took %.1fs" % elapsed)
93
94print('Processing files')
95for f in files:
96    sys.stdout.write('Processing %s... ' % f)
97    if not f.endswith(('.tar.zst', '.siginfo', '.sig')):
98        # Most likely a temp file, skip it
99        print('skipping')
100        continue
101    dst = os.path.join(sys.argv[3], os.path.relpath(f, sys.argv[2]))
102    destdir = os.path.dirname(dst)
103    mkdir(destdir)
104
105    src = os.path.realpath(f)
106    if os.path.exists(dst):
107        os.remove(dst)
108    if (os.stat(src).st_dev == os.stat(destdir).st_dev):
109        print('linking')
110        try:
111            os.link(src, dst)
112        except OSError as e:
113            print('hard linking failed, copying')
114            shutil.copyfile(src, dst)
115    else:
116        print('copying')
117        shutil.copyfile(src, dst)
118
119print('Done!')
120