mirror of
https://git.yoctoproject.org/git/poky
synced 2026-01-04 16:10:04 +00:00
e1c173621a
Using the glob function to map signatures to sstate files is very slow when the sstate is large and accessed over nfs. The lookup now only loads the necessary prefixes and doesn't use glob as all. Unfortunately I don't have access to the systems where the performance isse was noticed and on my test system the glob is fast enough that the performance numbers aren't useful. I could verify that file list returned by the new code is the same. [YOCTO #13539] (From OE-Core rev: ad36335b8592e0387dd36066920cd5ffefd375f8) Signed-off-by: Konrad Scherer <Konrad.Scherer@windriver.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
115 lines
3.1 KiB
Python
Executable File
115 lines
3.1 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
#
|
|
# SPDX-License-Identifier: GPL-2.0-only
|
|
#
|
|
|
|
import os
|
|
import sys
|
|
import shutil
|
|
import errno
|
|
import time
|
|
|
|
def mkdir(d):
|
|
try:
|
|
os.makedirs(d)
|
|
except OSError as e:
|
|
if e.errno != errno.EEXIST:
|
|
raise e
|
|
|
|
# extract the hash from past the last colon to last underscore
|
|
def extract_sha(filename):
|
|
return filename.split(':')[7].split('_')[0]
|
|
|
|
# get all files in a directory, extract hash and make
|
|
# a map from hash to list of file with that hash
|
|
def map_sha_to_files(dir_, prefix, sha_map):
|
|
sstate_prefix_path = dir_ + '/' + prefix + '/'
|
|
sstate_files = os.listdir(sstate_prefix_path)
|
|
for f in sstate_files:
|
|
try:
|
|
sha = extract_sha(f)
|
|
if sha not in sha_map:
|
|
sha_map[sha] = []
|
|
sha_map[sha].append(sstate_prefix_path + f)
|
|
except IndexError:
|
|
continue
|
|
|
|
# given a prefix build a map of hash to list of files
|
|
def build_sha_cache(prefix):
|
|
sha_map = {}
|
|
|
|
sstate_dir = sys.argv[2]
|
|
map_sha_to_files(sstate_dir, prefix, sha_map)
|
|
|
|
native_sstate_dir = sys.argv[2] + '/' + sys.argv[4]
|
|
map_sha_to_files(native_sstate_dir, prefix, sha_map)
|
|
|
|
return sha_map
|
|
|
|
if len(sys.argv) < 5:
|
|
print("Incorrect number of arguments specified")
|
|
print("syntax: gen-lockedsig-cache <locked-sigs.inc> <input-cachedir> <output-cachedir> <nativelsbstring> [filterfile]")
|
|
sys.exit(1)
|
|
|
|
filterlist = []
|
|
if len(sys.argv) > 5:
|
|
print('Reading filter file %s' % sys.argv[5])
|
|
with open(sys.argv[5]) as f:
|
|
for l in f.readlines():
|
|
if ":" in l:
|
|
filterlist.append(l.rstrip())
|
|
|
|
print('Reading %s' % sys.argv[1])
|
|
sigs = []
|
|
with open(sys.argv[1]) as f:
|
|
for l in f.readlines():
|
|
if ":" in l:
|
|
task, sig = l.split()[0].rsplit(':', 1)
|
|
if filterlist and not task in filterlist:
|
|
print('Filtering out %s' % task)
|
|
else:
|
|
sigs.append(sig)
|
|
|
|
print('Gathering file list')
|
|
start_time = time.perf_counter()
|
|
files = set()
|
|
sstate_content_cache = {}
|
|
for s in sigs:
|
|
prefix = s[:2]
|
|
if prefix not in sstate_content_cache:
|
|
sstate_content_cache[prefix] = build_sha_cache(prefix)
|
|
|
|
for f in sstate_content_cache[prefix][s]:
|
|
files.add(f)
|
|
|
|
elapsed = time.perf_counter() - start_time
|
|
print("Gathering file list took %.1fs" % elapsed)
|
|
|
|
print('Processing files')
|
|
for f in files:
|
|
sys.stdout.write('Processing %s... ' % f)
|
|
_, ext = os.path.splitext(f)
|
|
if not ext in ['.tgz', '.siginfo', '.sig']:
|
|
# Most likely a temp file, skip it
|
|
print('skipping')
|
|
continue
|
|
dst = os.path.join(sys.argv[3], os.path.relpath(f, sys.argv[2]))
|
|
destdir = os.path.dirname(dst)
|
|
mkdir(destdir)
|
|
|
|
src = os.path.realpath(f)
|
|
if os.path.exists(dst):
|
|
os.remove(dst)
|
|
if (os.stat(src).st_dev == os.stat(destdir).st_dev):
|
|
print('linking')
|
|
try:
|
|
os.link(src, dst)
|
|
except OSError as e:
|
|
print('hard linking failed, copying')
|
|
shutil.copyfile(src, dst)
|
|
else:
|
|
print('copying')
|
|
shutil.copyfile(src, dst)
|
|
|
|
print('Done!')
|