CaROS/poky
3
1
Fork 0
mirror of https://git.yoctoproject.org/git/poky synced 2026-01-04 16:10:04 +00:00
Code Issues Packages Projects Releases Wiki Activity

reproducible: Merge code into base.bbclass

Browse Source 
Reproducibility is here to stay and needs to be part of our default workflow.
Move the remaining code to base.bbclass so it is always a first class citizen
and it is clear people need to be mindful of it.

(From OE-Core rev: abb0671d2cebfd7e8df94796404bbe9c7f961058)

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
This commit is contained in:
Richard Purdie 2021-10-14 12:34:17 +01:00
parent f47a396964
commit 4b424bdfdb
9 changed files with 95 additions and 106 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

38
meta/classes/base.bbclass
View File

@ -180,6 +180,44 @@ python base_do_unpack() {
bb.fatal("Bitbake Fetcher Error: " + repr(e))
}
SSTATETASKS += "do_deploy_source_date_epoch"
do_deploy_source_date_epoch () {
mkdir -p ${SDE_DEPLOYDIR}
if [ -e ${SDE_FILE} ]; then
echo "Deploying SDE from ${SDE_FILE} -> ${SDE_DEPLOYDIR}."
cp -p ${SDE_FILE} ${SDE_DEPLOYDIR}/__source_date_epoch.txt
else
echo "${SDE_FILE} not found!"
fi
}
python do_deploy_source_date_epoch_setscene () {
sstate_setscene(d)
bb.utils.mkdirhier(d.getVar('SDE_DIR'))
sde_file = os.path.join(d.getVar('SDE_DEPLOYDIR'), '__source_date_epoch.txt')
if os.path.exists(sde_file):
target = d.getVar('SDE_FILE')
bb.debug(1, "Moving setscene SDE file %s -> %s" % (sde_file, target))
bb.utils.rename(sde_file, target)
else:
bb.debug(1, "%s not found!" % sde_file)
}
do_deploy_source_date_epoch[dirs] = "${SDE_DEPLOYDIR}"
do_deploy_source_date_epoch[sstate-plaindirs] = "${SDE_DEPLOYDIR}"
addtask do_deploy_source_date_epoch_setscene
addtask do_deploy_source_date_epoch before do_configure after do_patch
python create_source_date_epoch_stamp() {
source_date_epoch = oe.reproducible.get_source_date_epoch(d, d.getVar('S'))
oe.reproducible.epochfile_write(source_date_epoch, d.getVar('SDE_FILE'), d)
}
do_unpack[postfuncs] += "create_source_date_epoch_stamp"
def get_source_date_epoch_value(d):
return oe.reproducible.epochfile_read(d.getVar('SDE_FILE'), d)
def get_layers_branch_rev(d):
layers = (d.getVar("BBLAYERS") or "").split()
layers_branch_rev = ["%-20s = \"%s:%s\"" % (os.path.basename(i), \

4
meta/classes/externalsrc.bbclass
View File

@ -109,8 +109,8 @@ python () {
if local_srcuri and task in fetch_tasks:
continue
bb.build.deltask(task, d)
if bb.data.inherits_class('reproducible_build', d) and task == 'do_unpack':
# The reproducible_build's create_source_date_epoch_stamp function must
if task == 'do_unpack':
# The reproducible build create_source_date_epoch_stamp function must
# be run after the source is available and before the
# do_deploy_source_date_epoch task. In the normal case, it's attached
# to do_unpack as a postfuncs, but since we removed do_unpack (above)

2
meta/classes/image-artifact-names.bbclass
View File

@ -15,7 +15,7 @@ IMAGE_LINK_NAME ?= "${IMAGE_BASENAME}-${MACHINE}"
IMAGE_NAME_SUFFIX ??= ".rootfs"
python () {
if bb.data.inherits_class('reproducible_build', d) and bb.data.inherits_class('deploy', d) and d.getVar("IMAGE_VERSION_SUFFIX") == "-${DATETIME}":
if bb.data.inherits_class('deploy', d) and d.getVar("IMAGE_VERSION_SUFFIX") == "-${DATETIME}":
import datetime
d.setVar("IMAGE_VERSION_SUFFIX", "-" + datetime.datetime.fromtimestamp(int(d.getVar("SOURCE_DATE_EPOCH")), datetime.timezone.utc).strftime('%Y%m%d%H%M%S'))
d.setVarFlag("IMAGE_VERSION_SUFFIX", "vardepvalue", "")

3
meta/classes/package_deb.bbclass
View File

@ -315,8 +315,7 @@ python do_package_write_deb () {
do_package_write_deb[dirs] = "${PKGWRITEDIRDEB}"
do_package_write_deb[cleandirs] = "${PKGWRITEDIRDEB}"
do_package_write_deb[depends] += "${@oe.utils.build_depends_string(d.getVar('PACKAGE_WRITE_DEPS'), 'do_populate_sysroot')}"
EPOCHTASK ??= ""
addtask package_write_deb after do_packagedata do_package ${EPOCHTASK} before do_build
addtask package_write_deb after do_packagedata do_package do_deploy_source_date_epoch before do_build
PACKAGEINDEXDEPS += "dpkg-native:do_populate_sysroot"
PACKAGEINDEXDEPS += "apt-native:do_populate_sysroot"

3
meta/classes/package_ipk.bbclass
View File

@ -274,8 +274,7 @@ python do_package_write_ipk () {
do_package_write_ipk[dirs] = "${PKGWRITEDIRIPK}"
do_package_write_ipk[cleandirs] = "${PKGWRITEDIRIPK}"
do_package_write_ipk[depends] += "${@oe.utils.build_depends_string(d.getVar('PACKAGE_WRITE_DEPS'), 'do_populate_sysroot')}"
EPOCHTASK ??= ""
addtask package_write_ipk after do_packagedata do_package ${EPOCHTASK} before do_build
addtask package_write_ipk after do_packagedata do_package do_deploy_source_date_epoch before do_build
PACKAGEINDEXDEPS += "opkg-utils-native:do_populate_sysroot"
PACKAGEINDEXDEPS += "opkg-native:do_populate_sysroot"

3
meta/classes/package_rpm.bbclass
View File

@ -748,8 +748,7 @@ python do_package_write_rpm () {
do_package_write_rpm[dirs] = "${PKGWRITEDIRRPM}"
do_package_write_rpm[cleandirs] = "${PKGWRITEDIRRPM}"
do_package_write_rpm[depends] += "${@oe.utils.build_depends_string(d.getVar('PACKAGE_WRITE_DEPS'), 'do_populate_sysroot')}"
EPOCHTASK ??= ""
addtask package_write_rpm after do_packagedata do_package ${EPOCHTASK} before do_build
addtask package_write_rpm after do_packagedata do_package do_deploy_source_date_epoch before do_build
PACKAGEINDEXDEPS += "rpm-native:do_populate_sysroot"
PACKAGEINDEXDEPS += "createrepo-c-native:do_populate_sysroot"

96
meta/classes/reproducible_build.bbclass
View File

@ -1,96 +0,0 @@
# reproducible_build.bbclass
#
# Sets the default SOURCE_DATE_EPOCH in each component's build environment.
# The format is number of seconds since the system epoch.
#
# Upstream components (generally) respect this environment variable,
# using it in place of the "current" date and time.
# See https://reproducible-builds.org/specs/source-date-epoch/
#
# The default value of SOURCE_DATE_EPOCH comes from the function
# get_source_date_epoch_value which reads from the SDE_FILE, or if the file
# is not available (or set to 0) will use the fallback of
# SOURCE_DATE_EPOCH_FALLBACK.
#
# The SDE_FILE is normally constructed from the function
# create_source_date_epoch_stamp which is typically added as a postfuncs to
# the do_unpack task. If a recipe does NOT have do_unpack, it should be added
# to a task that runs after the source is available and before the
# do_deploy_source_date_epoch task is executed.
#
# If a recipe wishes to override the default behavior it should set it's own
# SOURCE_DATE_EPOCH or override the do_deploy_source_date_epoch_stamp task
# with recipe-specific functionality to write the appropriate
# SOURCE_DATE_EPOCH into the SDE_FILE.
#
# SOURCE_DATE_EPOCH is intended to be a reproducible value. This value should
# be reproducible for anyone who builds the same revision from the same
# sources.
#
# There are 4 ways the create_source_date_epoch_stamp function determines what
# becomes SOURCE_DATE_EPOCH:
#
# 1. Use the value from __source_date_epoch.txt file if this file exists.
# This file was most likely created in the previous build by one of the
# following methods 2,3,4.
# Alternatively, it can be provided by a recipe via SRC_URI.
#
# If the file does not exist:
#
# 2. If there is a git checkout, use the last git commit timestamp.
# Git does not preserve file timestamps on checkout.
#
# 3. Use the mtime of "known" files such as NEWS, CHANGLELOG, ...
# This works for well-kept repositories distributed via tarball.
#
# 4. Use the modification time of the youngest file in the source tree, if
# there is one.
# This will be the newest file from the distribution tarball, if any.
#
# 5. Fall back to a fixed timestamp (SOURCE_DATE_EPOCH_FALLBACK).
#
# Once the value is determined, it is stored in the recipe's SDE_FILE.
SSTATETASKS += "do_deploy_source_date_epoch"
do_deploy_source_date_epoch () {
mkdir -p ${SDE_DEPLOYDIR}
if [ -e ${SDE_FILE} ]; then
echo "Deploying SDE from ${SDE_FILE} -> ${SDE_DEPLOYDIR}."
cp -p ${SDE_FILE} ${SDE_DEPLOYDIR}/__source_date_epoch.txt
else
echo "${SDE_FILE} not found!"
fi
}
python do_deploy_source_date_epoch_setscene () {
sstate_setscene(d)
bb.utils.mkdirhier(d.getVar('SDE_DIR'))
sde_file = os.path.join(d.getVar('SDE_DEPLOYDIR'), '__source_date_epoch.txt')
if os.path.exists(sde_file):
target = d.getVar('SDE_FILE')
bb.debug(1, "Moving setscene SDE file %s -> %s" % (sde_file, target))
bb.utils.rename(sde_file, target)
else:
bb.debug(1, "%s not found!" % sde_file)
}
do_deploy_source_date_epoch[dirs] = "${SDE_DEPLOYDIR}"
do_deploy_source_date_epoch[sstate-plaindirs] = "${SDE_DEPLOYDIR}"
addtask do_deploy_source_date_epoch_setscene
addtask do_deploy_source_date_epoch before do_configure after do_patch
python create_source_date_epoch_stamp() {
source_date_epoch = oe.reproducible.get_source_date_epoch(d, d.getVar('S'))
oe.reproducible.epochfile_write(source_date_epoch, d.getVar('SDE_FILE'), d)
}
EPOCHTASK = "do_deploy_source_date_epoch"
# Generate the stamp after do_unpack runs
do_unpack[postfuncs] += "create_source_date_epoch_stamp"
def get_source_date_epoch_value(d):
return oe.reproducible.epochfile_read(d.getVar('SDE_FILE'), d)

51
meta/lib/oe/reproducible.py
View File

@ -5,6 +5,57 @@ import os
import subprocess
import bb
# For reproducible builds, this code sets the default SOURCE_DATE_EPOCH in each
# component's build environment. The format is number of seconds since the
# system epoch.
#
# Upstream components (generally) respect this environment variable,
# using it in place of the "current" date and time.
# See https://reproducible-builds.org/specs/source-date-epoch/
#
# The default value of SOURCE_DATE_EPOCH comes from the function
# get_source_date_epoch_value which reads from the SDE_FILE, or if the file
# is not available will use the fallback of SOURCE_DATE_EPOCH_FALLBACK.
#
# The SDE_FILE is normally constructed from the function
# create_source_date_epoch_stamp which is typically added as a postfuncs to
# the do_unpack task. If a recipe does NOT have do_unpack, it should be added
# to a task that runs after the source is available and before the
# do_deploy_source_date_epoch task is executed.
#
# If a recipe wishes to override the default behavior it should set it's own
# SOURCE_DATE_EPOCH or override the do_deploy_source_date_epoch_stamp task
# with recipe-specific functionality to write the appropriate
# SOURCE_DATE_EPOCH into the SDE_FILE.
#
# SOURCE_DATE_EPOCH is intended to be a reproducible value. This value should
# be reproducible for anyone who builds the same revision from the same
# sources.
#
# There are 4 ways the create_source_date_epoch_stamp function determines what
# becomes SOURCE_DATE_EPOCH:
#
# 1. Use the value from __source_date_epoch.txt file if this file exists.
# This file was most likely created in the previous build by one of the
# following methods 2,3,4.
# Alternatively, it can be provided by a recipe via SRC_URI.
#
# If the file does not exist:
#
# 2. If there is a git checkout, use the last git commit timestamp.
# Git does not preserve file timestamps on checkout.
#
# 3. Use the mtime of "known" files such as NEWS, CHANGLELOG, ...
# This works for well-kept repositories distributed via tarball.
#
# 4. Use the modification time of the youngest file in the source tree, if
# there is one.
# This will be the newest file from the distribution tarball, if any.
#
# 5. Fall back to a fixed timestamp (SOURCE_DATE_EPOCH_FALLBACK).
#
# Once the value is determined, it is stored in the recipe's SDE_FILE.
def get_source_date_epoch_from_known_files(d, sourcedir):
source_date_epoch = None
newest_file = None

1
meta/lib/oeqa/selftest/cases/reproducible.py
View File

@ -219,7 +219,6 @@ class ReproducibleTests(OESelftestTestCase):
bb.utils.remove(tmpdir, recurse=True)
config = textwrap.dedent('''\
INHERIT += "reproducible_build"
PACKAGE_CLASSES = "{package_classes}"
INHIBIT_PACKAGE_STRIP = "1"
TMPDIR = "{tmpdir}"