backport: SPDX 3.0 fixes and tasks from upstream version Walnascar

Backports the SPDX 3.0 support and fixes from upstream walnascar
commit 49f47169953b807d430461ca33f3a2b076119712 into upstream
scarthgap.

(From OE-Core rev: 9c9b9545049a2f6e5c99edcb079275d29a4d1ac6)

Signed-off-by: Kamel Bouhara (Schneider Electric) <kamel.bouhara@bootlin.com>
Signed-off-by: Steve Sakoman <steve@sakoman.com>

meta/classes-recipe/create-spdx-image-3.0.bbclass

@@ -0,0 +1,85 @@
+#
+# Copyright OpenEmbedded Contributors
+#
+# SPDX-License-Identifier: GPL-2.0-only
+#
+# SPDX image tasks
+
+SPDX_ROOTFS_PACKAGES = "${SPDXDIR}/rootfs-packages.json"
+SPDXIMAGEDEPLOYDIR = "${SPDXDIR}/image-deploy"
+SPDXROOTFSDEPLOY = "${SPDXDIR}/rootfs-deploy"
+
+python spdx_collect_rootfs_packages() {
+    import json
+    from pathlib import Path
+    from oe.rootfs import image_list_installed_packages
+
+    root_packages_file = Path(d.getVar("SPDX_ROOTFS_PACKAGES"))
+
+    packages = image_list_installed_packages(d)
+    if not packages:
+        packages = {}
+
+    root_packages_file.parent.mkdir(parents=True, exist_ok=True)
+    with root_packages_file.open("w") as f:
+        json.dump(packages, f)
+}
+ROOTFS_POSTUNINSTALL_COMMAND =+ "spdx_collect_rootfs_packages"
+
+python do_create_rootfs_spdx() {
+    import oe.spdx30_tasks
+    oe.spdx30_tasks.create_rootfs_spdx(d)
+}
+addtask do_create_rootfs_spdx after do_rootfs before do_image
+SSTATETASKS += "do_create_rootfs_spdx"
+do_create_rootfs_spdx[sstate-inputdirs] = "${SPDXROOTFSDEPLOY}"
+do_create_rootfs_spdx[sstate-outputdirs] = "${DEPLOY_DIR_SPDX}"
+do_create_rootfs_spdx[recrdeptask] += "do_create_spdx do_create_package_spdx"
+do_create_rootfs_spdx[cleandirs] += "${SPDXROOTFSDEPLOY}"
+do_create_rootfs_spdx[file-checksums] += "${SPDX3_LIB_DEP_FILES}"
+
+python do_create_rootfs_spdx_setscene() {
+    sstate_setscene(d)
+}
+addtask do_create_rootfs_spdx_setscene
+
+python do_create_image_spdx() {
+    import oe.spdx30_tasks
+    oe.spdx30_tasks.create_image_spdx(d)
+}
+addtask do_create_image_spdx after do_image_complete do_create_rootfs_spdx before do_build
+SSTATETASKS += "do_create_image_spdx"
+SSTATE_SKIP_CREATION:task-create-image-spdx = "1"
+do_create_image_spdx[sstate-inputdirs] = "${SPDXIMAGEWORK}"
+do_create_image_spdx[sstate-outputdirs] = "${DEPLOY_DIR_SPDX}"
+do_create_image_spdx[cleandirs] = "${SPDXIMAGEWORK}"
+do_create_image_spdx[dirs] = "${SPDXIMAGEWORK}"
+do_create_image_spdx[file-checksums] += "${SPDX3_LIB_DEP_FILES}"
+do_create_image_spdx[vardeps] += "\
+    SPDX_IMAGE_PURPOSE \
+    "
+
+python do_create_image_spdx_setscene() {
+    sstate_setscene(d)
+}
+addtask do_create_image_spdx_setscene
+
+
+python do_create_image_sbom_spdx() {
+    import oe.spdx30_tasks
+    oe.spdx30_tasks.create_image_sbom_spdx(d)
+}
+addtask do_create_image_sbom_spdx after do_create_rootfs_spdx do_create_image_spdx before do_build
+SSTATETASKS += "do_create_image_sbom_spdx"
+SSTATE_SKIP_CREATION:task-create-image-sbom = "1"
+do_create_image_sbom_spdx[sstate-inputdirs] = "${SPDXIMAGEDEPLOYDIR}"
+do_create_image_sbom_spdx[sstate-outputdirs] = "${DEPLOY_DIR_IMAGE}"
+do_create_image_sbom_spdx[stamp-extra-info] = "${MACHINE_ARCH}"
+do_create_image_sbom_spdx[cleandirs] = "${SPDXIMAGEDEPLOYDIR}"
+do_create_image_sbom_spdx[recrdeptask] += "do_create_spdx do_create_package_spdx"
+do_create_image_sbom_spdx[file-checksums] += "${SPDX3_LIB_DEP_FILES}"
+
+python do_create_image_sbom_spdx_setscene() {
+    sstate_setscene(d)
+}
+addtask do_create_image_sbom_spdx_setscene

meta/classes-recipe/create-spdx-sdk-3.0.bbclass

@@ -0,0 +1,74 @@
+#
+# Copyright OpenEmbedded Contributors
+#
+# SPDX-License-Identifier: GPL-2.0-only
+#
+# SPDX SDK tasks
+
+do_populate_sdk[recrdeptask] += "do_create_spdx do_create_package_spdx"
+do_populate_sdk[cleandirs] += "${SPDXSDKWORK}"
+do_populate_sdk[postfuncs] += "sdk_create_sbom"
+do_populate_sdk[file-checksums] += "${SPDX3_LIB_DEP_FILES}"
+POPULATE_SDK_POST_HOST_COMMAND:append:task-populate-sdk = " sdk_host_create_spdx"
+POPULATE_SDK_POST_TARGET_COMMAND:append:task-populate-sdk = " sdk_target_create_spdx"
+
+do_populate_sdk_ext[recrdeptask] += "do_create_spdx do_create_package_spdx"
+do_populate_sdk_ext[cleandirs] += "${SPDXSDKEXTWORK}"
+do_populate_sdk_ext[postfuncs] += "sdk_ext_create_sbom"
+do_populate_sdk_ext[file-checksums] += "${SPDX3_LIB_DEP_FILES}"
+POPULATE_SDK_POST_HOST_COMMAND:append:task-populate-sdk-ext = " sdk_ext_host_create_spdx"
+POPULATE_SDK_POST_TARGET_COMMAND:append:task-populate-sdk-ext = " sdk_ext_target_create_spdx"
+
+python sdk_host_create_spdx() {
+    from pathlib import Path
+    import oe.spdx30_tasks
+    spdx_work_dir = Path(d.getVar('SPDXSDKWORK'))
+
+    oe.spdx30_tasks.sdk_create_spdx(d, "host", spdx_work_dir, d.getVar("TOOLCHAIN_OUTPUTNAME"))
+}
+
+python sdk_target_create_spdx() {
+    from pathlib import Path
+    import oe.spdx30_tasks
+    spdx_work_dir = Path(d.getVar('SPDXSDKWORK'))
+
+    oe.spdx30_tasks.sdk_create_spdx(d, "target", spdx_work_dir, d.getVar("TOOLCHAIN_OUTPUTNAME"))
+}
+
+python sdk_ext_host_create_spdx() {
+    from pathlib import Path
+    import oe.spdx30_tasks
+    spdx_work_dir = Path(d.getVar('SPDXSDKEXTWORK'))
+
+    # TODO: This doesn't seem to work
+    oe.spdx30_tasks.sdk_create_spdx(d, "host", spdx_work_dir, d.getVar("TOOLCHAINEXT_OUTPUTNAME"))
+}
+
+python sdk_ext_target_create_spdx() {
+    from pathlib import Path
+    import oe.spdx30_tasks
+    spdx_work_dir = Path(d.getVar('SPDXSDKEXTWORK'))
+
+    # TODO: This doesn't seem to work
+    oe.spdx30_tasks.sdk_create_spdx(d, "target", spdx_work_dir, d.getVar("TOOLCHAINEXT_OUTPUTNAME"))
+}
+
+
+python sdk_create_sbom() {
+    from pathlib import Path
+    import oe.spdx30_tasks
+    sdk_deploydir = Path(d.getVar("SDKDEPLOYDIR"))
+    spdx_work_dir = Path(d.getVar('SPDXSDKWORK'))
+
+    oe.spdx30_tasks.create_sdk_sbom(d, sdk_deploydir, spdx_work_dir, d.getVar("TOOLCHAIN_OUTPUTNAME"))
+}
+
+python sdk_ext_create_sbom() {
+    from pathlib import Path
+    import oe.spdx30_tasks
+    sdk_deploydir = Path(d.getVar("SDKEXTDEPLOYDIR"))
+    spdx_work_dir = Path(d.getVar('SPDXSDKEXTWORK'))
+
+    oe.spdx30_tasks.create_sdk_sbom(d, sdk_deploydir, spdx_work_dir, d.getVar("TOOLCHAINEXT_OUTPUTNAME"))
+}
+

meta/classes-recipe/nospdx.bbclass

@@ -0,0 +1,13 @@
+#
+# Copyright OpenEmbedded Contributors
+#
+# SPDX-License-Identifier: MIT
+#
+
+deltask do_collect_spdx_deps
+deltask do_create_spdx
+deltask do_create_spdx_runtime
+deltask do_create_package_spdx
+deltask do_create_rootfs_spdx
+deltask do_create_image_spdx
+deltask do_create_image_sbom

meta/classes/create-spdx-3.0.bbclass

@@ -0,0 +1,200 @@
+#
+# Copyright OpenEmbedded Contributors
+#
+# SPDX-License-Identifier: GPL-2.0-only
+#
+
+inherit spdx-common
+
+SPDX_VERSION = "3.0.1"
+
+# The list of SPDX profiles generated documents will conform to
+SPDX_PROFILES ?= "core build software simpleLicensing security"
+
+SPDX_INCLUDE_BUILD_VARIABLES ??= "0"
+SPDX_INCLUDE_BUILD_VARIABLES[doc] = "If set to '1', the bitbake variables for a \
+    recipe will be included in the Build object. This will most likely result \
+    in non-reproducible SPDX output"
+
+SPDX_INCLUDE_BITBAKE_PARENT_BUILD ??= "0"
+SPDX_INCLUDE_BITBAKE_PARENT_BUILD[doc] = "Report the parent invocation of bitbake \
+    for each Build object. This allows you to know who invoked bitbake to perform \
+    a build, but will result in non-reproducible SPDX output."
+
+SPDX_PACKAGE_ADDITIONAL_PURPOSE ?= ""
+SPDX_PACKAGE_ADDITIONAL_PURPOSE[doc] = "The list of additional purposes to assign to \
+    the generated packages for a recipe. The primary purpose is always `install`. \
+    Packages overrides are allowed to override the additional purposes for \
+    individual packages."
+
+SPDX_IMAGE_PURPOSE ?= "filesystemImage"
+SPDX_IMAGE_PURPOSE[doc] = "The list of purposes to assign to the generated images. \
+    The first listed item will be the Primary Purpose and all additional items will \
+    be added as additional purposes"
+
+SPDX_SDK_PURPOSE ?= "install"
+SPDX_SDK_PURPOSE[doc] = "The list of purposes to assign to the generate SDK installer. \
+    The first listed item will be the Primary Purpose and all additional items will \
+    be added as additional purposes"
+
+SPDX_INCLUDE_VEX ??= "current"
+SPDX_INCLUDE_VEX[doc] = "Controls what VEX information is in the output. Set to \
+    'none' to disable all VEX data. Set to 'current' to only include VEX data \
+    for vulnerabilities not already fixed in the upstream source code \
+    (recommended). Set  to 'all' to get all known historical vulnerabilities, \
+    including those already fixed upstream (warning: This can be large and \
+    slow)."
+
+SPDX_INCLUDE_TIMESTAMPS ?= "0"
+SPDX_INCLUDE_TIMESTAMPS[doc] = "Include time stamps in SPDX output. This is \
+    useful if you want to know when artifacts were produced and when builds \
+    occurred, but will result in non-reproducible SPDX output"
+
+SPDX_IMPORTS ??= ""
+SPDX_IMPORTS[doc] = "SPDX_IMPORTS is the base variable that describes how to \
+    reference external SPDX ids. Each import is defined as a key in this \
+    variable with a suffix to describe to as a suffix to look up more \
+    information about the import. Each key can have the following variables: \
+        SPDX_IMPORTS_<key>_spdxid: The Fully qualified SPDX ID of the object \
+        SPDX_IMPORTS_<key>_uri: The URI where the SPDX Document that contains \
+            the external object can be found. Optional but recommended \
+        SPDX_IMPORTS_<key>_hash_<hash>: The Checksum of the SPDX Document that \
+            contains the External ID. <hash> must be one the valid SPDX hashing \
+            algorithms, as described by the HashAlgorithm vocabulary in the\
+            SPDX 3 spec. Optional but recommended"
+
+# Agents
+#   Bitbake variables can be used to describe an SPDX Agent that may be used
+#   during the build. An Agent is specified using a set of variables which all
+#   start with some common base name:
+#
+#   <BASE>_name: The name of the Agent (required)
+#   <BASE>_type: The type of Agent. Must be one of "person", "organization",
+#       "software", or "agent" (the default if not specified)
+#   <BASE>_comment: The comment for the Agent (optional)
+#   <BASE>_id_<ID>: And External Identifier for the Agent. <ID> must be a valid
+#       ExternalIdentifierType from the SPDX 3 spec. Commonly, an E-mail address
+#       can be specified with <BASE>_id_email
+#
+#   Alternatively, an Agent can be an external reference by referencing a key
+#   in SPDX_IMPORTS like so:
+#
+#   <BASE>_import = "<key>"
+#
+#   Finally, the same agent described by another set of agent variables can be
+#   referenced by specifying the basename of the variable that should be
+#   referenced:
+#
+#   SPDX_PACKAGE_SUPPLIER_ref = "SPDX_AUTHORS_openembedded"
+
+SPDX_AUTHORS ??= "openembedded"
+SPDX_AUTHORS[doc] = "A space separated list of the document authors. Each item \
+    is used to name a base variable like SPDX_AUTHORS_<AUTHOR> that \
+    describes the author."
+
+SPDX_AUTHORS_openembedded_name = "OpenEmbedded"
+SPDX_AUTHORS_openembedded_type = "organization"
+
+SPDX_BUILD_HOST[doc] = "The base variable name to describe the build host on \
+    which a build is running. Must be an SPDX_IMPORTS key. Requires \
+    SPDX_INCLUDE_BITBAKE_PARENT_BUILD. NOTE: Setting this will result in \
+    non-reproducible SPDX output"
+
+SPDX_INVOKED_BY[doc] = "The base variable name to describe the Agent that \
+    invoked the build, which builds will link to if specified. Requires \
+    SPDX_INCLUDE_BITBAKE_PARENT_BUILD. NOTE: Setting this will likely result in \
+    non-reproducible SPDX output"
+
+SPDX_ON_BEHALF_OF[doc] = "The base variable name to describe the Agent on who's \
+    behalf the invoking Agent (SPDX_INVOKED_BY) is running the build. Requires \
+    SPDX_INCLUDE_BITBAKE_PARENT_BUILD. NOTE: Setting this will likely result in \
+    non-reproducible SPDX output"
+
+SPDX_PACKAGE_SUPPLIER[doc] = "The base variable name to describe the Agent who \
+    is supplying artifacts produced by the build"
+
+SPDX_PACKAGE_VERSION ??= "${PV}"
+SPDX_PACKAGE_VERSION[doc] = "The version of a package, software_packageVersion \
+    in software_Package"
+
+IMAGE_CLASSES:append = " create-spdx-image-3.0"
+SDK_CLASSES += "create-spdx-sdk-3.0"
+
+oe.spdx30_tasks.set_timestamp_now[vardepsexclude] = "SPDX_INCLUDE_TIMESTAMPS"
+oe.spdx30_tasks.get_package_sources_from_debug[vardepsexclude] += "STAGING_KERNEL_DIR"
+oe.spdx30_tasks.collect_dep_objsets[vardepsexclude] = "SPDX_MULTILIB_SSTATE_ARCHS"
+
+
+# SPDX library code makes heavy use of classes, which bitbake cannot easily
+# parse out dependencies. As such, the library code files that make use of
+# classes are explicitly added as file checksum dependencies.
+SPDX3_LIB_DEP_FILES = "\
+    ${COREBASE}/meta/lib/oe/sbom30.py:True \
+    ${COREBASE}/meta/lib/oe/spdx30.py:True \
+    "
+
+python do_create_spdx() {
+    import oe.spdx30_tasks
+    oe.spdx30_tasks.create_spdx(d)
+}
+do_create_spdx[vardeps] += "\
+    SPDX_INCLUDE_BITBAKE_PARENT_BUILD \
+    SPDX_PACKAGE_ADDITIONAL_PURPOSE \
+    SPDX_PROFILES \
+    SPDX_NAMESPACE_PREFIX \
+    SPDX_UUID_NAMESPACE \
+    "
+
+addtask do_create_spdx after \
+    do_collect_spdx_deps \
+    do_deploy_source_date_epoch \
+    do_populate_sysroot do_package do_packagedata \
+    before do_populate_sdk do_populate_sdk_ext do_build do_rm_work
+
+SSTATETASKS += "do_create_spdx"
+do_create_spdx[sstate-inputdirs] = "${SPDXDEPLOY}"
+do_create_spdx[sstate-outputdirs] = "${DEPLOY_DIR_SPDX}"
+do_create_spdx[file-checksums] += "${SPDX3_LIB_DEP_FILES}"
+
+python do_create_spdx_setscene () {
+    sstate_setscene(d)
+}
+addtask do_create_spdx_setscene
+
+do_create_spdx[dirs] = "${SPDXWORK}"
+do_create_spdx[cleandirs] = "${SPDXDEPLOY} ${SPDXWORK}"
+do_create_spdx[depends] += " \
+    ${PATCHDEPENDENCY} \
+    ${@create_spdx_source_deps(d)} \
+"
+
+python do_create_package_spdx() {
+    import oe.spdx30_tasks
+    oe.spdx30_tasks.create_package_spdx(d)
+}
+oe.spdx30_tasks.create_package_spdx[vardepsexclude] = "OVERRIDES"
+
+addtask do_create_package_spdx after do_create_spdx before do_build do_rm_work
+SSTATETASKS += "do_create_package_spdx"
+do_create_package_spdx[sstate-inputdirs] = "${SPDXRUNTIMEDEPLOY}"
+do_create_package_spdx[sstate-outputdirs] = "${DEPLOY_DIR_SPDX}"
+do_create_package_spdx[file-checksums] += "${SPDX3_LIB_DEP_FILES}"
+
+python do_create_package_spdx_setscene () {
+    sstate_setscene(d)
+}
+addtask do_create_package_spdx_setscene
+
+do_create_package_spdx[dirs] = "${SPDXRUNTIMEDEPLOY}"
+do_create_package_spdx[cleandirs] = "${SPDXRUNTIMEDEPLOY}"
+do_create_package_spdx[rdeptask] = "do_create_spdx"
+
+python spdx30_build_started_handler () {
+    import oe.spdx30_tasks
+    d = e.data.createCopy()
+    oe.spdx30_tasks.write_bitbake_spdx(d)
+}
+
+addhandler spdx30_build_started_handler
+spdx30_build_started_handler[eventmask] = "bb.event.BuildStarted"
+

meta/classes/spdx-common.bbclass

@@ -0,0 +1,99 @@
+#
+# Copyright OpenEmbedded Contributors
+#
+# SPDX-License-Identifier: GPL-2.0-only
+#
+
+SPDX_VERSION ??= ""
+DEPLOY_DIR_SPDX ??= "${DEPLOY_DIR}/spdx/${SPDX_VERSION}"
+
+# The product name that the CVE database uses.  Defaults to BPN, but may need to
+# be overriden per recipe (for example tiff.bb sets CVE_PRODUCT=libtiff).
+CVE_PRODUCT ??= "${BPN}"
+CVE_VERSION ??= "${PV}"
+
+SPDXDIR ??= "${WORKDIR}/spdx/${SPDX_VERSION}"
+SPDXDEPLOY = "${SPDXDIR}/deploy"
+SPDXWORK = "${SPDXDIR}/work"
+SPDXIMAGEWORK = "${SPDXDIR}/image-work"
+SPDXSDKWORK = "${SPDXDIR}/sdk-work"
+SPDXSDKEXTWORK = "${SPDXDIR}/sdk-ext-work"
+SPDXDEPS = "${SPDXDIR}/deps.json"
+
+SPDX_TOOL_NAME ??= "oe-spdx-creator"
+SPDX_TOOL_VERSION ??= "1.0"
+
+SPDXRUNTIMEDEPLOY = "${SPDXDIR}/runtime-deploy"
+
+SPDX_INCLUDE_SOURCES ??= "0"
+
+SPDX_UUID_NAMESPACE ??= "sbom.openembedded.org"
+SPDX_NAMESPACE_PREFIX ??= "http://spdx.org/spdxdocs"
+SPDX_PRETTY ??= "0"
+
+SPDX_LICENSES ??= "${COREBASE}/meta/files/spdx-licenses.json"
+
+SPDX_CUSTOM_ANNOTATION_VARS ??= ""
+
+SPDX_MULTILIB_SSTATE_ARCHS ??= "${SSTATE_ARCHS}"
+
+def create_spdx_source_deps(d):
+    import oe.spdx_common
+
+    deps = []
+    if d.getVar("SPDX_INCLUDE_SOURCES") == "1":
+        pn = d.getVar('PN')
+        # do_unpack is a hack for now; we only need it to get the
+        # dependencies do_unpack already has so we can extract the source
+        # ourselves
+        if oe.spdx_common.has_task(d, "do_unpack"):
+            deps.append("%s:do_unpack" % pn)
+
+        if oe.spdx_common.is_work_shared_spdx(d) and \
+           oe.spdx_common.process_sources(d):
+            # For kernel source code
+            if oe.spdx_common.has_task(d, "do_shared_workdir"):
+                deps.append("%s:do_shared_workdir" % pn)
+            elif d.getVar('S') == d.getVar('STAGING_KERNEL_DIR'):
+                deps.append("virtual/kernel:do_shared_workdir")
+
+            # For gcc-source-${PV} source code
+            if oe.spdx_common.has_task(d, "do_preconfigure"):
+                deps.append("%s:do_preconfigure" % pn)
+            elif oe.spdx_common.has_task(d, "do_patch"):
+                deps.append("%s:do_patch" % pn)
+            # For gcc-cross-x86_64 source code
+            elif oe.spdx_common.has_task(d, "do_configure"):
+                deps.append("%s:do_configure" % pn)
+
+    return " ".join(deps)
+
+
+python do_collect_spdx_deps() {
+    # This task calculates the build time dependencies of the recipe, and is
+    # required because while a task can deptask on itself, those dependencies
+    # do not show up in BB_TASKDEPDATA. To work around that, this task does the
+    # deptask on do_create_spdx and writes out the dependencies it finds, then
+    # do_create_spdx reads in the found dependencies when writing the actual
+    # SPDX document
+    import json
+    import oe.spdx_common
+    from pathlib import Path
+
+    spdx_deps_file = Path(d.getVar("SPDXDEPS"))
+
+    deps = oe.spdx_common.collect_direct_deps(d, "do_create_spdx")
+
+    with spdx_deps_file.open("w") as f:
+        json.dump(deps, f)
+}
+# NOTE: depending on do_unpack is a hack that is necessary to get it's dependencies for archive the source
+addtask do_collect_spdx_deps after do_unpack
+do_collect_spdx_deps[depends] += "${PATCHDEPENDENCY}"
+do_collect_spdx_deps[deptask] = "do_create_spdx"
+do_collect_spdx_deps[dirs] = "${SPDXDIR}"
+
+oe.spdx_common.collect_direct_deps[vardepsexclude] += "BB_TASKDEPDATA"
+oe.spdx_common.collect_direct_deps[vardeps] += "DEPENDS"
+oe.spdx_common.collect_package_providers[vardepsexclude] += "BB_TASKDEPDATA"
+oe.spdx_common.get_patched_src[vardepsexclude] += "STAGING_KERNEL_DIR"

meta/lib/oe/__init__.py

@@ -9,4 +9,4 @@ __path__ = extend_path(__path__, __name__)
 
 BBIMPORTS = ["data", "path", "utils", "types", "package", "packagedata", \
              "packagegroup", "sstatesig", "lsb", "cachedpath", "license", \
-             "qa", "reproducible", "rust", "buildcfg", "go"]
+             "qa", "reproducible", "rust", "buildcfg", "go", "spdx30_tasks", "spdx_common"]

meta/lib/oe/spdx_common.py

@@ -0,0 +1,244 @@
+#
+# Copyright OpenEmbedded Contributors
+#
+# SPDX-License-Identifier: GPL-2.0-only
+#
+
+import bb
+import collections
+import json
+import oe.packagedata
+import re
+import shutil
+
+from pathlib import Path
+from dataclasses import dataclass
+
+LIC_REGEX = re.compile(
+    rb"^\W*SPDX-License-Identifier:\s*([ \w\d.()+-]+?)(?:\s+\W*)?$",
+    re.MULTILINE,
+)
+
+
+def extract_licenses(filename):
+    """
+    Extract SPDX License identifiers from a file
+    """
+    try:
+        with open(filename, "rb") as f:
+            size = min(15000, os.stat(filename).st_size)
+            txt = f.read(size)
+            licenses = re.findall(LIC_REGEX, txt)
+            if licenses:
+                ascii_licenses = [lic.decode("ascii") for lic in licenses]
+                return ascii_licenses
+    except Exception as e:
+        bb.warn(f"Exception reading {filename}: {e}")
+    return []
+
+
+def is_work_shared_spdx(d):
+    return '/work-shared/' in d.getVar('S')
+
+
+def load_spdx_license_data(d):
+    with open(d.getVar("SPDX_LICENSES"), "r") as f:
+        data = json.load(f)
+        # Transform the license array to a dictionary
+        data["licenses"] = {l["licenseId"]: l for l in data["licenses"]}
+
+    return data
+
+
+def process_sources(d):
+    """
+    Returns True if the sources for this recipe should be included in the SPDX
+    or False if not
+    """
+    pn = d.getVar("PN")
+    assume_provided = (d.getVar("ASSUME_PROVIDED") or "").split()
+    if pn in assume_provided:
+        for p in d.getVar("PROVIDES").split():
+            if p != pn:
+                pn = p
+                break
+
+    # glibc-locale: do_fetch, do_unpack and do_patch tasks have been deleted,
+    # so avoid archiving source here.
+    if pn.startswith("glibc-locale"):
+        return False
+    if d.getVar("PN") == "libtool-cross":
+        return False
+    if d.getVar("PN") == "libgcc-initial":
+        return False
+    if d.getVar("PN") == "shadow-sysroot":
+        return False
+
+    return True
+
+
+@dataclass(frozen=True)
+class Dep(object):
+    pn: str
+    hashfn: str
+    in_taskhash: bool
+
+
+def collect_direct_deps(d, dep_task):
+    """
+    Find direct dependencies of current task
+
+    Returns the list of recipes that have a dep_task that the current task
+    depends on
+    """
+    current_task = "do_" + d.getVar("BB_CURRENTTASK")
+    pn = d.getVar("PN")
+
+    taskdepdata = d.getVar("BB_TASKDEPDATA", False)
+
+    for this_dep in taskdepdata.values():
+        if this_dep[0] == pn and this_dep[1] == current_task:
+            break
+    else:
+        bb.fatal(f"Unable to find this {pn}:{current_task} in taskdepdata")
+
+    deps = set()
+
+    for dep_name in this_dep.deps:
+        dep_data = taskdepdata[dep_name]
+        if dep_data.taskname == dep_task and dep_data.pn != pn:
+            deps.add((dep_data.pn, dep_data.hashfn, dep_name in this_dep.taskhash_deps))
+
+    return sorted(deps)
+
+
+def get_spdx_deps(d):
+    """
+    Reads the SPDX dependencies JSON file and returns the data
+    """
+    spdx_deps_file = Path(d.getVar("SPDXDEPS"))
+
+    deps = []
+    with spdx_deps_file.open("r") as f:
+        for d in json.load(f):
+            deps.append(Dep(*d))
+    return deps
+
+
+def collect_package_providers(d):
+    """
+    Returns a dictionary where each RPROVIDES is mapped to the package that
+    provides it
+    """
+    deploy_dir_spdx = Path(d.getVar("DEPLOY_DIR_SPDX"))
+
+    providers = {}
+
+    deps = collect_direct_deps(d, "do_create_spdx")
+    deps.append((d.getVar("PN"), d.getVar("BB_HASHFILENAME"), True))
+
+    for dep_pn, dep_hashfn, _ in deps:
+        localdata = d
+        recipe_data = oe.packagedata.read_pkgdata(dep_pn, localdata)
+        if not recipe_data:
+            localdata = bb.data.createCopy(d)
+            localdata.setVar("PKGDATA_DIR", "${PKGDATA_DIR_SDK}")
+            recipe_data = oe.packagedata.read_pkgdata(dep_pn, localdata)
+
+        for pkg in recipe_data.get("PACKAGES", "").split():
+            pkg_data = oe.packagedata.read_subpkgdata_dict(pkg, localdata)
+            rprovides = set(
+                n
+                for n, _ in bb.utils.explode_dep_versions2(
+                    pkg_data.get("RPROVIDES", "")
+                ).items()
+            )
+            rprovides.add(pkg)
+
+            if "PKG" in pkg_data:
+                pkg = pkg_data["PKG"]
+                rprovides.add(pkg)
+
+            for r in rprovides:
+                providers[r] = (pkg, dep_hashfn)
+
+    return providers
+
+
+def get_patched_src(d):
+    """
+    Save patched source of the recipe in SPDX_WORKDIR.
+    """
+    spdx_workdir = d.getVar("SPDXWORK")
+    spdx_sysroot_native = d.getVar("STAGING_DIR_NATIVE")
+    pn = d.getVar("PN")
+
+    workdir = d.getVar("WORKDIR")
+
+    try:
+        # The kernel class functions require it to be on work-shared, so we dont change WORKDIR
+        if not is_work_shared_spdx(d):
+            # Change the WORKDIR to make do_unpack do_patch run in another dir.
+            d.setVar("WORKDIR", spdx_workdir)
+            # Restore the original path to recipe's native sysroot (it's relative to WORKDIR).
+            d.setVar("STAGING_DIR_NATIVE", spdx_sysroot_native)
+
+            # The changed 'WORKDIR' also caused 'B' changed, create dir 'B' for the
+            # possibly requiring of the following tasks (such as some recipes's
+            # do_patch required 'B' existed).
+            bb.utils.mkdirhier(d.getVar("B"))
+
+            bb.build.exec_func("do_unpack", d)
+
+            if d.getVar("SRC_URI") != "":
+                if bb.data.inherits_class('dos2unix', d):
+                    bb.build.exec_func('do_convert_crlf_to_lf', d)
+                bb.build.exec_func("do_patch", d)
+
+        # Copy source from work-share to spdx_workdir
+        if is_work_shared_spdx(d):
+            share_src = d.getVar('S')
+            d.setVar("WORKDIR", spdx_workdir)
+            d.setVar("STAGING_DIR_NATIVE", spdx_sysroot_native)
+            # Copy source to ${SPDXWORK}, same basename dir of ${S};
+            src_dir = (
+                spdx_workdir
+                + "/"
+                + os.path.basename(share_src)
+            )
+            # For kernel souce, rename suffix dir 'kernel-source'
+            # to ${BP} (${BPN}-${PV})
+            if bb.data.inherits_class("kernel", d):
+                src_dir = spdx_workdir + "/" + d.getVar('BP')
+
+            bb.note(f"copyhardlinktree {share_src} to {src_dir}")
+            oe.path.copyhardlinktree(share_src, src_dir)
+
+        # Some userland has no source.
+        if not os.path.exists(spdx_workdir):
+            bb.utils.mkdirhier(spdx_workdir)
+    finally:
+        d.setVar("WORKDIR", workdir)
+
+
+def has_task(d, task):
+    return bool(d.getVarFlag(task, "task", False)) and not bool(d.getVarFlag(task, "noexec", False))
+
+
+def fetch_data_to_uri(fd, name):
+    """
+    Translates a bitbake FetchData to a string URI
+    """
+    uri = fd.type
+    # Map gitsm to git, since gitsm:// is not a valid URI protocol
+    if uri == "gitsm":
+        uri = "git"
+    proto = getattr(fd, "proto", None)
+    if proto is not None:
+        uri = uri + "+" + proto
+    uri = uri + "://" + fd.host + fd.path
+
+    if fd.method.supports_srcrev():
+        uri = uri + "@" + fd.revision
+
+    return uri