bitbake: lib/bb: Add filter support

Add the python API for applying filters to a string and being able to
register functions as filters.

Filter functions are pure functions where an input is translated into
an output and there are no external data accesses. This means translations
can be cached as they won't change.

(Bitbake rev: 7d25d7511ca14213eea78ee739d260295cfa4045)

Signed-off-by: Joshua Watt <JPEWhacker@gmail.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>

bitbake/bin/bitbake-selftest

@@ -32,6 +32,7 @@ tests = ["bb.tests.codeparser",
          "bb.tests.siggen",
          "bb.tests.utils",
          "bb.tests.compression",
+         "bb.tests.filter",
          "hashserv.tests",
          "prserv.tests",
          "layerindexlib.tests.layerindexobj",

bitbake/lib/bb/filter.py

@@ -0,0 +1,142 @@
+#
+# Copyright (C) 2025 Garmin Ltd. or its subsidiaries
+#
+# SPDX-License-Identifier: GPL-2.0-only
+#
+
+import builtins
+
+# Purposely blank out __builtins__ which prevents users from
+# calling any normal builtin python functions
+FILTERS = {
+    "__builtins__": {},
+}
+
+CACHE = {}
+
+
+def apply_filters(val, expressions):
+    g = FILTERS.copy()
+
+    for e in expressions:
+        e = e.strip()
+        if not e:
+            continue
+
+        k = (val, e)
+        if k not in CACHE:
+            # Set val as a local so it can be cleared out while keeping the
+            # globals
+            l = {"val": val}
+
+            CACHE[k] = eval(e, g, l)
+
+        val = CACHE[k]
+
+    return val
+
+
+class Namespace(object):
+    """
+    Helper class to simulate a python namespace. The object properties can be
+    set as if it were a dictionary. Properties cannot be changed or deleted
+    through the object interface
+    """
+
+    def __getitem__(self, name):
+        return self.__dict__[name]
+
+    def __setitem__(self, name, value):
+        self.__dict__[name] = value
+
+    def __contains__(self, name):
+        return name in self.__dict__
+
+    def __setattr__(self, name, value):
+        raise AttributeError(f"Attribute {name!r} cannot be changed")
+
+    def __delattr__(self, name):
+        raise AttributeError(f"Attribute {name!r} cannot be deleted")
+
+
+def filter_proc(*, name=None):
+    """
+    Decorator to mark a function that can be called in `apply_filters`, either
+    directly in a filter expression, or indirectly. The `name` argument can be
+    used to specify an alternate name for the function if the actual name is
+    not desired. The `name` can be a fully qualified namespace if desired.
+
+    All functions must be "pure" in that they do not depend on global state and
+    have no global side effects (e.g. the output only depends on the input
+    arguments); the results of filter expressions are cached to optimize
+    repeated calls.
+    """
+
+    def inner(func):
+        global FILTERS
+        nonlocal name
+
+        if name is None:
+            name = func.__name__
+
+        ns = name.split(".")
+        o = FILTERS
+        for n in ns[:-1]:
+            if not n in o:
+                o[n] = Namespace()
+            o = o[n]
+
+        o[ns[-1]] = func
+
+        return func
+
+    return inner
+
+
+# A select set of builtins that are supported in filter expressions
+filter_proc()(all)
+filter_proc()(all)
+filter_proc()(any)
+filter_proc()(bin)
+filter_proc()(bool)
+filter_proc()(chr)
+filter_proc()(enumerate)
+filter_proc()(float)
+filter_proc()(format)
+filter_proc()(hex)
+filter_proc()(int)
+filter_proc()(len)
+filter_proc()(map)
+filter_proc()(max)
+filter_proc()(min)
+filter_proc()(oct)
+filter_proc()(ord)
+filter_proc()(pow)
+filter_proc()(str)
+filter_proc()(sum)
+
+
+@filter_proc()
+def suffix(val, suffix):
+    return " ".join(v + suffix for v in val.split())
+
+
+@filter_proc()
+def prefix(val, prefix):
+    return " ".join(prefix + v for v in val.split())
+
+
+@filter_proc()
+def sort(val):
+    return " ".join(sorted(val.split()))
+
+
+@filter_proc()
+def remove(val, remove, sep=None):
+    if isinstance(remove, str):
+        remove = remove.split(sep)
+    new = [i for i in val.split(sep) if not i in remove]
+
+    if not sep:
+        return " ".join(new)
+    return sep.join(new)

bitbake/lib/bb/tests/filter.py

@@ -0,0 +1,88 @@
+#
+# Copyright (C) 2025 Garmin Ltd. or its subsidiaries
+#
+# SPDX-License-Identifier: GPL-2.0-only
+#
+
+import unittest
+import bb.filter
+
+
+class BuiltinFilterTest(unittest.TestCase):
+    def test_disallowed_builtins(self):
+        with self.assertRaises(NameError):
+            val = bb.filter.apply_filters("1", ["open('foo.txt', 'rb')"])
+
+    def test_prefix(self):
+        val = bb.filter.apply_filters("1 2 3", ["prefix(val, 'a')"])
+        self.assertEqual(val, "a1 a2 a3")
+
+        val = bb.filter.apply_filters("", ["prefix(val, 'a')"])
+        self.assertEqual(val, "")
+
+    def test_suffix(self):
+        val = bb.filter.apply_filters("1 2 3", ["suffix(val, 'b')"])
+        self.assertEqual(val, "1b 2b 3b")
+
+        val = bb.filter.apply_filters("", ["suffix(val, 'b')"])
+        self.assertEqual(val, "")
+
+    def test_sort(self):
+        val = bb.filter.apply_filters("z y x", ["sort(val)"])
+        self.assertEqual(val, "x y z")
+
+        val = bb.filter.apply_filters("", ["sort(val)"])
+        self.assertEqual(val, "")
+
+    def test_identity(self):
+        val = bb.filter.apply_filters("1 2 3", ["val"])
+        self.assertEqual(val, "1 2 3")
+
+        val = bb.filter.apply_filters("123", ["val"])
+        self.assertEqual(val, "123")
+
+    def test_empty(self):
+        val = bb.filter.apply_filters("1 2 3", ["", "prefix(val, 'a')", ""])
+        self.assertEqual(val, "a1 a2 a3")
+
+    def test_nested(self):
+        val = bb.filter.apply_filters("1 2 3", ["prefix(prefix(val, 'a'), 'b')"])
+        self.assertEqual(val, "ba1 ba2 ba3")
+
+        val = bb.filter.apply_filters("1 2 3", ["prefix(prefix(val, 'b'), 'a')"])
+        self.assertEqual(val, "ab1 ab2 ab3")
+
+    def test_filter_order(self):
+        val = bb.filter.apply_filters("1 2 3", ["prefix(val, 'a')", "prefix(val, 'b')"])
+        self.assertEqual(val, "ba1 ba2 ba3")
+
+        val = bb.filter.apply_filters("1 2 3", ["prefix(val, 'b')", "prefix(val, 'a')"])
+        self.assertEqual(val, "ab1 ab2 ab3")
+
+        val = bb.filter.apply_filters("1 2 3", ["prefix(val, 'a')", "suffix(val, 'b')"])
+        self.assertEqual(val, "a1b a2b a3b")
+
+        val = bb.filter.apply_filters("1 2 3", ["suffix(val, 'b')", "prefix(val, 'a')"])
+        self.assertEqual(val, "a1b a2b a3b")
+
+    def test_remove(self):
+        val = bb.filter.apply_filters("1 2 3", ["remove(val, ['2'])"])
+        self.assertEqual(val, "1 3")
+
+        val = bb.filter.apply_filters("1,2,3", ["remove(val, ['2'], ',')"])
+        self.assertEqual(val, "1,3")
+
+        val = bb.filter.apply_filters("1 2 3", ["remove(val, ['4'])"])
+        self.assertEqual(val, "1 2 3")
+
+        val = bb.filter.apply_filters("1 2 3", ["remove(val, ['1', '2'])"])
+        self.assertEqual(val, "3")
+
+        val = bb.filter.apply_filters("1 2 3", ["remove(val, '2')"])
+        self.assertEqual(val, "1 3")
+
+        val = bb.filter.apply_filters("1 2 3", ["remove(val, '4')"])
+        self.assertEqual(val, "1 2 3")
+
+        val = bb.filter.apply_filters("1 2 3", ["remove(val, '1 2')"])
+        self.assertEqual(val, "3")