scripts/contrib/image-manifest: add new script

image-manifest: script to generate product/image specific BOM

The image-manifest script generates image specific reports based on
an image manifest file. Currently there is data generated by buildhistory,
pkgdata, and license manifest but this data is poorly formated and spread
across multiple text files. This script can generate a single JSON output
file that is machine readable by other tools.

The manifest-info collects package information and stores the information
in a tarball. manifest-info can be configured using a json configuration
file. The default configuration including all possible options can be
dumped using the dump-config subcommand.

image-manifest takes an image manifest file as input to get the runtime
dependencies. As an option image-manifest can also use the build dependency
file, pn-buildlist, to get the build dependencies excluding native
packages.

This script extends the oe-image-manifest script [0] done by Paul Eggleton

[0]
https://github.com/intel/clear-linux-dissector-web/blob/master/layerindex/static/files/oe-image-manifest

------------------------------------------------------
usage: image-manifest [-h] [-d] [-q] <subcommand> ...

Image manifest utility

options:
  -h, --help     show this help message and exit
  -d, --debug    Enable debug output
  -q, --quiet    Print only errors

subcommands:
  recipe-info    Get recipe info
  list-depends   List dependencies
  list-recipes   List recipes producing packages within an image
  list-packages  List packages within an image
  list-layers    List included layers
  dump-config    Dump default config
  manifest-info  Export recipe info for a manifest
Use image-manifest <subcommand> --help to get help on a specific command

Co-developed-by: Paul Eggleton <bluelightning@bluelightning.org>
(From OE-Core rev: ad8fec9ce1704866df925bda18a240d6889b1ed5)

Signed-off-by: Anders Wallin <anders.wallin@windriver.com>
Signed-off-by: Saul Wold <saul.wold@windriver.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>

scripts/contrib/image-manifest

@@ -0,0 +1,523 @@
+#!/usr/bin/env python3
+
+# Script to extract information from image manifests
+#
+# Copyright (C) 2018 Intel Corporation
+# Copyright (C) 2021 Wind River Systems, Inc.
+#
+# SPDX-License-Identifier: GPL-2.0-only
+#
+
+import sys
+import os
+import argparse
+import logging
+import json
+import shutil
+import tempfile
+import tarfile
+from collections import OrderedDict
+
+scripts_path = os.path.dirname(__file__)
+lib_path = scripts_path + '/../lib'
+sys.path = sys.path + [lib_path]
+
+import scriptutils
+logger = scriptutils.logger_create(os.path.basename(__file__))
+
+import argparse_oe
+import scriptpath
+bitbakepath = scriptpath.add_bitbake_lib_path()
+if not bitbakepath:
+    logger.error("Unable to find bitbake by searching parent directory of this script or PATH")
+    sys.exit(1)
+logger.debug('Using standard bitbake path %s' % bitbakepath)
+scriptpath.add_oe_lib_path()
+
+import bb.tinfoil
+import bb.utils
+import oe.utils
+import oe.recipeutils
+
+def get_pkg_list(manifest):
+    pkglist = []
+    with open(manifest, 'r') as f:
+        for line in f:
+            linesplit = line.split()
+            if len(linesplit) == 3:
+                # manifest file
+                pkglist.append(linesplit[0])
+            elif len(linesplit) == 1:
+                # build dependency file
+                pkglist.append(linesplit[0])
+    return sorted(pkglist)
+
+def list_packages(args):
+    pkglist = get_pkg_list(args.manifest)
+    for pkg in pkglist:
+        print('%s' % pkg)
+
+def pkg2recipe(tinfoil, pkg):
+    if "-native" in pkg:
+        logger.info('skipping %s' % pkg)
+        return None
+
+    pkgdata_dir = tinfoil.config_data.getVar('PKGDATA_DIR')
+    pkgdatafile = os.path.join(pkgdata_dir, 'runtime-reverse', pkg)
+    logger.debug('pkgdatafile %s' % pkgdatafile)
+    try:
+        f = open(pkgdatafile, 'r')
+        for line in f:
+            if line.startswith('PN:'):
+                recipe = line.split(':', 1)[1].strip()
+                return recipe
+    except Exception:
+        logger.warning('%s is missing' % pkgdatafile)
+        return None
+
+def get_recipe_list(manifest, tinfoil):
+    pkglist = get_pkg_list(manifest)
+    recipelist = []
+    for pkg in pkglist:
+        recipe = pkg2recipe(tinfoil,pkg)
+        if recipe:
+            if not recipe in recipelist:
+                recipelist.append(recipe)
+
+    return sorted(recipelist)
+
+def list_recipes(args):
+    import bb.tinfoil
+    with bb.tinfoil.Tinfoil() as tinfoil:
+        tinfoil.logger.setLevel(logger.getEffectiveLevel())
+        tinfoil.prepare(config_only=True)
+        recipelist = get_recipe_list(args.manifest, tinfoil)
+        for recipe in sorted(recipelist):
+            print('%s' % recipe)
+
+def list_layers(args):
+
+    def find_git_repo(pth):
+        checkpth = pth
+        while checkpth != os.sep:
+            if os.path.exists(os.path.join(checkpth, '.git')):
+                return checkpth
+            checkpth = os.path.dirname(checkpth)
+        return None
+
+    def get_git_remote_branch(repodir):
+        try:
+            stdout, _ = bb.process.run(['git', 'rev-parse', '--abbrev-ref', '--symbolic-full-name', '@{u}'], cwd=repodir)
+        except bb.process.ExecutionError as e:
+            stdout = None
+        if stdout:
+            return stdout.strip()
+        else:
+            return None
+
+    def get_git_head_commit(repodir):
+        try:
+            stdout, _ = bb.process.run(['git', 'rev-parse', 'HEAD'], cwd=repodir)
+        except bb.process.ExecutionError as e:
+            stdout = None
+        if stdout:
+            return stdout.strip()
+        else:
+            return None
+
+    def get_git_repo_url(repodir, remote='origin'):
+        import bb.process
+        # Try to get upstream repo location from origin remote
+        try:
+            stdout, _ = bb.process.run(['git', 'remote', '-v'], cwd=repodir)
+        except bb.process.ExecutionError as e:
+            stdout = None
+        if stdout:
+            for line in stdout.splitlines():
+                splitline = line.split()
+                if len(splitline) > 1:
+                    if splitline[0] == remote and scriptutils.is_src_url(splitline[1]):
+                        return splitline[1]
+        return None
+
+    with bb.tinfoil.Tinfoil() as tinfoil:
+        tinfoil.logger.setLevel(logger.getEffectiveLevel())
+        tinfoil.prepare(config_only=False)
+        layers = OrderedDict()
+        for layerdir in tinfoil.config_data.getVar('BBLAYERS').split():
+            layerdata = OrderedDict()
+            layername = os.path.basename(layerdir)
+            logger.debug('layername %s, layerdir %s' % (layername, layerdir))
+            if layername in layers:
+                logger.warning('layername %s is not unique in configuration' % layername)
+                layername = os.path.basename(os.path.dirname(layerdir)) + '_' + os.path.basename(layerdir)
+                logger.debug('trying layername %s' % layername)
+                if layername in layers:
+                    logger.error('Layer name %s is not unique in configuration' % layername)
+                    sys.exit(2)
+            repodir = find_git_repo(layerdir)
+            if repodir:
+                remotebranch = get_git_remote_branch(repodir)
+                remote = 'origin'
+                if remotebranch and '/' in remotebranch:
+                    rbsplit = remotebranch.split('/', 1)
+                    layerdata['actual_branch'] = rbsplit[1]
+                    remote = rbsplit[0]
+                layerdata['vcs_url'] = get_git_repo_url(repodir, remote)
+                if os.path.abspath(repodir) != os.path.abspath(layerdir):
+                    layerdata['vcs_subdir'] = os.path.relpath(layerdir, repodir)
+                commit = get_git_head_commit(repodir)
+                if commit:
+                    layerdata['vcs_commit'] = commit
+            layers[layername] = layerdata
+
+    json.dump(layers, args.output, indent=2)
+
+def get_recipe(args):
+    with bb.tinfoil.Tinfoil() as tinfoil:
+        tinfoil.logger.setLevel(logger.getEffectiveLevel())
+        tinfoil.prepare(config_only=True)
+
+        recipe = pkg2recipe(tinfoil, args.package)
+        print(' %s package provided by %s' % (args.package, recipe))
+
+def pkg_dependencies(args):
+    def get_recipe_info(tinfoil, recipe):
+        try:
+            info = tinfoil.get_recipe_info(recipe)
+        except Exception:
+            logger.error('Failed to get recipe info for: %s' % recipe)
+            sys.exit(1)
+        if not info:
+            logger.warning('No recipe info found for: %s' % recipe)
+            sys.exit(1)
+        append_files = tinfoil.get_file_appends(info.fn)
+        appends = True
+        data = tinfoil.parse_recipe_file(info.fn, appends, append_files)
+        data.pn = info.pn
+        data.pv = info.pv
+        return data
+
+    def find_dependencies(tinfoil, assume_provided, recipe_info, packages, rn, order):
+        spaces = '  ' * order
+        data = recipe_info[rn]
+        if args.native:
+            logger.debug('%s- %s' % (spaces, data.pn))
+        elif "-native" not in data.pn:
+            if "cross" not in data.pn:
+                logger.debug('%s- %s' % (spaces, data.pn))
+
+        depends = []
+        for dep in data.depends:
+            if dep not in assume_provided:
+                depends.append(dep)
+
+        # First find all dependencies not in package list.
+        for dep in depends:
+            if dep not in packages:
+                packages.append(dep)
+                dep_data = get_recipe_info(tinfoil, dep)
+                # Do this once now to reduce the number of bitbake calls.
+                dep_data.depends = dep_data.getVar('DEPENDS').split()
+                recipe_info[dep] = dep_data
+
+        # Then recursively analyze all of the dependencies for the current recipe.
+        for dep in depends:
+            find_dependencies(tinfoil, assume_provided, recipe_info, packages, dep, order + 1)
+
+    with bb.tinfoil.Tinfoil() as tinfoil:
+        tinfoil.logger.setLevel(logger.getEffectiveLevel())
+        tinfoil.prepare()
+
+        assume_provided = tinfoil.config_data.getVar('ASSUME_PROVIDED').split()
+        logger.debug('assumed provided:')
+        for ap in sorted(assume_provided):
+            logger.debug(' - %s' % ap)
+
+        recipe = pkg2recipe(tinfoil, args.package)
+        data = get_recipe_info(tinfoil, recipe)
+        data.depends = []
+        depends = data.getVar('DEPENDS').split()
+        for dep in depends:
+            if dep not in assume_provided:
+                data.depends.append(dep)
+
+        recipe_info = dict([(recipe, data)])
+        packages = []
+        find_dependencies(tinfoil, assume_provided, recipe_info, packages, recipe, order=1)
+
+        print('\nThe following packages are required to build %s' % recipe)
+        for p in sorted(packages):
+            data = recipe_info[p]
+            if "-native" not in data.pn:
+                if "cross" not in data.pn:
+                    print(" %s (%s)" % (data.pn,p))
+
+        if args.native:
+            print('\nThe following native packages are required to build %s' % recipe)
+            for p in sorted(packages):
+                data = recipe_info[p]
+                if "-native" in data.pn:
+                    print(" %s(%s)" % (data.pn,p))
+                if "cross" in data.pn:
+                    print(" %s(%s)" % (data.pn,p))
+
+def default_config():
+    vlist = OrderedDict()
+    vlist['PV'] = 'yes'
+    vlist['SUMMARY'] = 'no'
+    vlist['DESCRIPTION'] = 'no'
+    vlist['SECTION'] = 'no'
+    vlist['LICENSE'] = 'yes'
+    vlist['HOMEPAGE'] = 'no'
+    vlist['BUGTRACKER'] = 'no'
+    vlist['PROVIDES'] = 'no'
+    vlist['BBCLASSEXTEND'] = 'no'
+    vlist['DEPENDS'] = 'no'
+    vlist['PACKAGECONFIG'] = 'no'
+    vlist['SRC_URI'] = 'yes'
+    vlist['SRCREV'] = 'yes'
+    vlist['EXTRA_OECONF'] = 'no'
+    vlist['EXTRA_OESCONS'] = 'no'
+    vlist['EXTRA_OECMAKE'] = 'no'
+    vlist['EXTRA_OEMESON'] = 'no'
+
+    clist = OrderedDict()
+    clist['variables'] = vlist
+    clist['filepath'] = 'no'
+    clist['sha256sum'] = 'no'
+    clist['layerdir'] = 'no'
+    clist['layer'] = 'no'
+    clist['inherits'] = 'no'
+    clist['source_urls'] = 'no'
+    clist['packageconfig_opts'] = 'no'
+    clist['patches'] = 'no'
+    clist['packagedir'] = 'no'
+    return clist
+
+def dump_config(args):
+    config = default_config()
+    f = open('default_config.json', 'w')
+    json.dump(config, f, indent=2)
+    logger.info('Default config list dumped to default_config.json')
+
+def export_manifest_info(args):
+
+    def handle_value(value):
+        if value:
+            return oe.utils.squashspaces(value)
+        else:
+            return value
+
+    if args.config:
+        logger.debug('config: %s' % args.config)
+        f = open(args.config, 'r')
+        config = json.load(f, object_pairs_hook=OrderedDict)
+    else:
+        config = default_config()
+    if logger.isEnabledFor(logging.DEBUG):
+        print('Configuration:')
+        json.dump(config, sys.stdout, indent=2)
+        print('')
+
+    tmpoutdir = tempfile.mkdtemp(prefix=os.path.basename(__file__)+'-')
+    logger.debug('tmp dir: %s' % tmpoutdir)
+
+    # export manifest
+    shutil.copy2(args.manifest,os.path.join(tmpoutdir, "manifest"))
+
+    with bb.tinfoil.Tinfoil(tracking=True) as tinfoil:
+        tinfoil.logger.setLevel(logger.getEffectiveLevel())
+        tinfoil.prepare(config_only=False)
+
+        pkglist = get_pkg_list(args.manifest)
+        # export pkg list
+        f = open(os.path.join(tmpoutdir, "pkgs"), 'w')
+        for pkg in pkglist:
+            f.write('%s\n' % pkg)
+        f.close()
+
+        recipelist = []
+        for pkg in pkglist:
+            recipe = pkg2recipe(tinfoil,pkg)
+            if recipe:
+                if not recipe in recipelist:
+                    recipelist.append(recipe)
+        recipelist.sort()
+        # export recipe list
+        f = open(os.path.join(tmpoutdir, "recipes"), 'w')
+        for recipe in recipelist:
+            f.write('%s\n' % recipe)
+        f.close()
+
+        try:
+            rvalues = OrderedDict()
+            for pn in sorted(recipelist):
+                logger.debug('Package: %s' % pn)
+                rd = tinfoil.parse_recipe(pn)
+
+                rvalues[pn] = OrderedDict()
+
+                for varname in config['variables']:
+                    if config['variables'][varname] == 'yes':
+                        rvalues[pn][varname] = handle_value(rd.getVar(varname))
+
+                fpth = rd.getVar('FILE')
+                layerdir = oe.recipeutils.find_layerdir(fpth)
+                if config['filepath'] == 'yes':
+                    rvalues[pn]['filepath'] = os.path.relpath(fpth, layerdir)
+                    if config['sha256sum'] == 'yes':
+                        rvalues[pn]['sha256sum'] = bb.utils.sha256_file(fpth)
+
+                if config['layerdir'] == 'yes':
+                    rvalues[pn]['layerdir'] = layerdir
+
+                if config['layer'] == 'yes':
+                    rvalues[pn]['layer'] = os.path.basename(layerdir)
+
+                if config['inherits'] == 'yes':
+                    gr = set(tinfoil.config_data.getVar("__inherit_cache") or [])
+                    lr = set(rd.getVar("__inherit_cache") or [])
+                    rvalues[pn]['inherits'] = sorted({os.path.splitext(os.path.basename(r))[0] for r in lr if r not in gr})
+
+                if config['source_urls'] == 'yes':
+                    rvalues[pn]['source_urls'] = []
+                    for url in (rd.getVar('SRC_URI') or '').split():
+                        if not url.startswith('file://'):
+                            url = url.split(';')[0]
+                            rvalues[pn]['source_urls'].append(url)
+
+                if config['packageconfig_opts'] == 'yes':
+                    rvalues[pn]['packageconfig_opts'] = OrderedDict()
+                    for key in rd.getVarFlags('PACKAGECONFIG').keys():
+                        if key == 'doc':
+                            continue
+                        rvalues[pn]['packageconfig_opts'][key] = rd.getVarFlag('PACKAGECONFIG', key, True)
+
+                if config['patches'] == 'yes':
+                    patches = oe.recipeutils.get_recipe_patches(rd)
+                    rvalues[pn]['patches'] = []
+                    if patches:
+                        recipeoutdir = os.path.join(tmpoutdir, pn, 'patches')
+                        bb.utils.mkdirhier(recipeoutdir)
+                        for patch in patches:
+                            # Patches may be in other layers too
+                            patchlayerdir = oe.recipeutils.find_layerdir(patch)
+                            # patchlayerdir will be None for remote patches, which we ignore
+                            # (since currently they are considered as part of sources)
+                            if patchlayerdir:
+                                rvalues[pn]['patches'].append((os.path.basename(patchlayerdir), os.path.relpath(patch, patchlayerdir)))
+                                shutil.copy(patch, recipeoutdir)
+
+                if config['packagedir'] == 'yes':
+                    pn_dir = os.path.join(tmpoutdir, pn)
+                    bb.utils.mkdirhier(pn_dir)
+                    f = open(os.path.join(pn_dir, 'recipe.json'), 'w')
+                    json.dump(rvalues[pn], f, indent=2)
+                    f.close()
+
+            with open(os.path.join(tmpoutdir, 'recipes.json'), 'w') as f:
+                json.dump(rvalues, f, indent=2)
+
+            if args.output:
+                outname = os.path.basename(args.output)
+            else:
+                outname = os.path.splitext(os.path.basename(args.manifest))[0]
+            if outname.endswith('.tar.gz'):
+                outname = outname[:-7]
+            elif outname.endswith('.tgz'):
+                outname = outname[:-4]
+
+            tarfn = outname
+            if tarfn.endswith(os.sep):
+                tarfn = tarfn[:-1]
+            if not tarfn.endswith(('.tar.gz', '.tgz')):
+                tarfn += '.tar.gz'
+            with open(tarfn, 'wb') as f:
+                with tarfile.open(None, "w:gz", f) as tar:
+                    tar.add(tmpoutdir, outname)
+        finally:
+            shutil.rmtree(tmpoutdir)
+
+
+def main():
+    parser = argparse_oe.ArgumentParser(description="Image manifest utility",
+                                        epilog="Use %(prog)s <subcommand> --help to get help on a specific command")
+    parser.add_argument('-d', '--debug', help='Enable debug output', action='store_true')
+    parser.add_argument('-q', '--quiet', help='Print only errors', action='store_true')
+    subparsers = parser.add_subparsers(dest="subparser_name", title='subcommands', metavar='<subcommand>')
+    subparsers.required = True
+
+    # get recipe info
+    parser_get_recipes = subparsers.add_parser('recipe-info',
+                                          help='Get recipe info',
+                                          description='Get recipe information for a package')
+    parser_get_recipes.add_argument('package', help='Package name')
+    parser_get_recipes.set_defaults(func=get_recipe)
+
+    # list runtime dependencies
+    parser_pkg_dep = subparsers.add_parser('list-depends',
+                                          help='List dependencies',
+                                          description='List dependencies required to build the package')
+    parser_pkg_dep.add_argument('--native', help='also print native  and cross packages', action='store_true')
+    parser_pkg_dep.add_argument('package', help='Package name')
+    parser_pkg_dep.set_defaults(func=pkg_dependencies)
+
+    # list recipes
+    parser_recipes = subparsers.add_parser('list-recipes',
+                                          help='List recipes producing packages within an image',
+                                          description='Lists recipes producing the packages that went into an image, using the manifest and pkgdata')
+    parser_recipes.add_argument('manifest', help='Manifest file')
+    parser_recipes.set_defaults(func=list_recipes)
+
+    # list packages
+    parser_packages = subparsers.add_parser('list-packages',
+                                          help='List packages within an image',
+                                          description='Lists packages that went into an image, using the manifest')
+    parser_packages.add_argument('manifest', help='Manifest file')
+    parser_packages.set_defaults(func=list_packages)
+
+    # list layers
+    parser_layers = subparsers.add_parser('list-layers',
+                                          help='List included layers',
+                                          description='Lists included layers')
+    parser_layers.add_argument('-o', '--output', help='Output file - defaults to stdout if not specified',
+                                default=sys.stdout, type=argparse.FileType('w'))
+    parser_layers.set_defaults(func=list_layers)
+
+    # dump default configuration file
+    parser_dconfig = subparsers.add_parser('dump-config',
+                                          help='Dump default config',
+                                          description='Dump default config to default_config.json')
+    parser_dconfig.set_defaults(func=dump_config)
+
+    # export recipe info for packages in manifest
+    parser_export = subparsers.add_parser('manifest-info',
+                                          help='Export recipe info for a manifest',
+                                          description='Export recipe information using the manifest')
+    parser_export.add_argument('-c', '--config', help='load config from json file')
+    parser_export.add_argument('-o', '--output', help='Output file (tarball) - defaults to manifest name if not specified')
+    parser_export.add_argument('manifest', help='Manifest file')
+    parser_export.set_defaults(func=export_manifest_info)
+
+    args = parser.parse_args()
+
+    if args.debug:
+        logger.setLevel(logging.DEBUG)
+        logger.debug("Debug Enabled")
+    elif args.quiet:
+        logger.setLevel(logging.ERROR)
+
+    ret = args.func(args)
+
+    return ret
+
+
+if __name__ == "__main__":
+    try:
+        ret = main()
+    except Exception:
+        ret = 1
+        import traceback
+        traceback.print_exc()
+    sys.exit(ret)