poky/meta/classes-global/retain.bbclass

# Creates a tarball of the work directory for a recipe when one of its
# tasks fails, or any other nominated directories.
# Useful in cases where the environment in which builds are run is
# ephemeral or otherwise inaccessible for examination during
# debugging.
#
# To enable, simply add the following to your configuration:
#
# INHERIT += "retain"
#
# You can specify the recipe-specific directories to save upon failure
# or always (space-separated) e.g.:
#
# RETAIN_DIRS_FAILURE = "${WORKDIR};prefix=workdir"    # default
# RETAIN_DIRS_ALWAYS = "${T}"
#
# Naturally you can use overrides to limit it to a specific recipe:
# RETAIN_DIRS_ALWAYS:pn-somerecipe = "${T}"
#
# You can also specify global (non-recipe-specific) directories to save:
#
# RETAIN_DIRS_GLOBAL_FAILURE = "${LOG_DIR}"
# RETAIN_DIRS_GLOBAL_ALWAYS = "${BUILDSTATS_BASE}"
#
# If you wish to use a different tarball name prefix than the default of
# the directory name, you can do so by specifying a ;prefix= followed by
# the desired prefix (no spaces) in any of the RETAIN_DIRS_* variables.
# e.g. to always save the log files with a "recipelogs" as the prefix for
# the tarball of ${T} you would do this:
#
# RETAIN_DIRS_ALWAYS = "${T};prefix=recipelogs"
#
# Notes:
# * For this to be useful you also need corresponding logic in your build
#   orchestration tool to pick up any files written out to RETAIN_OUTDIR
#   (with the other assumption being that no files are present there at
#   the start of the build, since there is no logic to purge old files).
# * Work directories can be quite large, so saving them can take some time
#   and of course space.
# * Tarball creation is deferred to the end of the build, thus you will
#   get the state at the end, not immediately upon failure.
# * Extra directories must naturally be populated at the time the retain
#   class goes to save them (build completion); to try ensure this for
#   things that are also saved on build completion (e.g. buildstats), put
#   the INHERIT += "retain" after the INHERIT += lines for the class that
#   is writing out the data that you wish to save.
# * The tarballs have the tarball name as a top-level directory so that
#   multiple tarballs can be extracted side-by-side easily.
#
# Copyright (c) 2020, 2024 Microsoft Corporation
#
# SPDX-License-Identifier: GPL-2.0-only
#

RETAIN_OUTDIR ?= "${TMPDIR}/retained"
RETAIN_DIRS_FAILURE ?= "${WORKDIR};prefix=workdir"
RETAIN_DIRS_ALWAYS ?= ""
RETAIN_DIRS_GLOBAL_FAILURE ?= ""
RETAIN_DIRS_GLOBAL_ALWAYS ?= ""
RETAIN_TARBALL_SUFFIX ?= "${DATETIME}.tar.gz"
RETAIN_ENABLED ?= "1"


def retain_retain_dir(desc, tarprefix, path, tarbasepath, d):
    import datetime

    outdir = d.getVar('RETAIN_OUTDIR')
    bb.utils.mkdirhier(outdir)
    suffix = d.getVar('RETAIN_TARBALL_SUFFIX')
    tarname = '%s_%s' % (tarprefix, suffix)
    tarfp = os.path.join(outdir, '%s' % tarname)
    tardir = os.path.relpath(path, tarbasepath)
    cmdargs = ['tar', 'cfa', tarfp]
    # Prefix paths within the tarball with the tarball name so that
    # multiple tarballs can be extracted side-by-side
    tarname_noext = os.path.splitext(tarname)[0]
    if tarname_noext.endswith('.tar'):
        tarname_noext = tarname_noext[:-4]
    cmdargs += ['--transform', 's:^:%s/:' % tarname_noext]
    cmdargs += [tardir]
    try:
        bb.process.run(cmdargs, cwd=tarbasepath)
    except bb.process.ExecutionError as e:
        # It is possible for other tasks to be writing to the workdir
        # while we are tarring it up, in which case tar will return 1,
        # but we don't care in this situation (tar returns 2 for other
        # errors so we we will see those)
        if e.exitcode != 1:
            bb.warn('retain: error saving %s: %s' % (desc, str(e)))


addhandler retain_task_handler
retain_task_handler[eventmask] = "bb.build.TaskFailed bb.build.TaskSucceeded"

addhandler retain_build_handler
retain_build_handler[eventmask] = "bb.event.BuildStarted bb.event.BuildCompleted"

python retain_task_handler() {
    if d.getVar('RETAIN_ENABLED') != '1':
        return

    dirs = d.getVar('RETAIN_DIRS_ALWAYS')
    if isinstance(e, bb.build.TaskFailed):
        dirs += ' ' + d.getVar('RETAIN_DIRS_FAILURE')

    dirs = dirs.strip().split()
    if dirs:
        outdir = d.getVar('RETAIN_OUTDIR')
        bb.utils.mkdirhier(outdir)
        dirlist_file = os.path.join(outdir, 'retain_dirs.list')
        pn = d.getVar('PN')
        taskname = d.getVar('BB_CURRENTTASK')
        with open(dirlist_file, 'a') as f:
            for entry in dirs:
                f.write('%s %s %s\n' % (pn, taskname, entry))
}

python retain_build_handler() {
    outdir = d.getVar('RETAIN_OUTDIR')
    dirlist_file = os.path.join(outdir, 'retain_dirs.list')

    if isinstance(e, bb.event.BuildStarted):
        if os.path.exists(dirlist_file):
            os.remove(dirlist_file)
        return

    if d.getVar('RETAIN_ENABLED') != '1':
        return

    savedirs = {}
    try:
        with open(dirlist_file, 'r') as f:
            for line in f:
                pn, _, path = line.rstrip().split()
                if not path in savedirs:
                    savedirs[path] = pn
        os.remove(dirlist_file)
    except FileNotFoundError:
        pass

    if e.getFailures():
        for path in (d.getVar('RETAIN_DIRS_GLOBAL_FAILURE') or '').strip().split():
            savedirs[path] = ''

    for path in (d.getVar('RETAIN_DIRS_GLOBAL_ALWAYS') or '').strip().split():
        savedirs[path] = ''

    if savedirs:
        bb.plain('NOTE: retain: retaining build output...')
        count = 0
        for path, pn in savedirs.items():
            prefix = None
            if ';' in path:
                pathsplit = path.split(';')
                path = pathsplit[0]
                for param in pathsplit[1:]:
                    if '=' in param:
                        name, value = param.split('=', 1)
                        if name == 'prefix':
                            prefix = value
                        else:
                            bb.error('retain: invalid parameter "%s" in RETAIN_* variable value' % param)
                            return
                    else:
                        bb.error('retain: parameter "%s" missing value in RETAIN_* variable value' % param)
                        return
            if prefix:
                itemname = prefix
            else:
                itemname = os.path.basename(path)
            if pn:
                # Always add the recipe name in front
                itemname = pn + '_' + itemname
            if os.path.exists(path):
                retain_retain_dir(itemname, itemname, path, os.path.dirname(path), d)
                count += 1
            else:
                bb.warn('retain: path %s does not currently exist' % path)
        if count:
            item = 'archive' if count == 1 else 'archives'
            bb.plain('NOTE: retain: saved %d %s to %s' % (count, item, outdir))
}