retain.bbclass 6.8 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182
  1. # Creates a tarball of the work directory for a recipe when one of its
  2. # tasks fails, or any other nominated directories.
  3. # Useful in cases where the environment in which builds are run is
  4. # ephemeral or otherwise inaccessible for examination during
  5. # debugging.
  6. #
  7. # To enable, simply add the following to your configuration:
  8. #
  9. # INHERIT += "retain"
  10. #
  11. # You can specify the recipe-specific directories to save upon failure
  12. # or always (space-separated) e.g.:
  13. #
  14. # RETAIN_DIRS_FAILURE = "${WORKDIR};prefix=workdir" # default
  15. # RETAIN_DIRS_ALWAYS = "${T}"
  16. #
  17. # Naturally you can use overrides to limit it to a specific recipe:
  18. # RETAIN_DIRS_ALWAYS:pn-somerecipe = "${T}"
  19. #
  20. # You can also specify global (non-recipe-specific) directories to save:
  21. #
  22. # RETAIN_DIRS_GLOBAL_FAILURE = "${LOG_DIR}"
  23. # RETAIN_DIRS_GLOBAL_ALWAYS = "${BUILDSTATS_BASE}"
  24. #
  25. # If you wish to use a different tarball name prefix than the default of
  26. # the directory name, you can do so by specifying a ;prefix= followed by
  27. # the desired prefix (no spaces) in any of the RETAIN_DIRS_* variables.
  28. # e.g. to always save the log files with a "recipelogs" as the prefix for
  29. # the tarball of ${T} you would do this:
  30. #
  31. # RETAIN_DIRS_ALWAYS = "${T};prefix=recipelogs"
  32. #
  33. # Notes:
  34. # * For this to be useful you also need corresponding logic in your build
  35. # orchestration tool to pick up any files written out to RETAIN_OUTDIR
  36. # (with the other assumption being that no files are present there at
  37. # the start of the build, since there is no logic to purge old files).
  38. # * Work directories can be quite large, so saving them can take some time
  39. # and of course space.
  40. # * Tarball creation is deferred to the end of the build, thus you will
  41. # get the state at the end, not immediately upon failure.
  42. # * Extra directories must naturally be populated at the time the retain
  43. # class goes to save them (build completion); to try ensure this for
  44. # things that are also saved on build completion (e.g. buildstats), put
  45. # the INHERIT += "retain" after the INHERIT += lines for the class that
  46. # is writing out the data that you wish to save.
  47. # * The tarballs have the tarball name as a top-level directory so that
  48. # multiple tarballs can be extracted side-by-side easily.
  49. #
  50. # Copyright (c) 2020, 2024 Microsoft Corporation
  51. #
  52. # SPDX-License-Identifier: GPL-2.0-only
  53. #
  54. RETAIN_OUTDIR ?= "${TMPDIR}/retained"
  55. RETAIN_DIRS_FAILURE ?= "${WORKDIR};prefix=workdir"
  56. RETAIN_DIRS_ALWAYS ?= ""
  57. RETAIN_DIRS_GLOBAL_FAILURE ?= ""
  58. RETAIN_DIRS_GLOBAL_ALWAYS ?= ""
  59. RETAIN_TARBALL_SUFFIX ?= "${DATETIME}.tar.gz"
  60. RETAIN_ENABLED ?= "1"
  61. def retain_retain_dir(desc, tarprefix, path, tarbasepath, d):
  62. import datetime
  63. outdir = d.getVar('RETAIN_OUTDIR')
  64. bb.utils.mkdirhier(outdir)
  65. suffix = d.getVar('RETAIN_TARBALL_SUFFIX')
  66. tarname = '%s_%s' % (tarprefix, suffix)
  67. tarfp = os.path.join(outdir, '%s' % tarname)
  68. tardir = os.path.relpath(path, tarbasepath)
  69. cmdargs = ['tar', 'cfa', tarfp]
  70. # Prefix paths within the tarball with the tarball name so that
  71. # multiple tarballs can be extracted side-by-side
  72. tarname_noext = os.path.splitext(tarname)[0]
  73. if tarname_noext.endswith('.tar'):
  74. tarname_noext = tarname_noext[:-4]
  75. cmdargs += ['--transform', 's:^:%s/:' % tarname_noext]
  76. cmdargs += [tardir]
  77. try:
  78. bb.process.run(cmdargs, cwd=tarbasepath)
  79. except bb.process.ExecutionError as e:
  80. # It is possible for other tasks to be writing to the workdir
  81. # while we are tarring it up, in which case tar will return 1,
  82. # but we don't care in this situation (tar returns 2 for other
  83. # errors so we we will see those)
  84. if e.exitcode != 1:
  85. bb.warn('retain: error saving %s: %s' % (desc, str(e)))
  86. addhandler retain_task_handler
  87. retain_task_handler[eventmask] = "bb.build.TaskFailed bb.build.TaskSucceeded"
  88. addhandler retain_build_handler
  89. retain_build_handler[eventmask] = "bb.event.BuildStarted bb.event.BuildCompleted"
  90. python retain_task_handler() {
  91. if d.getVar('RETAIN_ENABLED') != '1':
  92. return
  93. dirs = d.getVar('RETAIN_DIRS_ALWAYS')
  94. if isinstance(e, bb.build.TaskFailed):
  95. dirs += ' ' + d.getVar('RETAIN_DIRS_FAILURE')
  96. dirs = dirs.strip().split()
  97. if dirs:
  98. outdir = d.getVar('RETAIN_OUTDIR')
  99. bb.utils.mkdirhier(outdir)
  100. dirlist_file = os.path.join(outdir, 'retain_dirs.list')
  101. pn = d.getVar('PN')
  102. taskname = d.getVar('BB_CURRENTTASK')
  103. with open(dirlist_file, 'a') as f:
  104. for entry in dirs:
  105. f.write('%s %s %s\n' % (pn, taskname, entry))
  106. }
  107. python retain_build_handler() {
  108. outdir = d.getVar('RETAIN_OUTDIR')
  109. dirlist_file = os.path.join(outdir, 'retain_dirs.list')
  110. if isinstance(e, bb.event.BuildStarted):
  111. if os.path.exists(dirlist_file):
  112. os.remove(dirlist_file)
  113. return
  114. if d.getVar('RETAIN_ENABLED') != '1':
  115. return
  116. savedirs = {}
  117. try:
  118. with open(dirlist_file, 'r') as f:
  119. for line in f:
  120. pn, _, path = line.rstrip().split()
  121. if not path in savedirs:
  122. savedirs[path] = pn
  123. os.remove(dirlist_file)
  124. except FileNotFoundError:
  125. pass
  126. if e.getFailures():
  127. for path in (d.getVar('RETAIN_DIRS_GLOBAL_FAILURE') or '').strip().split():
  128. savedirs[path] = ''
  129. for path in (d.getVar('RETAIN_DIRS_GLOBAL_ALWAYS') or '').strip().split():
  130. savedirs[path] = ''
  131. if savedirs:
  132. bb.plain('NOTE: retain: retaining build output...')
  133. count = 0
  134. for path, pn in savedirs.items():
  135. prefix = None
  136. if ';' in path:
  137. pathsplit = path.split(';')
  138. path = pathsplit[0]
  139. for param in pathsplit[1:]:
  140. if '=' in param:
  141. name, value = param.split('=', 1)
  142. if name == 'prefix':
  143. prefix = value
  144. else:
  145. bb.error('retain: invalid parameter "%s" in RETAIN_* variable value' % param)
  146. return
  147. else:
  148. bb.error('retain: parameter "%s" missing value in RETAIN_* variable value' % param)
  149. return
  150. if prefix:
  151. itemname = prefix
  152. else:
  153. itemname = os.path.basename(path)
  154. if pn:
  155. # Always add the recipe name in front
  156. itemname = pn + '_' + itemname
  157. if os.path.exists(path):
  158. retain_retain_dir(itemname, itemname, path, os.path.dirname(path), d)
  159. count += 1
  160. else:
  161. bb.warn('retain: path %s does not currently exist' % path)
  162. if count:
  163. item = 'archive' if count == 1 else 'archives'
  164. bb.plain('NOTE: retain: saved %d %s to %s' % (count, item, outdir))
  165. }