populate_sdk_ext.bbclass 32 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683
  1. # Extensible SDK
  2. inherit populate_sdk_base
  3. # NOTE: normally you cannot use task overrides for this kind of thing - this
  4. # only works because of get_sdk_ext_rdepends()
  5. TOOLCHAIN_HOST_TASK_task-populate-sdk-ext = " \
  6. meta-environment-extsdk-${MACHINE} \
  7. "
  8. TOOLCHAIN_TARGET_TASK_task-populate-sdk-ext = ""
  9. SDK_RDEPENDS_append_task-populate-sdk-ext = " ${SDK_TARGETS}"
  10. SDK_RELOCATE_AFTER_INSTALL_task-populate-sdk-ext = "0"
  11. SDK_EXT = ""
  12. SDK_EXT_task-populate-sdk-ext = "-ext"
  13. # Options are full or minimal
  14. SDK_EXT_TYPE ?= "full"
  15. SDK_INCLUDE_PKGDATA ?= "0"
  16. SDK_INCLUDE_TOOLCHAIN ?= "${@'1' if d.getVar('SDK_EXT_TYPE', True) == 'full' else '0'}"
  17. SDK_RECRDEP_TASKS ?= ""
  18. SDK_LOCAL_CONF_WHITELIST ?= ""
  19. SDK_LOCAL_CONF_BLACKLIST ?= "CONF_VERSION \
  20. BB_NUMBER_THREADS \
  21. BB_NUMBER_PARSE_THREADS \
  22. PARALLEL_MAKE \
  23. PRSERV_HOST \
  24. SSTATE_MIRRORS \
  25. DL_DIR \
  26. SSTATE_DIR \
  27. TMPDIR \
  28. "
  29. SDK_INHERIT_BLACKLIST ?= "buildhistory icecc"
  30. SDK_UPDATE_URL ?= ""
  31. SDK_TARGETS ?= "${PN}"
  32. def get_sdk_install_targets(d, images_only=False):
  33. sdk_install_targets = ''
  34. if images_only or d.getVar('SDK_EXT_TYPE', True) != 'minimal':
  35. sdk_install_targets = d.getVar('SDK_TARGETS', True)
  36. depd = d.getVar('BB_TASKDEPDATA', False)
  37. for v in depd.values():
  38. if v[1] == 'do_image_complete':
  39. if v[0] not in sdk_install_targets:
  40. sdk_install_targets += ' {}'.format(v[0])
  41. if not images_only:
  42. if d.getVar('SDK_INCLUDE_PKGDATA', True) == '1':
  43. sdk_install_targets += ' meta-world-pkgdata:do_allpackagedata'
  44. if d.getVar('SDK_INCLUDE_TOOLCHAIN', True) == '1':
  45. sdk_install_targets += ' meta-extsdk-toolchain:do_populate_sysroot'
  46. return sdk_install_targets
  47. get_sdk_install_targets[vardepsexclude] = "BB_TASKDEPDATA"
  48. OE_INIT_ENV_SCRIPT ?= "oe-init-build-env"
  49. # The files from COREBASE that you want preserved in the COREBASE copied
  50. # into the sdk. This allows someone to have their own setup scripts in
  51. # COREBASE be preserved as well as untracked files.
  52. COREBASE_FILES ?= " \
  53. oe-init-build-env \
  54. oe-init-build-env-memres \
  55. scripts \
  56. LICENSE \
  57. .templateconf \
  58. "
  59. SDK_DIR_task-populate-sdk-ext = "${WORKDIR}/sdk-ext"
  60. B_task-populate-sdk-ext = "${SDK_DIR}"
  61. TOOLCHAINEXT_OUTPUTNAME ?= "${SDK_NAME}-toolchain-ext-${SDK_VERSION}"
  62. TOOLCHAIN_OUTPUTNAME_task-populate-sdk-ext = "${TOOLCHAINEXT_OUTPUTNAME}"
  63. SDK_EXT_TARGET_MANIFEST = "${SDK_DEPLOY}/${TOOLCHAINEXT_OUTPUTNAME}.target.manifest"
  64. SDK_EXT_HOST_MANIFEST = "${SDK_DEPLOY}/${TOOLCHAINEXT_OUTPUTNAME}.host.manifest"
  65. SDK_TITLE_task-populate-sdk-ext = "${@d.getVar('DISTRO_NAME', True) or d.getVar('DISTRO', True)} Extensible SDK"
  66. def clean_esdk_builddir(d, sdkbasepath):
  67. """Clean up traces of the fake build for create_filtered_tasklist()"""
  68. import shutil
  69. cleanpaths = 'cache conf/sanity_info tmp'.split()
  70. for pth in cleanpaths:
  71. fullpth = os.path.join(sdkbasepath, pth)
  72. if os.path.isdir(fullpth):
  73. shutil.rmtree(fullpth)
  74. elif os.path.isfile(fullpth):
  75. os.remove(fullpth)
  76. def create_filtered_tasklist(d, sdkbasepath, tasklistfile, conf_initpath):
  77. """
  78. Create a filtered list of tasks. Also double-checks that the build system
  79. within the SDK basically works and required sstate artifacts are available.
  80. """
  81. import tempfile
  82. import shutil
  83. import oe.copy_buildsystem
  84. # Create a temporary build directory that we can pass to the env setup script
  85. shutil.copyfile(sdkbasepath + '/conf/local.conf', sdkbasepath + '/conf/local.conf.bak')
  86. try:
  87. with open(sdkbasepath + '/conf/local.conf', 'a') as f:
  88. # Force the use of sstate from the build system
  89. f.write('\nSSTATE_DIR_forcevariable = "%s"\n' % d.getVar('SSTATE_DIR', True))
  90. f.write('SSTATE_MIRRORS_forcevariable = ""\n')
  91. # Ensure TMPDIR is the default so that clean_esdk_builddir() can delete it
  92. f.write('TMPDIR_forcevariable = "${TOPDIR}/tmp"\n')
  93. # Drop uninative if the build isn't using it (or else NATIVELSBSTRING will
  94. # be different and we won't be able to find our native sstate)
  95. if not bb.data.inherits_class('uninative', d):
  96. f.write('INHERIT_remove = "uninative"\n')
  97. # Unfortunately the default SDKPATH (or even a custom value) may contain characters that bitbake
  98. # will not allow in its COREBASE path, so we need to rename the directory temporarily
  99. temp_sdkbasepath = d.getVar('SDK_OUTPUT', True) + '/tmp-renamed-sdk'
  100. # Delete any existing temp dir
  101. try:
  102. shutil.rmtree(temp_sdkbasepath)
  103. except FileNotFoundError:
  104. pass
  105. os.rename(sdkbasepath, temp_sdkbasepath)
  106. try:
  107. cmdprefix = '. %s .; ' % conf_initpath
  108. logfile = d.getVar('WORKDIR', True) + '/tasklist_bb_log.txt'
  109. try:
  110. oe.copy_buildsystem.check_sstate_task_list(d, get_sdk_install_targets(d), tasklistfile, cmdprefix=cmdprefix, cwd=temp_sdkbasepath, logfile=logfile)
  111. except bb.process.ExecutionError as e:
  112. msg = 'Failed to generate filtered task list for extensible SDK:\n%s' % e.stdout.rstrip()
  113. if 'attempted to execute unexpectedly and should have been setscened' in e.stdout:
  114. msg += '\n----------\n\nNOTE: "attempted to execute unexpectedly and should have been setscened" errors indicate this may be caused by missing sstate artifacts that were likely produced in earlier builds, but have been subsequently deleted for some reason.\n'
  115. bb.fatal(msg)
  116. finally:
  117. os.rename(temp_sdkbasepath, sdkbasepath)
  118. # Clean out residue of running bitbake, which check_sstate_task_list()
  119. # will effectively do
  120. clean_esdk_builddir(d, sdkbasepath)
  121. finally:
  122. os.replace(sdkbasepath + '/conf/local.conf.bak', sdkbasepath + '/conf/local.conf')
  123. python copy_buildsystem () {
  124. import re
  125. import shutil
  126. import glob
  127. import oe.copy_buildsystem
  128. oe_init_env_script = d.getVar('OE_INIT_ENV_SCRIPT', True)
  129. conf_bbpath = ''
  130. conf_initpath = ''
  131. core_meta_subdir = ''
  132. # Copy in all metadata layers + bitbake (as repositories)
  133. buildsystem = oe.copy_buildsystem.BuildSystem('extensible SDK', d)
  134. baseoutpath = d.getVar('SDK_OUTPUT', True) + '/' + d.getVar('SDKPATH', True)
  135. # Determine if we're building a derivative extensible SDK (from devtool build-sdk)
  136. derivative = (d.getVar('SDK_DERIVATIVE', True) or '') == '1'
  137. if derivative:
  138. workspace_name = 'orig-workspace'
  139. else:
  140. workspace_name = None
  141. layers_copied = buildsystem.copy_bitbake_and_layers(baseoutpath + '/layers', workspace_name)
  142. sdkbblayers = []
  143. corebase = os.path.basename(d.getVar('COREBASE', True))
  144. for layer in layers_copied:
  145. if corebase == os.path.basename(layer):
  146. conf_bbpath = os.path.join('layers', layer, 'bitbake')
  147. else:
  148. sdkbblayers.append(layer)
  149. for path in os.listdir(baseoutpath + '/layers'):
  150. relpath = os.path.join('layers', path, oe_init_env_script)
  151. if os.path.exists(os.path.join(baseoutpath, relpath)):
  152. conf_initpath = relpath
  153. relpath = os.path.join('layers', path, 'scripts', 'devtool')
  154. if os.path.exists(os.path.join(baseoutpath, relpath)):
  155. scriptrelpath = os.path.dirname(relpath)
  156. relpath = os.path.join('layers', path, 'meta')
  157. if os.path.exists(os.path.join(baseoutpath, relpath, 'lib', 'oe')):
  158. core_meta_subdir = relpath
  159. d.setVar('oe_init_build_env_path', conf_initpath)
  160. d.setVar('scriptrelpath', scriptrelpath)
  161. # Write out config file for devtool
  162. import configparser
  163. config = configparser.SafeConfigParser()
  164. config.add_section('General')
  165. config.set('General', 'bitbake_subdir', conf_bbpath)
  166. config.set('General', 'init_path', conf_initpath)
  167. config.set('General', 'core_meta_subdir', core_meta_subdir)
  168. config.add_section('SDK')
  169. config.set('SDK', 'sdk_targets', d.getVar('SDK_TARGETS', True))
  170. updateurl = d.getVar('SDK_UPDATE_URL', True)
  171. if updateurl:
  172. config.set('SDK', 'updateserver', updateurl)
  173. bb.utils.mkdirhier(os.path.join(baseoutpath, 'conf'))
  174. with open(os.path.join(baseoutpath, 'conf', 'devtool.conf'), 'w') as f:
  175. config.write(f)
  176. unlockedsigs = os.path.join(baseoutpath, 'conf', 'unlocked-sigs.inc')
  177. with open(unlockedsigs, 'w') as f:
  178. pass
  179. # Create a layer for new recipes / appends
  180. bbpath = d.getVar('BBPATH', True)
  181. bb.process.run(['devtool', '--bbpath', bbpath, '--basepath', baseoutpath, 'create-workspace', '--create-only', os.path.join(baseoutpath, 'workspace')])
  182. # Create bblayers.conf
  183. bb.utils.mkdirhier(baseoutpath + '/conf')
  184. with open(baseoutpath + '/conf/bblayers.conf', 'w') as f:
  185. f.write('# WARNING: this configuration has been automatically generated and in\n')
  186. f.write('# most cases should not be edited. If you need more flexibility than\n')
  187. f.write('# this configuration provides, it is strongly suggested that you set\n')
  188. f.write('# up a proper instance of the full build system and use that instead.\n\n')
  189. # LCONF_VERSION may not be set, for example when using meta-poky
  190. # so don't error if it isn't found
  191. lconf_version = d.getVar('LCONF_VERSION', False)
  192. if lconf_version is not None:
  193. f.write('LCONF_VERSION = "%s"\n\n' % lconf_version)
  194. f.write('BBPATH = "$' + '{TOPDIR}"\n')
  195. f.write('SDKBASEMETAPATH = "$' + '{TOPDIR}"\n')
  196. f.write('BBLAYERS := " \\\n')
  197. for layerrelpath in sdkbblayers:
  198. f.write(' $' + '{SDKBASEMETAPATH}/layers/%s \\\n' % layerrelpath)
  199. f.write(' $' + '{SDKBASEMETAPATH}/workspace \\\n')
  200. f.write(' "\n')
  201. # Copy uninative tarball
  202. # For now this is where uninative.bbclass expects the tarball
  203. uninative_file = d.expand('${SDK_DEPLOY}/${BUILD_ARCH}-nativesdk-libc.tar.bz2')
  204. uninative_checksum = bb.utils.sha256_file(uninative_file)
  205. uninative_outdir = '%s/downloads/uninative/%s' % (baseoutpath, uninative_checksum)
  206. bb.utils.mkdirhier(uninative_outdir)
  207. shutil.copy(uninative_file, uninative_outdir)
  208. env_whitelist = (d.getVar('BB_ENV_EXTRAWHITE', True) or '').split()
  209. env_whitelist_values = {}
  210. # Create local.conf
  211. builddir = d.getVar('TOPDIR', True)
  212. if derivative:
  213. shutil.copyfile(builddir + '/conf/local.conf', baseoutpath + '/conf/local.conf')
  214. else:
  215. local_conf_whitelist = (d.getVar('SDK_LOCAL_CONF_WHITELIST', True) or '').split()
  216. local_conf_blacklist = (d.getVar('SDK_LOCAL_CONF_BLACKLIST', True) or '').split()
  217. def handle_var(varname, origvalue, op, newlines):
  218. if varname in local_conf_blacklist or (origvalue.strip().startswith('/') and not varname in local_conf_whitelist):
  219. newlines.append('# Removed original setting of %s\n' % varname)
  220. return None, op, 0, True
  221. else:
  222. if varname in env_whitelist:
  223. env_whitelist_values[varname] = origvalue
  224. return origvalue, op, 0, True
  225. varlist = ['[^#=+ ]*']
  226. with open(builddir + '/conf/local.conf', 'r') as f:
  227. oldlines = f.readlines()
  228. (updated, newlines) = bb.utils.edit_metadata(oldlines, varlist, handle_var)
  229. with open(baseoutpath + '/conf/local.conf', 'w') as f:
  230. f.write('# WARNING: this configuration has been automatically generated and in\n')
  231. f.write('# most cases should not be edited. If you need more flexibility than\n')
  232. f.write('# this configuration provides, it is strongly suggested that you set\n')
  233. f.write('# up a proper instance of the full build system and use that instead.\n\n')
  234. for line in newlines:
  235. if line.strip() and not line.startswith('#'):
  236. f.write(line)
  237. # Write a newline just in case there's none at the end of the original
  238. f.write('\n')
  239. f.write('DL_DIR = "${TOPDIR}/downloads"\n')
  240. f.write('INHERIT += "%s"\n' % 'uninative')
  241. f.write('UNINATIVE_CHECKSUM[%s] = "%s"\n\n' % (d.getVar('BUILD_ARCH', True), uninative_checksum))
  242. f.write('CONF_VERSION = "%s"\n\n' % d.getVar('CONF_VERSION', False))
  243. # Some classes are not suitable for SDK, remove them from INHERIT
  244. f.write('INHERIT_remove = "%s"\n' % d.getVar('SDK_INHERIT_BLACKLIST', False))
  245. # Bypass the default connectivity check if any
  246. f.write('CONNECTIVITY_CHECK_URIS = ""\n\n')
  247. # This warning will come out if reverse dependencies for a task
  248. # don't have sstate as well as the task itself. We already know
  249. # this will be the case for the extensible sdk, so turn off the
  250. # warning.
  251. f.write('SIGGEN_LOCKEDSIGS_SSTATE_EXISTS_CHECK = "none"\n\n')
  252. # Warn if the sigs in the locked-signature file don't match
  253. # the sig computed from the metadata.
  254. f.write('SIGGEN_LOCKEDSIGS_TASKSIG_CHECK = "warn"\n\n')
  255. # Set up whitelist for run on install
  256. f.write('BB_SETSCENE_ENFORCE_WHITELIST = "%:* *:do_shared_workdir *:do_rm_work"\n\n')
  257. # Hide the config information from bitbake output (since it's fixed within the SDK)
  258. f.write('BUILDCFG_HEADER = ""\n')
  259. # Allow additional config through sdk-extra.conf
  260. fn = bb.cookerdata.findConfigFile('sdk-extra.conf', d)
  261. if fn:
  262. with open(fn, 'r') as xf:
  263. for line in xf:
  264. f.write(line)
  265. # If you define a sdk_extraconf() function then it can contain additional config
  266. # (Though this is awkward; sdk-extra.conf should probably be used instead)
  267. extraconf = (d.getVar('sdk_extraconf', True) or '').strip()
  268. if extraconf:
  269. # Strip off any leading / trailing spaces
  270. for line in extraconf.splitlines():
  271. f.write(line.strip() + '\n')
  272. f.write('require conf/locked-sigs.inc\n')
  273. f.write('require conf/unlocked-sigs.inc\n')
  274. if os.path.exists(builddir + '/conf/auto.conf'):
  275. if derivative:
  276. shutil.copyfile(builddir + '/conf/auto.conf', baseoutpath + '/conf/auto.conf')
  277. else:
  278. with open(builddir + '/conf/auto.conf', 'r') as f:
  279. oldlines = f.readlines()
  280. (updated, newlines) = bb.utils.edit_metadata(oldlines, varlist, handle_var)
  281. with open(baseoutpath + '/conf/auto.conf', 'w') as f:
  282. f.write('# WARNING: this configuration has been automatically generated and in\n')
  283. f.write('# most cases should not be edited. If you need more flexibility than\n')
  284. f.write('# this configuration provides, it is strongly suggested that you set\n')
  285. f.write('# up a proper instance of the full build system and use that instead.\n\n')
  286. for line in newlines:
  287. if line.strip() and not line.startswith('#'):
  288. f.write(line)
  289. # Write a templateconf.cfg
  290. with open(baseoutpath + '/conf/templateconf.cfg', 'w') as f:
  291. f.write('meta/conf\n')
  292. # Ensure any variables set from the external environment (by way of
  293. # BB_ENV_EXTRAWHITE) are set in the SDK's configuration
  294. extralines = []
  295. for name, value in env_whitelist_values.items():
  296. actualvalue = d.getVar(name, True) or ''
  297. if value != actualvalue:
  298. extralines.append('%s = "%s"\n' % (name, actualvalue))
  299. if extralines:
  300. with open(baseoutpath + '/conf/local.conf', 'a') as f:
  301. f.write('\n')
  302. f.write('# Extra settings from environment:\n')
  303. for line in extralines:
  304. f.write(line)
  305. f.write('\n')
  306. # Filter the locked signatures file to just the sstate tasks we are interested in
  307. excluded_targets = get_sdk_install_targets(d, images_only=True)
  308. sigfile = d.getVar('WORKDIR', True) + '/locked-sigs.inc'
  309. lockedsigs_pruned = baseoutpath + '/conf/locked-sigs.inc'
  310. oe.copy_buildsystem.prune_lockedsigs([],
  311. excluded_targets.split(),
  312. sigfile,
  313. lockedsigs_pruned)
  314. sstate_out = baseoutpath + '/sstate-cache'
  315. bb.utils.remove(sstate_out, True)
  316. # uninative.bbclass sets NATIVELSBSTRING to 'universal'
  317. fixedlsbstring = 'universal'
  318. sdk_include_toolchain = (d.getVar('SDK_INCLUDE_TOOLCHAIN', True) == '1')
  319. sdk_ext_type = d.getVar('SDK_EXT_TYPE', True)
  320. if sdk_ext_type != 'minimal' or sdk_include_toolchain or derivative:
  321. # Create the filtered task list used to generate the sstate cache shipped with the SDK
  322. tasklistfn = d.getVar('WORKDIR', True) + '/tasklist.txt'
  323. create_filtered_tasklist(d, baseoutpath, tasklistfn, conf_initpath)
  324. else:
  325. tasklistfn = None
  326. # Add packagedata if enabled
  327. if d.getVar('SDK_INCLUDE_PKGDATA', True) == '1':
  328. lockedsigs_base = d.getVar('WORKDIR', True) + '/locked-sigs-base.inc'
  329. lockedsigs_copy = d.getVar('WORKDIR', True) + '/locked-sigs-copy.inc'
  330. shutil.move(lockedsigs_pruned, lockedsigs_base)
  331. oe.copy_buildsystem.merge_lockedsigs(['do_packagedata'],
  332. lockedsigs_base,
  333. d.getVar('STAGING_DIR_HOST', True) + '/world-pkgdata/locked-sigs-pkgdata.inc',
  334. lockedsigs_pruned,
  335. lockedsigs_copy)
  336. if sdk_include_toolchain:
  337. lockedsigs_base = d.getVar('WORKDIR', True) + '/locked-sigs-base2.inc'
  338. lockedsigs_toolchain = d.getVar('STAGING_DIR_HOST', True) + '/locked-sigs/locked-sigs-extsdk-toolchain.inc'
  339. shutil.move(lockedsigs_pruned, lockedsigs_base)
  340. oe.copy_buildsystem.merge_lockedsigs([],
  341. lockedsigs_base,
  342. lockedsigs_toolchain,
  343. lockedsigs_pruned)
  344. oe.copy_buildsystem.create_locked_sstate_cache(lockedsigs_toolchain,
  345. d.getVar('SSTATE_DIR', True),
  346. sstate_out, d,
  347. fixedlsbstring,
  348. filterfile=tasklistfn)
  349. if sdk_ext_type == 'minimal':
  350. if derivative:
  351. # Assume the user is not going to set up an additional sstate
  352. # mirror, thus we need to copy the additional artifacts (from
  353. # workspace recipes) into the derivative SDK
  354. lockedsigs_orig = d.getVar('TOPDIR', True) + '/conf/locked-sigs.inc'
  355. if os.path.exists(lockedsigs_orig):
  356. lockedsigs_extra = d.getVar('WORKDIR', True) + '/locked-sigs-extra.inc'
  357. oe.copy_buildsystem.merge_lockedsigs(None,
  358. lockedsigs_orig,
  359. lockedsigs_pruned,
  360. None,
  361. lockedsigs_extra)
  362. oe.copy_buildsystem.create_locked_sstate_cache(lockedsigs_extra,
  363. d.getVar('SSTATE_DIR', True),
  364. sstate_out, d,
  365. fixedlsbstring,
  366. filterfile=tasklistfn)
  367. else:
  368. oe.copy_buildsystem.create_locked_sstate_cache(lockedsigs_pruned,
  369. d.getVar('SSTATE_DIR', True),
  370. sstate_out, d,
  371. fixedlsbstring,
  372. filterfile=tasklistfn)
  373. # We don't need sstate do_package files
  374. for root, dirs, files in os.walk(sstate_out):
  375. for name in files:
  376. if name.endswith("_package.tgz"):
  377. f = os.path.join(root, name)
  378. os.remove(f)
  379. # Write manifest file
  380. # Note: at the moment we cannot include the env setup script here to keep
  381. # it updated, since it gets modified during SDK installation (see
  382. # sdk_ext_postinst() below) thus the checksum we take here would always
  383. # be different.
  384. manifest_file_list = ['conf/*']
  385. manifest_file = os.path.join(baseoutpath, 'conf', 'sdk-conf-manifest')
  386. with open(manifest_file, 'w') as f:
  387. for item in manifest_file_list:
  388. for fn in glob.glob(os.path.join(baseoutpath, item)):
  389. if fn == manifest_file:
  390. continue
  391. chksum = bb.utils.sha256_file(fn)
  392. f.write('%s\t%s\n' % (chksum, os.path.relpath(fn, baseoutpath)))
  393. }
  394. def get_current_buildtools(d):
  395. """Get the file name of the current buildtools installer"""
  396. import glob
  397. btfiles = glob.glob(os.path.join(d.getVar('SDK_DEPLOY', True), '*-buildtools-nativesdk-standalone-*.sh'))
  398. btfiles.sort(key=os.path.getctime)
  399. return os.path.basename(btfiles[-1])
  400. def get_sdk_required_utilities(buildtools_fn, d):
  401. """Find required utilities that aren't provided by the buildtools"""
  402. sanity_required_utilities = (d.getVar('SANITY_REQUIRED_UTILITIES', True) or '').split()
  403. sanity_required_utilities.append(d.expand('${BUILD_PREFIX}gcc'))
  404. sanity_required_utilities.append(d.expand('${BUILD_PREFIX}g++'))
  405. buildtools_installer = os.path.join(d.getVar('SDK_DEPLOY', True), buildtools_fn)
  406. filelist, _ = bb.process.run('%s -l' % buildtools_installer)
  407. localdata = bb.data.createCopy(d)
  408. localdata.setVar('SDKPATH', '.')
  409. sdkpathnative = localdata.getVar('SDKPATHNATIVE', True)
  410. sdkbindirs = [localdata.getVar('bindir_nativesdk', True),
  411. localdata.getVar('sbindir_nativesdk', True),
  412. localdata.getVar('base_bindir_nativesdk', True),
  413. localdata.getVar('base_sbindir_nativesdk', True)]
  414. for line in filelist.splitlines():
  415. splitline = line.split()
  416. if len(splitline) > 5:
  417. fn = splitline[5]
  418. if not fn.startswith('./'):
  419. fn = './%s' % fn
  420. if fn.startswith(sdkpathnative):
  421. relpth = '/' + os.path.relpath(fn, sdkpathnative)
  422. for bindir in sdkbindirs:
  423. if relpth.startswith(bindir):
  424. relpth = os.path.relpath(relpth, bindir)
  425. if relpth in sanity_required_utilities:
  426. sanity_required_utilities.remove(relpth)
  427. break
  428. return ' '.join(sanity_required_utilities)
  429. install_tools() {
  430. install -d ${SDK_OUTPUT}/${SDKPATHNATIVE}${bindir_nativesdk}
  431. scripts="devtool recipetool oe-find-native-sysroot runqemu*"
  432. for script in $scripts; do
  433. for scriptfn in `find ${SDK_OUTPUT}/${SDKPATH}/${scriptrelpath} -maxdepth 1 -executable -name "$script"`; do
  434. lnr ${scriptfn} ${SDK_OUTPUT}/${SDKPATHNATIVE}${bindir_nativesdk}/`basename $scriptfn`
  435. done
  436. done
  437. # We can't use the same method as above because files in the sysroot won't exist at this point
  438. # (they get populated from sstate on installation)
  439. unfsd_path="${SDK_OUTPUT}/${SDKPATHNATIVE}${bindir_nativesdk}/unfsd"
  440. if [ "${SDK_INCLUDE_TOOLCHAIN}" == "1" -a ! -e $unfsd_path ] ; then
  441. binrelpath=${@os.path.relpath(d.getVar('STAGING_BINDIR_NATIVE',True), d.getVar('TOPDIR', True))}
  442. lnr ${SDK_OUTPUT}/${SDKPATH}/$binrelpath/unfsd $unfsd_path
  443. fi
  444. touch ${SDK_OUTPUT}/${SDKPATH}/.devtoolbase
  445. # find latest buildtools-tarball and install it
  446. install ${SDK_DEPLOY}/${SDK_BUILDTOOLS_INSTALLER} ${SDK_OUTPUT}/${SDKPATH}
  447. install -m 0644 ${COREBASE}/meta/files/ext-sdk-prepare.py ${SDK_OUTPUT}/${SDKPATH}
  448. }
  449. do_populate_sdk_ext[file-checksums] += "${COREBASE}/meta/files/ext-sdk-prepare.py:True"
  450. sdk_ext_preinst() {
  451. # Since bitbake won't run as root it doesn't make sense to try and install
  452. # the extensible sdk as root.
  453. if [ "`id -u`" = "0" ]; then
  454. echo "ERROR: The extensible sdk cannot be installed as root."
  455. exit 1
  456. fi
  457. if ! command -v locale > /dev/null; then
  458. echo "ERROR: The installer requires the locale command, please install it first"
  459. exit 1
  460. fi
  461. # Check setting of LC_ALL set above
  462. canonicalised_locale=`echo $LC_ALL | sed 's/UTF-8/utf8/'`
  463. if ! locale -a | grep -q $canonicalised_locale ; then
  464. echo "ERROR: the installer requires the $LC_ALL locale to be installed (but not selected), please install it first"
  465. exit 1
  466. fi
  467. # The relocation script used by buildtools installer requires python
  468. if ! command -v python > /dev/null; then
  469. echo "ERROR: The installer requires python, please install it first"
  470. exit 1
  471. fi
  472. missing_utils=""
  473. for util in ${SDK_REQUIRED_UTILITIES}; do
  474. if ! command -v $util > /dev/null; then
  475. missing_utils="$missing_utils $util"
  476. fi
  477. done
  478. if [ -n "$missing_utils" ] ; then
  479. echo "ERROR: the SDK requires the following missing utilities, please install them: $missing_utils"
  480. exit 1
  481. fi
  482. SDK_EXTENSIBLE="1"
  483. if [ "$publish" = "1" ] ; then
  484. EXTRA_TAR_OPTIONS="$EXTRA_TAR_OPTIONS --exclude=ext-sdk-prepare.py"
  485. if [ "${SDK_EXT_TYPE}" = "minimal" ] ; then
  486. EXTRA_TAR_OPTIONS="$EXTRA_TAR_OPTIONS --exclude=sstate-cache"
  487. fi
  488. fi
  489. }
  490. SDK_PRE_INSTALL_COMMAND_task-populate-sdk-ext = "${sdk_ext_preinst}"
  491. # FIXME this preparation should be done as part of the SDK construction
  492. sdk_ext_postinst() {
  493. printf "\nExtracting buildtools...\n"
  494. cd $target_sdk_dir
  495. env_setup_script="$target_sdk_dir/environment-setup-${REAL_MULTIMACH_TARGET_SYS}"
  496. printf "buildtools\ny" | ./${SDK_BUILDTOOLS_INSTALLER} > buildtools.log || { printf 'ERROR: buildtools installation failed:\n' ; cat buildtools.log ; echo "printf 'ERROR: this SDK was not fully installed and needs reinstalling\n'" >> $env_setup_script ; exit 1 ; }
  497. # Delete the buildtools tar file since it won't be used again
  498. rm -f ./${SDK_BUILDTOOLS_INSTALLER}
  499. # We don't need the log either since it succeeded
  500. rm -f buildtools.log
  501. # Make sure when the user sets up the environment, they also get
  502. # the buildtools-tarball tools in their path.
  503. echo ". $target_sdk_dir/buildtools/environment-setup*" >> $env_setup_script
  504. # Allow bitbake environment setup to be ran as part of this sdk.
  505. echo "export OE_SKIP_SDK_CHECK=1" >> $env_setup_script
  506. # A bit of another hack, but we need this in the path only for devtool
  507. # so put it at the end of $PATH.
  508. echo "export PATH=$target_sdk_dir/sysroots/${SDK_SYS}${bindir_nativesdk}:\$PATH" >> $env_setup_script
  509. echo "printf 'SDK environment now set up; additionally you may now run devtool to perform development tasks.\nRun devtool --help for further details.\n'" >> $env_setup_script
  510. # Warn if trying to use external bitbake and the ext SDK together
  511. echo "(which bitbake > /dev/null 2>&1 && echo 'WARNING: attempting to use the extensible SDK in an environment set up to run bitbake - this may lead to unexpected results. Please source this script in a new shell session instead.') || true" >> $env_setup_script
  512. if [ "$prepare_buildsystem" != "no" ]; then
  513. printf "Preparing build system...\n"
  514. # dash which is /bin/sh on Ubuntu will not preserve the
  515. # current working directory when first ran, nor will it set $1 when
  516. # sourcing a script. That is why this has to look so ugly.
  517. LOGFILE="$target_sdk_dir/preparing_build_system.log"
  518. sh -c ". buildtools/environment-setup* > $LOGFILE && cd $target_sdk_dir/`dirname ${oe_init_build_env_path}` && set $target_sdk_dir && . $target_sdk_dir/${oe_init_build_env_path} $target_sdk_dir >> $LOGFILE && python $target_sdk_dir/ext-sdk-prepare.py $LOGFILE '${SDK_INSTALL_TARGETS}'" || { echo "printf 'ERROR: this SDK was not fully installed and needs reinstalling\n'" >> $env_setup_script ; exit 1 ; }
  519. rm $target_sdk_dir/ext-sdk-prepare.py
  520. fi
  521. echo done
  522. }
  523. SDK_POST_INSTALL_COMMAND_task-populate-sdk-ext = "${sdk_ext_postinst}"
  524. SDK_POSTPROCESS_COMMAND_prepend_task-populate-sdk-ext = "copy_buildsystem; install_tools; "
  525. SDK_INSTALL_TARGETS = ""
  526. fakeroot python do_populate_sdk_ext() {
  527. # FIXME hopefully we can remove this restriction at some point, but uninative
  528. # currently forces this upon us
  529. if d.getVar('SDK_ARCH', True) != d.getVar('BUILD_ARCH', True):
  530. bb.fatal('The extensible SDK can currently only be built for the same architecture as the machine being built on - SDK_ARCH is set to %s (likely via setting SDKMACHINE) which is different from the architecture of the build machine (%s). Unable to continue.' % (d.getVar('SDK_ARCH', True), d.getVar('BUILD_ARCH', True)))
  531. d.setVar('SDK_INSTALL_TARGETS', get_sdk_install_targets(d))
  532. buildtools_fn = get_current_buildtools(d)
  533. d.setVar('SDK_REQUIRED_UTILITIES', get_sdk_required_utilities(buildtools_fn, d))
  534. d.setVar('SDK_BUILDTOOLS_INSTALLER', buildtools_fn)
  535. d.setVar('SDKDEPLOYDIR', '${SDKEXTDEPLOYDIR}')
  536. populate_sdk_common(d)
  537. }
  538. def get_ext_sdk_depends(d):
  539. # Note: the deps varflag is a list not a string, so we need to specify expand=False
  540. deps = d.getVarFlag('do_image_complete', 'deps', False)
  541. pn = d.getVar('PN', True)
  542. deplist = ['%s:%s' % (pn, dep) for dep in deps]
  543. for task in ['do_image_complete', 'do_rootfs', 'do_build']:
  544. deplist.extend((d.getVarFlag(task, 'depends', True) or '').split())
  545. return ' '.join(deplist)
  546. python do_sdk_depends() {
  547. # We have to do this separately in its own task so we avoid recursing into
  548. # dependencies we don't need to (e.g. buildtools-tarball) and bringing those
  549. # into the SDK's sstate-cache
  550. import oe.copy_buildsystem
  551. sigfile = d.getVar('WORKDIR', True) + '/locked-sigs.inc'
  552. oe.copy_buildsystem.generate_locked_sigs(sigfile, d)
  553. }
  554. addtask sdk_depends
  555. do_sdk_depends[dirs] = "${WORKDIR}"
  556. do_sdk_depends[depends] = "${@get_ext_sdk_depends(d)}"
  557. do_sdk_depends[recrdeptask] = "${@d.getVarFlag('do_populate_sdk', 'recrdeptask', False)}"
  558. do_sdk_depends[recrdeptask] += "do_populate_lic do_package_qa do_populate_sysroot do_deploy ${SDK_RECRDEP_TASKS}"
  559. do_sdk_depends[rdepends] = "${@get_sdk_ext_rdepends(d)}"
  560. def get_sdk_ext_rdepends(d):
  561. localdata = d.createCopy()
  562. localdata.appendVar('OVERRIDES', ':task-populate-sdk-ext')
  563. bb.data.update_data(localdata)
  564. return localdata.getVarFlag('do_populate_sdk', 'rdepends', True)
  565. do_populate_sdk_ext[dirs] = "${@d.getVarFlag('do_populate_sdk', 'dirs', False)}"
  566. do_populate_sdk_ext[depends] = "${@d.getVarFlag('do_populate_sdk', 'depends', False)} \
  567. buildtools-tarball:do_populate_sdk uninative-tarball:do_populate_sdk \
  568. ${@'meta-world-pkgdata:do_collect_packagedata' if d.getVar('SDK_INCLUDE_PKGDATA', True) == '1' else ''} \
  569. ${@'meta-extsdk-toolchain:do_locked_sigs' if d.getVar('SDK_INCLUDE_TOOLCHAIN', True) == '1' else ''}"
  570. do_populate_sdk_ext[rdepends] += "${@' '.join([x + ':do_build' for x in d.getVar('SDK_TARGETS', True).split()])}"
  571. # Make sure code changes can result in rebuild
  572. do_populate_sdk_ext[vardeps] += "copy_buildsystem \
  573. sdk_ext_postinst"
  574. # Since any change in the metadata of any layer should cause a rebuild of the
  575. # sdk(since the layers are put in the sdk) set the task to nostamp so it
  576. # always runs.
  577. do_populate_sdk_ext[nostamp] = "1"
  578. SDKEXTDEPLOYDIR = "${WORKDIR}/deploy-${PN}-populate-sdk-ext"
  579. SSTATETASKS += "do_populate_sdk_ext"
  580. SSTATE_SKIP_CREATION_task-populate-sdk-ext = '1'
  581. do_populate_sdk_ext[cleandirs] = "${SDKDEPLOYDIR}"
  582. do_populate_sdk_ext[sstate-inputdirs] = "${SDKEXTDEPLOYDIR}"
  583. do_populate_sdk_ext[sstate-outputdirs] = "${SDK_DEPLOY}"
  584. do_populate_sdk_ext[stamp-extra-info] = "${MACHINE}"
  585. addtask populate_sdk_ext after do_sdk_depends