staging.bbclass 27 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702
  1. #
  2. # Copyright OpenEmbedded Contributors
  3. #
  4. # SPDX-License-Identifier: MIT
  5. #
  6. # These directories will be staged in the sysroot
  7. SYSROOT_DIRS = " \
  8. ${includedir} \
  9. ${libdir} \
  10. ${base_libdir} \
  11. ${nonarch_base_libdir} \
  12. ${datadir} \
  13. /sysroot-only \
  14. "
  15. # These directories are also staged in the sysroot when they contain files that
  16. # are usable on the build system
  17. SYSROOT_DIRS_NATIVE = " \
  18. ${bindir} \
  19. ${sbindir} \
  20. ${base_bindir} \
  21. ${base_sbindir} \
  22. ${libexecdir} \
  23. ${sysconfdir} \
  24. ${localstatedir} \
  25. "
  26. SYSROOT_DIRS:append:class-native = " ${SYSROOT_DIRS_NATIVE}"
  27. SYSROOT_DIRS:append:class-cross = " ${SYSROOT_DIRS_NATIVE}"
  28. SYSROOT_DIRS:append:class-crosssdk = " ${SYSROOT_DIRS_NATIVE}"
  29. # These directories will not be staged in the sysroot
  30. SYSROOT_DIRS_IGNORE = " \
  31. ${mandir} \
  32. ${docdir} \
  33. ${infodir} \
  34. ${datadir}/X11/locale \
  35. ${datadir}/applications \
  36. ${datadir}/bash-completion \
  37. ${datadir}/fonts \
  38. ${datadir}/gtk-doc/html \
  39. ${datadir}/installed-tests \
  40. ${datadir}/locale \
  41. ${datadir}/pixmaps \
  42. ${datadir}/terminfo \
  43. ${libdir}/${BPN}/ptest \
  44. "
  45. sysroot_stage_dir() {
  46. src="$1"
  47. dest="$2"
  48. # if the src doesn't exist don't do anything
  49. if [ ! -d "$src" ]; then
  50. return
  51. fi
  52. mkdir -p "$dest"
  53. rdest=$(realpath --relative-to="$src" "$dest")
  54. (
  55. cd $src
  56. find . -print0 | cpio --null -pdlu $rdest
  57. )
  58. }
  59. sysroot_stage_dirs() {
  60. from="$1"
  61. to="$2"
  62. for dir in ${SYSROOT_DIRS}; do
  63. sysroot_stage_dir "$from$dir" "$to$dir"
  64. done
  65. # Remove directories we do not care about
  66. for dir in ${SYSROOT_DIRS_IGNORE}; do
  67. rm -rf "$to$dir"
  68. done
  69. }
  70. sysroot_stage_all() {
  71. sysroot_stage_dirs ${D} ${SYSROOT_DESTDIR}
  72. }
  73. python sysroot_strip () {
  74. inhibit_sysroot = d.getVar('INHIBIT_SYSROOT_STRIP')
  75. if inhibit_sysroot and oe.types.boolean(inhibit_sysroot):
  76. return
  77. dstdir = d.getVar('SYSROOT_DESTDIR')
  78. pn = d.getVar('PN')
  79. libdir = d.getVar("libdir")
  80. base_libdir = d.getVar("base_libdir")
  81. qa_already_stripped = 'already-stripped' in (d.getVar('INSANE_SKIP:' + pn) or "").split()
  82. strip_cmd = d.getVar("STRIP")
  83. max_process = oe.utils.get_bb_number_threads(d)
  84. oe.package.strip_execs(pn, dstdir, strip_cmd, libdir, base_libdir, max_process,
  85. qa_already_stripped=qa_already_stripped)
  86. }
  87. do_populate_sysroot[dirs] = "${SYSROOT_DESTDIR}"
  88. addtask populate_sysroot after do_install
  89. SYSROOT_PREPROCESS_FUNCS ?= ""
  90. SYSROOT_DESTDIR = "${WORKDIR}/sysroot-destdir"
  91. python do_populate_sysroot () {
  92. # SYSROOT 'version' 2
  93. bb.build.exec_func("sysroot_stage_all", d)
  94. bb.build.exec_func("sysroot_strip", d)
  95. for f in (d.getVar('SYSROOT_PREPROCESS_FUNCS') or '').split():
  96. bb.build.exec_func(f, d)
  97. pn = d.getVar("PN")
  98. multiprov = d.getVar("BB_MULTI_PROVIDER_ALLOWED").split()
  99. provdir = d.expand("${SYSROOT_DESTDIR}${base_prefix}/sysroot-providers/")
  100. bb.utils.mkdirhier(provdir)
  101. for p in d.getVar("PROVIDES").split():
  102. if p in multiprov:
  103. continue
  104. p = p.replace("/", "_")
  105. with open(provdir + p, "w") as f:
  106. f.write(pn)
  107. }
  108. do_populate_sysroot[vardeps] += "${SYSROOT_PREPROCESS_FUNCS}"
  109. do_populate_sysroot[vardepsexclude] += "BB_MULTI_PROVIDER_ALLOWED"
  110. POPULATESYSROOTDEPS = ""
  111. POPULATESYSROOTDEPS:class-target = "virtual/cross-binutils:do_populate_sysroot"
  112. POPULATESYSROOTDEPS:class-nativesdk = "virtual/nativesdk-cross-binutils:do_populate_sysroot"
  113. do_populate_sysroot[depends] += "${POPULATESYSROOTDEPS}"
  114. SSTATETASKS += "do_populate_sysroot"
  115. do_populate_sysroot[cleandirs] = "${SYSROOT_DESTDIR}"
  116. do_populate_sysroot[sstate-inputdirs] = "${SYSROOT_DESTDIR}"
  117. do_populate_sysroot[sstate-outputdirs] = "${COMPONENTS_DIR}/${PACKAGE_ARCH}/${PN}"
  118. do_populate_sysroot[sstate-fixmedir] = "${COMPONENTS_DIR}/${PACKAGE_ARCH}/${PN}"
  119. python do_populate_sysroot_setscene () {
  120. sstate_setscene(d)
  121. }
  122. addtask do_populate_sysroot_setscene
  123. def staging_copyfile(c, target, dest, postinsts, seendirs):
  124. import errno
  125. destdir = os.path.dirname(dest)
  126. if destdir not in seendirs:
  127. bb.utils.mkdirhier(destdir)
  128. seendirs.add(destdir)
  129. if "/usr/bin/postinst-" in c:
  130. postinsts.append(dest)
  131. if os.path.islink(c):
  132. linkto = os.readlink(c)
  133. if os.path.lexists(dest):
  134. if not os.path.islink(dest):
  135. raise OSError(errno.EEXIST, "Link %s already exists as a file" % dest, dest)
  136. if os.readlink(dest) == linkto:
  137. return dest
  138. raise OSError(errno.EEXIST, "Link %s already exists to a different location? (%s vs %s)" % (dest, os.readlink(dest), linkto), dest)
  139. os.symlink(linkto, dest)
  140. #bb.warn(c)
  141. else:
  142. try:
  143. os.link(c, dest)
  144. except OSError as err:
  145. if err.errno == errno.EXDEV:
  146. bb.utils.copyfile(c, dest)
  147. else:
  148. raise
  149. return dest
  150. def staging_copydir(c, target, dest, seendirs):
  151. if dest not in seendirs:
  152. bb.utils.mkdirhier(dest)
  153. seendirs.add(dest)
  154. def staging_processfixme(fixme, target, recipesysroot, recipesysrootnative, d):
  155. import subprocess
  156. if not fixme:
  157. return
  158. cmd = "sed -e 's:^[^/]*/:%s/:g' %s | xargs sed -i -e 's:FIXMESTAGINGDIRTARGET:%s:g; s:FIXMESTAGINGDIRHOST:%s:g'" % (target, " ".join(fixme), recipesysroot, recipesysrootnative)
  159. for fixmevar in ['PSEUDO_SYSROOT', 'HOSTTOOLS_DIR', 'PKGDATA_DIR', 'PSEUDO_LOCALSTATEDIR', 'LOGFIFO']:
  160. fixme_path = d.getVar(fixmevar)
  161. cmd += " -e 's:FIXME_%s:%s:g'" % (fixmevar, fixme_path)
  162. bb.debug(2, cmd)
  163. subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT)
  164. def staging_populate_sysroot_dir(targetsysroot, nativesysroot, native, d):
  165. import glob
  166. import subprocess
  167. import errno
  168. fixme = []
  169. postinsts = []
  170. seendirs = set()
  171. stagingdir = d.getVar("STAGING_DIR")
  172. if native:
  173. pkgarchs = ['${BUILD_ARCH}', '${BUILD_ARCH}_*']
  174. targetdir = nativesysroot
  175. else:
  176. pkgarchs = ['${MACHINE_ARCH}']
  177. pkgarchs = pkgarchs + list(reversed(d.getVar("PACKAGE_EXTRA_ARCHS").split()))
  178. pkgarchs.append('allarch')
  179. targetdir = targetsysroot
  180. bb.utils.mkdirhier(targetdir)
  181. for pkgarch in pkgarchs:
  182. for manifest in glob.glob(d.expand("${SSTATE_MANIFESTS}/manifest-%s-*.populate_sysroot" % pkgarch)):
  183. if manifest.endswith("-initial.populate_sysroot"):
  184. # skip libgcc-initial due to file overlap
  185. continue
  186. if not native and (manifest.endswith("-native.populate_sysroot") or "nativesdk-" in manifest):
  187. continue
  188. if native and not (manifest.endswith("-native.populate_sysroot") or manifest.endswith("-cross.populate_sysroot") or "-cross-" in manifest):
  189. continue
  190. tmanifest = targetdir + "/" + os.path.basename(manifest)
  191. if os.path.exists(tmanifest):
  192. continue
  193. try:
  194. os.link(manifest, tmanifest)
  195. except OSError as err:
  196. if err.errno == errno.EXDEV:
  197. bb.utils.copyfile(manifest, tmanifest)
  198. else:
  199. raise
  200. with open(manifest, "r") as f:
  201. for l in f:
  202. l = l.strip()
  203. if l.endswith("/fixmepath"):
  204. fixme.append(l)
  205. continue
  206. if l.endswith("/fixmepath.cmd"):
  207. continue
  208. dest = l.replace(stagingdir, "")
  209. dest = targetdir + "/" + "/".join(dest.split("/")[3:])
  210. if l.endswith("/"):
  211. staging_copydir(l, targetdir, dest, seendirs)
  212. continue
  213. try:
  214. staging_copyfile(l, targetdir, dest, postinsts, seendirs)
  215. except FileExistsError:
  216. continue
  217. staging_processfixme(fixme, targetdir, targetsysroot, nativesysroot, d)
  218. for p in sorted(postinsts):
  219. bb.note("Running postinst {}, output:\n{}".format(p, subprocess.check_output(p, shell=True, stderr=subprocess.STDOUT)))
  220. #
  221. # Manifests here are complicated. The main sysroot area has the unpacked sstate
  222. # which us unrelocated and tracked by the main sstate manifests. Each recipe
  223. # specific sysroot has manifests for each dependency that is installed there.
  224. # The task hash is used to tell whether the data needs to be reinstalled. We
  225. # use a symlink to point to the currently installed hash. There is also a
  226. # "complete" stamp file which is used to mark if installation completed. If
  227. # something fails (e.g. a postinst), this won't get written and we would
  228. # remove and reinstall the dependency. This also means partially installed
  229. # dependencies should get cleaned up correctly.
  230. #
  231. python extend_recipe_sysroot() {
  232. import copy
  233. import subprocess
  234. import errno
  235. import collections
  236. import glob
  237. taskdepdata = d.getVar("BB_TASKDEPDATA", False)
  238. mytaskname = d.getVar("BB_RUNTASK")
  239. if mytaskname.endswith("_setscene"):
  240. mytaskname = mytaskname.replace("_setscene", "")
  241. workdir = d.getVar("WORKDIR")
  242. #bb.warn(str(taskdepdata))
  243. pn = d.getVar("PN")
  244. stagingdir = d.getVar("STAGING_DIR")
  245. sharedmanifests = d.getVar("COMPONENTS_DIR") + "/manifests"
  246. # only needed by multilib cross-canadian since it redefines RECIPE_SYSROOT
  247. manifestprefix = d.getVar("RECIPE_SYSROOT_MANIFEST_SUBDIR")
  248. if manifestprefix:
  249. sharedmanifests = sharedmanifests + "/" + manifestprefix
  250. recipesysroot = d.getVar("RECIPE_SYSROOT")
  251. recipesysrootnative = d.getVar("RECIPE_SYSROOT_NATIVE")
  252. # Detect bitbake -b usage
  253. nodeps = d.getVar("BB_LIMITEDDEPS") or False
  254. if nodeps:
  255. lock = bb.utils.lockfile(recipesysroot + "/sysroot.lock")
  256. staging_populate_sysroot_dir(recipesysroot, recipesysrootnative, True, d)
  257. staging_populate_sysroot_dir(recipesysroot, recipesysrootnative, False, d)
  258. bb.utils.unlockfile(lock)
  259. return
  260. start = None
  261. configuredeps = []
  262. owntaskdeps = []
  263. for dep in taskdepdata:
  264. data = taskdepdata[dep]
  265. if data[1] == mytaskname and data[0] == pn:
  266. start = dep
  267. elif data[0] == pn:
  268. owntaskdeps.append(data[1])
  269. if start is None:
  270. bb.fatal("Couldn't find ourself in BB_TASKDEPDATA?")
  271. # We need to figure out which sysroot files we need to expose to this task.
  272. # This needs to match what would get restored from sstate, which is controlled
  273. # ultimately by calls from bitbake to setscene_depvalid().
  274. # That function expects a setscene dependency tree. We build a dependency tree
  275. # condensed to inter-sstate task dependencies, similar to that used by setscene
  276. # tasks. We can then call into setscene_depvalid() and decide
  277. # which dependencies we can "see" and should expose in the recipe specific sysroot.
  278. setscenedeps = copy.deepcopy(taskdepdata)
  279. start = set([start])
  280. sstatetasks = d.getVar("SSTATETASKS").split()
  281. # Add recipe specific tasks referenced by setscene_depvalid()
  282. sstatetasks.append("do_stash_locale")
  283. sstatetasks.append("do_deploy")
  284. def print_dep_tree(deptree):
  285. data = ""
  286. for dep in deptree:
  287. deps = " " + "\n ".join(deptree[dep][3]) + "\n"
  288. data = data + "%s:\n %s\n %s\n%s %s\n %s\n" % (deptree[dep][0], deptree[dep][1], deptree[dep][2], deps, deptree[dep][4], deptree[dep][5])
  289. return data
  290. #bb.note("Full dep tree is:\n%s" % print_dep_tree(taskdepdata))
  291. #bb.note(" start2 is %s" % str(start))
  292. # If start is an sstate task (like do_package) we need to add in its direct dependencies
  293. # else the code below won't recurse into them.
  294. for dep in set(start):
  295. for dep2 in setscenedeps[dep][3]:
  296. start.add(dep2)
  297. start.remove(dep)
  298. #bb.note(" start3 is %s" % str(start))
  299. # Create collapsed do_populate_sysroot -> do_populate_sysroot tree
  300. for dep in taskdepdata:
  301. data = setscenedeps[dep]
  302. if data[1] not in sstatetasks:
  303. for dep2 in setscenedeps:
  304. data2 = setscenedeps[dep2]
  305. if dep in data2[3]:
  306. data2[3].update(setscenedeps[dep][3])
  307. data2[3].remove(dep)
  308. if dep in start:
  309. start.update(setscenedeps[dep][3])
  310. start.remove(dep)
  311. del setscenedeps[dep]
  312. # Remove circular references
  313. for dep in setscenedeps:
  314. if dep in setscenedeps[dep][3]:
  315. setscenedeps[dep][3].remove(dep)
  316. #bb.note("Computed dep tree is:\n%s" % print_dep_tree(setscenedeps))
  317. #bb.note(" start is %s" % str(start))
  318. # Direct dependencies should be present and can be depended upon
  319. for dep in sorted(set(start)):
  320. if setscenedeps[dep][1] == "do_populate_sysroot":
  321. if dep not in configuredeps:
  322. configuredeps.append(dep)
  323. bb.note("Direct dependencies are %s" % str(configuredeps))
  324. #bb.note(" or %s" % str(start))
  325. msgbuf = []
  326. # Call into setscene_depvalid for each sub-dependency and only copy sysroot files
  327. # for ones that would be restored from sstate.
  328. done = list(start)
  329. next = list(start)
  330. while next:
  331. new = []
  332. for dep in next:
  333. data = setscenedeps[dep]
  334. for datadep in data[3]:
  335. if datadep in done:
  336. continue
  337. taskdeps = {}
  338. taskdeps[dep] = setscenedeps[dep][:2]
  339. taskdeps[datadep] = setscenedeps[datadep][:2]
  340. retval = setscene_depvalid(datadep, taskdeps, [], d, msgbuf)
  341. if retval:
  342. msgbuf.append("Skipping setscene dependency %s for installation into the sysroot" % datadep)
  343. continue
  344. done.append(datadep)
  345. new.append(datadep)
  346. if datadep not in configuredeps and setscenedeps[datadep][1] == "do_populate_sysroot":
  347. configuredeps.append(datadep)
  348. msgbuf.append("Adding dependency on %s" % setscenedeps[datadep][0])
  349. else:
  350. msgbuf.append("Following dependency on %s" % setscenedeps[datadep][0])
  351. next = new
  352. # This logging is too verbose for day to day use sadly
  353. #bb.debug(2, "\n".join(msgbuf))
  354. depdir = recipesysrootnative + "/installeddeps"
  355. bb.utils.mkdirhier(depdir)
  356. bb.utils.mkdirhier(sharedmanifests)
  357. lock = bb.utils.lockfile(recipesysroot + "/sysroot.lock")
  358. fixme = {}
  359. seendirs = set()
  360. postinsts = []
  361. multilibs = {}
  362. manifests = {}
  363. # All files that we're going to be installing, to find conflicts.
  364. fileset = {}
  365. invalidate_tasks = set()
  366. for f in os.listdir(depdir):
  367. removed = []
  368. if not f.endswith(".complete"):
  369. continue
  370. f = depdir + "/" + f
  371. if os.path.islink(f) and not os.path.exists(f):
  372. bb.note("%s no longer exists, removing from sysroot" % f)
  373. lnk = os.readlink(f.replace(".complete", ""))
  374. sstate_clean_manifest(depdir + "/" + lnk, d, canrace=True, prefix=workdir)
  375. os.unlink(f)
  376. os.unlink(f.replace(".complete", ""))
  377. removed.append(os.path.basename(f.replace(".complete", "")))
  378. # If we've removed files from the sysroot above, the task that installed them may still
  379. # have a stamp file present for the task. This is probably invalid right now but may become
  380. # valid again if the user were to change configuration back for example. Since we've removed
  381. # the files a task might need, remove the stamp file too to force it to rerun.
  382. # YOCTO #14790
  383. if removed:
  384. for i in glob.glob(depdir + "/index.*"):
  385. if i.endswith("." + mytaskname):
  386. continue
  387. with open(i, "r") as f:
  388. for l in f:
  389. if l.startswith("TaskDeps:"):
  390. continue
  391. l = l.strip()
  392. if l in removed:
  393. invalidate_tasks.add(i.rsplit(".", 1)[1])
  394. break
  395. for t in invalidate_tasks:
  396. bb.note("Invalidating stamps for task %s" % t)
  397. bb.build.clean_stamp(t, d)
  398. installed = []
  399. for dep in configuredeps:
  400. c = setscenedeps[dep][0]
  401. if mytaskname in ["do_sdk_depends", "do_populate_sdk_ext"] and c.endswith("-initial"):
  402. bb.note("Skipping initial setscene dependency %s for installation into the sysroot" % c)
  403. continue
  404. installed.append(c)
  405. # We want to remove anything which this task previously installed but is no longer a dependency
  406. taskindex = depdir + "/" + "index." + mytaskname
  407. if os.path.exists(taskindex):
  408. potential = []
  409. with open(taskindex, "r") as f:
  410. for l in f:
  411. l = l.strip()
  412. if l not in installed:
  413. fl = depdir + "/" + l
  414. if not os.path.exists(fl):
  415. # Was likely already uninstalled
  416. continue
  417. potential.append(l)
  418. # We need to ensure no other task needs this dependency. We hold the sysroot
  419. # lock so we ca search the indexes to check
  420. if potential:
  421. for i in glob.glob(depdir + "/index.*"):
  422. if i.endswith("." + mytaskname):
  423. continue
  424. with open(i, "r") as f:
  425. for l in f:
  426. if l.startswith("TaskDeps:"):
  427. prevtasks = l.split()[1:]
  428. if mytaskname in prevtasks:
  429. # We're a dependency of this task so we can clear items out the sysroot
  430. break
  431. l = l.strip()
  432. if l in potential:
  433. potential.remove(l)
  434. for l in potential:
  435. fl = depdir + "/" + l
  436. bb.note("Task %s no longer depends on %s, removing from sysroot" % (mytaskname, l))
  437. lnk = os.readlink(fl)
  438. sstate_clean_manifest(depdir + "/" + lnk, d, canrace=True, prefix=workdir)
  439. os.unlink(fl)
  440. os.unlink(fl + ".complete")
  441. msg_exists = []
  442. msg_adding = []
  443. # Handle all removals first since files may move between recipes
  444. for dep in configuredeps:
  445. c = setscenedeps[dep][0]
  446. if c not in installed:
  447. continue
  448. taskhash = setscenedeps[dep][5]
  449. taskmanifest = depdir + "/" + c + "." + taskhash
  450. if os.path.exists(depdir + "/" + c):
  451. lnk = os.readlink(depdir + "/" + c)
  452. if lnk == c + "." + taskhash and os.path.exists(depdir + "/" + c + ".complete"):
  453. continue
  454. else:
  455. bb.note("%s exists in sysroot, but is stale (%s vs. %s), removing." % (c, lnk, c + "." + taskhash))
  456. sstate_clean_manifest(depdir + "/" + lnk, d, canrace=True, prefix=workdir)
  457. os.unlink(depdir + "/" + c)
  458. if os.path.lexists(depdir + "/" + c + ".complete"):
  459. os.unlink(depdir + "/" + c + ".complete")
  460. elif os.path.lexists(depdir + "/" + c):
  461. os.unlink(depdir + "/" + c)
  462. binfiles = {}
  463. # Now handle installs
  464. for dep in sorted(configuredeps):
  465. c = setscenedeps[dep][0]
  466. if c not in installed:
  467. continue
  468. taskhash = setscenedeps[dep][5]
  469. taskmanifest = depdir + "/" + c + "." + taskhash
  470. if os.path.exists(depdir + "/" + c):
  471. lnk = os.readlink(depdir + "/" + c)
  472. if lnk == c + "." + taskhash and os.path.exists(depdir + "/" + c + ".complete"):
  473. msg_exists.append(c)
  474. continue
  475. msg_adding.append(c)
  476. os.symlink(c + "." + taskhash, depdir + "/" + c)
  477. manifest, d2 = oe.sstatesig.find_sstate_manifest(c, setscenedeps[dep][2], "populate_sysroot", d, multilibs)
  478. if d2 is not d:
  479. # If we don't do this, the recipe sysroot will be placed in the wrong WORKDIR for multilibs
  480. # We need a consistent WORKDIR for the image
  481. d2.setVar("WORKDIR", d.getVar("WORKDIR"))
  482. destsysroot = d2.getVar("RECIPE_SYSROOT")
  483. # We put allarch recipes into the default sysroot
  484. if manifest and "allarch" in manifest:
  485. destsysroot = d.getVar("RECIPE_SYSROOT")
  486. native = False
  487. if c.endswith("-native") or "-cross-" in c or "-crosssdk" in c:
  488. native = True
  489. if manifest:
  490. newmanifest = collections.OrderedDict()
  491. targetdir = destsysroot
  492. if native:
  493. targetdir = recipesysrootnative
  494. if targetdir not in fixme:
  495. fixme[targetdir] = []
  496. fm = fixme[targetdir]
  497. with open(manifest, "r") as f:
  498. manifests[dep] = manifest
  499. for l in f:
  500. l = l.strip()
  501. if l.endswith("/fixmepath"):
  502. fm.append(l)
  503. continue
  504. if l.endswith("/fixmepath.cmd"):
  505. continue
  506. dest = l.replace(stagingdir, "")
  507. dest = "/" + "/".join(dest.split("/")[3:])
  508. newmanifest[l] = targetdir + dest
  509. # Check if files have already been installed by another
  510. # recipe and abort if they have, explaining what recipes are
  511. # conflicting.
  512. hashname = targetdir + dest
  513. if not hashname.endswith("/"):
  514. if hashname in fileset:
  515. bb.fatal("The file %s is installed by both %s and %s, aborting" % (dest, c, fileset[hashname]))
  516. else:
  517. fileset[hashname] = c
  518. # Having multiple identical manifests in each sysroot eats diskspace so
  519. # create a shared pool of them and hardlink if we can.
  520. # We create the manifest in advance so that if something fails during installation,
  521. # or the build is interrupted, subsequent exeuction can cleanup.
  522. sharedm = sharedmanifests + "/" + os.path.basename(taskmanifest)
  523. if not os.path.exists(sharedm):
  524. smlock = bb.utils.lockfile(sharedm + ".lock")
  525. # Can race here. You'd think it just means we may not end up with all copies hardlinked to each other
  526. # but python can lose file handles so we need to do this under a lock.
  527. if not os.path.exists(sharedm):
  528. with open(sharedm, 'w') as m:
  529. for l in newmanifest:
  530. dest = newmanifest[l]
  531. m.write(dest.replace(workdir + "/", "") + "\n")
  532. bb.utils.unlockfile(smlock)
  533. try:
  534. os.link(sharedm, taskmanifest)
  535. except OSError as err:
  536. if err.errno == errno.EXDEV:
  537. bb.utils.copyfile(sharedm, taskmanifest)
  538. else:
  539. raise
  540. # Finally actually install the files
  541. for l in newmanifest:
  542. dest = newmanifest[l]
  543. if l.endswith("/"):
  544. staging_copydir(l, targetdir, dest, seendirs)
  545. continue
  546. if "/bin/" in l or "/sbin/" in l:
  547. # defer /*bin/* files until last in case they need libs
  548. binfiles[l] = (targetdir, dest)
  549. else:
  550. staging_copyfile(l, targetdir, dest, postinsts, seendirs)
  551. # Handle deferred binfiles
  552. for l in binfiles:
  553. (targetdir, dest) = binfiles[l]
  554. staging_copyfile(l, targetdir, dest, postinsts, seendirs)
  555. bb.note("Installed into sysroot: %s" % str(msg_adding))
  556. bb.note("Skipping as already exists in sysroot: %s" % str(msg_exists))
  557. for f in fixme:
  558. staging_processfixme(fixme[f], f, recipesysroot, recipesysrootnative, d)
  559. for p in sorted(postinsts):
  560. bb.note("Running postinst {}, output:\n{}".format(p, subprocess.check_output(p, shell=True, stderr=subprocess.STDOUT)))
  561. for dep in manifests:
  562. c = setscenedeps[dep][0]
  563. os.symlink(manifests[dep], depdir + "/" + c + ".complete")
  564. with open(taskindex, "w") as f:
  565. f.write("TaskDeps: " + " ".join(owntaskdeps) + "\n")
  566. for l in sorted(installed):
  567. f.write(l + "\n")
  568. bb.utils.unlockfile(lock)
  569. }
  570. extend_recipe_sysroot[vardepsexclude] += "MACHINE_ARCH PACKAGE_EXTRA_ARCHS SDK_ARCH BUILD_ARCH SDK_OS BB_TASKDEPDATA"
  571. do_prepare_recipe_sysroot[deptask] = "do_populate_sysroot"
  572. python do_prepare_recipe_sysroot () {
  573. bb.build.exec_func("extend_recipe_sysroot", d)
  574. }
  575. addtask do_prepare_recipe_sysroot before do_configure after do_fetch
  576. python staging_taskhandler() {
  577. EXCLUDED_TASKS = (
  578. "do_prepare_recipe_sysroot",
  579. "do_create_spdx",
  580. )
  581. bbtasks = e.tasklist
  582. for task in bbtasks:
  583. if task in EXCLUDED_TASKS:
  584. continue
  585. deps = d.getVarFlag(task, "depends")
  586. if task == "do_configure" or (deps and "populate_sysroot" in deps):
  587. d.prependVarFlag(task, "prefuncs", "extend_recipe_sysroot ")
  588. }
  589. staging_taskhandler[eventmask] = "bb.event.RecipeTaskPreProcess"
  590. addhandler staging_taskhandler
  591. #
  592. # Target build output, stored in do_populate_sysroot or do_package can depend
  593. # not only upon direct dependencies but also indirect ones. A good example is
  594. # linux-libc-headers. The toolchain depends on this but most target recipes do
  595. # not. There are some headers which are not used by the toolchain build and do
  596. # not change the toolchain task output, hence the task hashes can change without
  597. # changing the sysroot output of that recipe yet they can influence others.
  598. #
  599. # A specific example is rtc.h which can change rtcwake.c in util-linux but is not
  600. # used in the glibc or gcc build. To account for this, we need to account for the
  601. # populate_sysroot hashes in the task output hashes.
  602. #
  603. python target_add_sysroot_deps () {
  604. current_task = "do_" + d.getVar("BB_CURRENTTASK")
  605. if current_task not in ["do_populate_sysroot", "do_package"]:
  606. return
  607. pn = d.getVar("PN")
  608. if pn.endswith("-native"):
  609. return
  610. taskdepdata = d.getVar("BB_TASKDEPDATA", False)
  611. deps = {}
  612. for dep in taskdepdata.values():
  613. if dep[1] == "do_populate_sysroot" and not dep[0].endswith(("-native", "-initial")) and "-cross-" not in dep[0] and dep[0] != pn:
  614. deps[dep[0]] = dep[6]
  615. d.setVar("HASHEQUIV_EXTRA_SIGDATA", "\n".join("%s: %s" % (k, deps[k]) for k in sorted(deps.keys())))
  616. }
  617. SSTATECREATEFUNCS += "target_add_sysroot_deps"