base: Switch UNPACKDIR to a subdir of WORKDIR

Change do_unpack to unpack files to a subdirectory of WORKDIR instead of WORKDIR
itself. There are several good reasons for this but it is mainly about being able
to isolate the output of the unpack task and tell the files apart from other things
which are created in workdir (logs, sysroots, temp dirs and more).

This means that when the do_unpack task reruns, we can clean UNPACKDIR and know
we have a standard point to start builds from.

It also makes code in tools like devtool and recipetool easier.

To reduce the impact to users, if a subdirectory under UNPACKDIR matches
the first subdirectory under WORKDIR of S, that directory is moved into position
inside WORKDIR. This preserves the behaviour of S = "${WORKDIR}/git",
S = "${WORKDIR}/${BPN}" and other commonly used source directory setups.

The directory is moved since sadly many autotools based projects can't cope with
symlinks in their paths.

The patch also updates reproducible and SOURCE_DATE_EPOCH handling to
match the new potential source locations. We can get rid of the horrible
list of hardcoded directories in WORKDIR to ignore from that code.

(From OE-Core rev: b84eec5c4cbf4b39d6712800dd0d2fe5337721cb)

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>

meta/classes-global/base.bbclass

@@ -153,20 +153,38 @@ python base_do_fetch() {
 }
 
 addtask unpack after do_fetch
-do_unpack[dirs] = "${UNPACKDIR}"
-
-do_unpack[cleandirs] = "${@d.getVar('S') if os.path.normpath(d.getVar('S')) != os.path.normpath(d.getVar('WORKDIR')) else os.path.join('${S}', 'patches')}"
+do_unpack[cleandirs] = "${UNPACKDIR}"
 
 python base_do_unpack() {
+    import shutil
+
+    sourcedir = d.getVar('S')
+    # Intentionally keep SOURCE_BASEDIR internal to the task just for SDE
+    d.setVar("SOURCE_BASEDIR", sourcedir)
+
     src_uri = (d.getVar('SRC_URI') or "").split()
     if not src_uri:
         return
 
+    basedir = None
+    unpackdir = d.getVar('UNPACKDIR')
+    workdir = d.getVar('WORKDIR')
+    if sourcedir.startswith(workdir) and not sourcedir.startswith(unpackdir):
+        basedir = sourcedir.replace(workdir, '').strip("/").split('/')[0]
+        if basedir:
+            bb.utils.remove(workdir + '/' + basedir, True)
+            d.setVar("SOURCE_BASEDIR", workdir + '/' + basedir)
+
     try:
         fetcher = bb.fetch2.Fetch(src_uri, d)
         fetcher.unpack(d.getVar('UNPACKDIR'))
     except bb.fetch2.BBFetchException as e:
         bb.fatal("Bitbake Fetcher Error: " + repr(e))
+
+    if basedir and os.path.exists(unpackdir + '/' + basedir):
+        # Compatibility magic to ensure ${WORKDIR}/git and ${WORKDIR}/${BP}
+        # as often used in S work as expected.
+        shutil.move(unpackdir + '/' + basedir, workdir + '/' + basedir)
 }
 
 SSTATETASKS += "do_deploy_source_date_epoch"
@@ -199,8 +217,8 @@ addtask do_deploy_source_date_epoch_setscene
 addtask do_deploy_source_date_epoch before do_configure after do_patch
 
 python create_source_date_epoch_stamp() {
-    # Version: 1
-    source_date_epoch = oe.reproducible.get_source_date_epoch(d, d.getVar('S'))
+    # Version: 2
+    source_date_epoch = oe.reproducible.get_source_date_epoch(d, d.getVar('SOURCE_BASEDIR') or d.getVar('S'))
     oe.reproducible.epochfile_write(source_date_epoch, d.getVar('SDE_FILE'), d)
 }
 do_unpack[postfuncs] += "create_source_date_epoch_stamp"

meta/conf/bitbake.conf

@@ -405,7 +405,7 @@ STAMP = "${STAMPS_DIR}/${MULTIMACH_TARGET_SYS}/${PN}/${PV}"
 STAMPCLEAN = "${STAMPS_DIR}/${MULTIMACH_TARGET_SYS}/${PN}/*-*"
 BASE_WORKDIR ?= "${TMPDIR}/work"
 WORKDIR = "${BASE_WORKDIR}/${MULTIMACH_TARGET_SYS}/${PN}/${PV}"
-UNPACKDIR ??= "${WORKDIR}"
+UNPACKDIR ??= "${WORKDIR}/sources-unpack"
 T = "${WORKDIR}/temp"
 D = "${WORKDIR}/image"
 S = "${WORKDIR}/${BP}"

meta/lib/oe/reproducible.py

@@ -75,10 +75,10 @@ def get_source_date_epoch_from_known_files(d, sourcedir):
     return source_date_epoch
 
 def find_git_folder(d, sourcedir):
-    # First guess: WORKDIR/git
+    # First guess: UNPACKDIR/git
     # This is the default git fetcher unpack path
-    workdir = d.getVar('WORKDIR')
-    gitpath = os.path.join(workdir, "git/.git")
+    unpackdir = d.getVar('UNPACKDIR')
+    gitpath = os.path.join(unpackdir, "git/.git")
     if os.path.isdir(gitpath):
         return gitpath
 
@@ -88,15 +88,16 @@ def find_git_folder(d, sourcedir):
         return gitpath
 
     # Perhaps there was a subpath or destsuffix specified.
-    # Go looking in the WORKDIR
-    exclude = set(["build", "image", "license-destdir", "patches", "pseudo",
-                   "recipe-sysroot", "recipe-sysroot-native", "sysroot-destdir", "temp"])
-    for root, dirs, files in os.walk(workdir, topdown=True):
-        dirs[:] = [d for d in dirs if d not in exclude]
+    # Go looking in the UNPACKDIR
+    for root, dirs, files in os.walk(unpackdir, topdown=True):
         if '.git' in dirs:
             return os.path.join(root, ".git")
 
-    bb.warn("Failed to find a git repository in WORKDIR: %s" % workdir)
+    for root, dirs, files in os.walk(sourcedir, topdown=True):
+        if '.git' in dirs:
+            return os.path.join(root, ".git")
+
+    bb.warn("Failed to find a git repository in UNPACKDIR: %s" % unpackdir)
     return None
 
 def get_source_date_epoch_from_git(d, sourcedir):