scripts: Implement oe-git-archive

A helper script for committing data to git and pushing it upstream. The
motivation for the script stems from the need to archiving QA test
results in git, but it could be used elsewhere, of course. The script
needs to be run under an initialized build environment because it
utilizes bitbake configuration information.

[YOCTO #10582]

(From OE-Core rev: 801e612b137b9d5366639d5b1635151347da5983)

Signed-off-by: Markus Lehtonen <markus.lehtonen@linux.intel.com>
Signed-off-by: Ross Burton <ross.burton@intel.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>

scripts/oe-git-archive

@@ -0,0 +1,244 @@
+#!/usr/bin/python3
+#
+# Helper script for committing data to git and pushing upstream
+#
+# Copyright (c) 2017, Intel Corporation.
+#
+# This program is free software; you can redistribute it and/or modify it
+# under the terms and conditions of the GNU General Public License,
+# version 2, as published by the Free Software Foundation.
+#
+# This program is distributed in the hope it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+# more details.
+#
+import argparse
+import glob
+import json
+import logging
+import math
+import os
+import re
+import sys
+from collections import namedtuple, OrderedDict
+from datetime import datetime, timedelta, tzinfo
+from operator import attrgetter
+
+# Import oe and bitbake libs
+scripts_path = os.path.dirname(os.path.realpath(__file__))
+sys.path.append(os.path.join(scripts_path, 'lib'))
+import scriptpath
+scriptpath.add_bitbake_lib_path()
+scriptpath.add_oe_lib_path()
+
+from oeqa.utils.git import GitRepo, GitError
+from oeqa.utils.metadata import metadata_from_bb
+
+
+# Setup logging
+logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s")
+log = logging.getLogger()
+
+
+class ArchiveError(Exception):
+    """Internal error handling of this script"""
+
+
+def format_str(string, fields):
+    """Format string using the given fields (dict)"""
+    try:
+        return string.format(**fields)
+    except KeyError as err:
+        raise ArchiveError("Unable to expand string '{}': unknown field {} "
+                           "(valid fields are: {})".format(
+                               string, err, ', '.join(sorted(fields.keys()))))
+
+
+def init_git_repo(path, no_create):
+    """Initialize local Git repository"""
+    path = os.path.abspath(path)
+    if os.path.isfile(path):
+        raise ArchiveError("Invalid Git repo at {}: path exists but is not a "
+                           "directory".format(path))
+    if not os.path.isdir(path) or not os.listdir(path):
+        if no_create:
+            raise ArchiveError("No git repo at {}, refusing to create "
+                               "one".format(path))
+        if not os.path.isdir(path):
+            try:
+                os.mkdir(path)
+            except (FileNotFoundError, PermissionError) as err:
+                raise ArchiveError("Failed to mkdir {}: {}".format(path, err))
+        if not os.listdir(path):
+            log.info("Initializing a new Git repo at %s", path)
+            repo = GitRepo.init(path)
+    try:
+        repo = GitRepo(path, is_topdir=True)
+    except GitError:
+        raise ArchiveError("Non-empty directory that is not a Git repository "
+                           "at {}\nPlease specify an existing Git repository, "
+                           "an empty directory or a non-existing directory "
+                           "path.".format(path))
+    return repo
+
+
+def git_commit_data(repo, data_dir, branch, message):
+    """Commit data into a Git repository"""
+    log.info("Committing data into to branch %s", branch)
+    tmp_index = os.path.join(repo.git_dir, 'index.oe-git-archive')
+    try:
+        # Create new tree object from the data
+        env_update = {'GIT_INDEX_FILE': tmp_index,
+                      'GIT_WORK_TREE': os.path.abspath(data_dir)}
+        repo.run_cmd('add .', env_update)
+        tree = repo.run_cmd('write-tree', env_update)
+
+        # Create new commit object from the tree
+        parent = repo.rev_parse(branch)
+        git_cmd = ['commit-tree', tree, '-m', message]
+        if parent:
+            git_cmd += ['-p', parent]
+        commit = repo.run_cmd(git_cmd, env_update)
+
+        # Update branch head
+        git_cmd = ['update-ref', 'refs/heads/' + branch, commit]
+        if parent:
+            git_cmd.append(parent)
+        repo.run_cmd(git_cmd)
+
+        # Update current HEAD, if we're on branch 'branch'
+        if repo.get_current_branch() == branch:
+            log.info("Updating %s HEAD to latest commit", repo.top_dir)
+            repo.run_cmd('reset --hard')
+
+        return commit
+    finally:
+        if os.path.exists(tmp_index):
+            os.unlink(tmp_index)
+
+
+def expand_tag_strings(repo, name_pattern, msg_subj_pattern, msg_body_pattern,
+                       keywords):
+    """Generate tag name and message, with support for running id number"""
+    keyws = keywords.copy()
+    # Tag number is handled specially: if not defined, we autoincrement it
+    if 'tag_number' not in keyws:
+        # Fill in all other fields than 'tag_number'
+        keyws['tag_number'] = '{tag_number}'
+        tag_re = format_str(name_pattern, keyws)
+        # Replace parentheses for proper regex matching
+        tag_re = tag_re.replace('(', '\(').replace(')', '\)') + '$'
+        # Inject regex group pattern for 'tag_number'
+        tag_re = tag_re.format(tag_number='(?P<tag_number>[0-9]{1,5})')
+
+        keyws['tag_number'] = 0
+        for existing_tag in repo.run_cmd('tag').splitlines():
+            match = re.match(tag_re, existing_tag)
+
+            if match and int(match.group('tag_number')) >= keyws['tag_number']:
+                keyws['tag_number'] = int(match.group('tag_number')) + 1
+
+    tag_name = format_str(name_pattern, keyws)
+    msg_subj= format_str(msg_subj_pattern.strip(), keyws)
+    msg_body = format_str(msg_body_pattern, keyws)
+    return tag_name, msg_subj + '\n\n' + msg_body
+
+
+def parse_args(argv):
+    """Parse command line arguments"""
+    parser = argparse.ArgumentParser(
+            description="Commit data to git and push upstream",
+            formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+
+    parser.add_argument('--debug', '-D', action='store_true',
+                        help="Verbose logging")
+    parser.add_argument('--git-dir', '-g', required=True,
+                        help="Local git directory to use")
+    parser.add_argument('--no-create', action='store_true',
+                        help="If GIT_DIR is not a valid Git repository, do not "
+                             "try to create one")
+    parser.add_argument('--push', '-p', nargs='?', default=False, const=True,
+                        help="Push to remote")
+    parser.add_argument('--branch-name', '-b',
+                        default='{hostname}/{branch}/{machine}',
+                        help="Git branch name (pattern) to use")
+    parser.add_argument('--no-tag', action='store_true',
+                        help="Do not create Git tag")
+    parser.add_argument('--tag-name', '-t',
+                        default='{hostname}/{branch}/{machine}/{commit_count}-g{commit}/{tag_number}',
+                        help="Tag name (pattern) to use")
+    parser.add_argument('--commit-msg-subject',
+                        default='Results of {branch}:{commit} on {hostname}',
+                        help="Subject line (pattern) to use in the commit message")
+    parser.add_argument('--commit-msg-body',
+                        default='branch: {branch}\ncommit: {commit}\nhostname: {hostname}',
+                        help="Commit message body (pattern)")
+    parser.add_argument('--tag-msg-subject',
+                        default='Test run #{tag_number} of {branch}:{commit} on {hostname}',
+                        help="Subject line (pattern) of the tag message")
+    parser.add_argument('--tag-msg-body',
+                        default='',
+                        help="Tag message body (pattern)")
+    parser.add_argument('data_dir', metavar='DATA_DIR',
+                        help="Data to commit")
+    return parser.parse_args(argv)
+
+
+def main(argv=None):
+    """Script entry point"""
+    args = parse_args(argv)
+    if args.debug:
+        log.setLevel(logging.DEBUG)
+
+    try:
+        if not os.path.isdir(args.data_dir):
+            raise ArchiveError("Not a directory: {}".format(args.data_dir))
+
+        data_repo = init_git_repo(args.git_dir, args.no_create)
+
+        # Get keywords to be used in tag and branch names and messages
+        metadata = metadata_from_bb()
+        keywords = {'hostname': metadata['hostname'],
+                    'branch': metadata['layers']['meta']['branch'],
+                    'commit': metadata['layers']['meta']['commit'],
+                    'commit_count': metadata['layers']['meta']['commit_count'],
+                    'machine': metadata['config']['MACHINE']}
+
+        # Expand strings early in order to avoid getting into inconsistent
+        # state (e.g. no tag even if data was committed)
+        commit_msg = format_str(args.commit_msg_subject.strip(), keywords)
+        commit_msg += '\n\n' + format_str(args.commit_msg_body, keywords)
+        branch_name = format_str(args.branch_name, keywords)
+        tag_name = None
+        if not args.no_tag and args.tag_name:
+            tag_name, tag_msg = expand_tag_strings(data_repo, args.tag_name,
+                                                   args.tag_msg_subject,
+                                                   args.tag_msg_body, keywords)
+
+        # Commit data
+        commit = git_commit_data(data_repo, args.data_dir, branch_name,
+                                 commit_msg)
+
+        # Create tag
+        if tag_name:
+            log.info("Creating tag %s", tag_name)
+            data_repo.run_cmd(['tag', '-a', '-m', tag_msg, tag_name, commit])
+
+        # Push data to remote
+        if args.push:
+            cmd = ['push', '--tags']
+            if args.push is not True:
+                cmd.extend(['--repo', args.push])
+                cmd.append(branch_name)
+            log.info("Pushing data to remote")
+            data_repo.run_cmd(cmd)
+
+    except ArchiveError as err:
+        log.error(str(err))
+        return 1
+
+    return 0
+
+if __name__ == "__main__":
+    sys.exit(main())