resulttool: Load results from URL

Adds support for resulttool to load JSON files directly from a http://
or https:// URL

(From OE-Core rev: 3e48404afe27c93fa6ffbd8d66bc52dcd6216005)

Signed-off-by: Joshua Watt <JPEWhacker@gmail.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Signed-off-by: Armin Kuster <akuster808@gmail.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>

scripts/lib/resulttool/merge.py

@@ -17,7 +17,7 @@ import json
 import resulttool.resultutils as resultutils
 
 def merge(args, logger):
-    if os.path.isdir(args.target_results):
+    if resultutils.is_url(args.target_results) or os.path.isdir(args.target_results):
         results = resultutils.load_resultsdata(args.target_results, configmap=resultutils.store_map)
         resultutils.append_resultsdata(results, args.base_results, configmap=resultutils.store_map)
         resultutils.save_resultsdata(results, args.target_results)
@@ -31,12 +31,12 @@ def merge(args, logger):
 
 def register_commands(subparsers):
     """Register subcommands from this plugin"""
-    parser_build = subparsers.add_parser('merge', help='merge test result files/directories',
-                                         description='merge the results from multiple files/directories into the target file or directory',
+    parser_build = subparsers.add_parser('merge', help='merge test result files/directories/URLs',
+                                         description='merge the results from multiple files/directories/URLs into the target file or directory',
                                          group='setup')
     parser_build.set_defaults(func=merge)
     parser_build.add_argument('base_results',
-                              help='the results file/directory to import')
+                              help='the results file/directory/URL to import')
     parser_build.add_argument('target_results',
                               help='the target file or directory to merge the base_results with')
 

scripts/lib/resulttool/regression.py

@@ -161,9 +161,9 @@ def register_commands(subparsers):
                                          group='analysis')
     parser_build.set_defaults(func=regression)
     parser_build.add_argument('base_result',
-                              help='base result file/directory for the comparison')
+                              help='base result file/directory/URL for the comparison')
     parser_build.add_argument('target_result',
-                              help='target result file/directory to compare with')
+                              help='target result file/directory/URL to compare with')
     parser_build.add_argument('-b', '--base-result-id', default='',
                               help='(optional) filter the base results to this result ID')
     parser_build.add_argument('-t', '--target-result-id', default='',

scripts/lib/resulttool/report.py

@@ -143,7 +143,7 @@ def register_commands(subparsers):
                                          group='analysis')
     parser_build.set_defaults(func=report)
     parser_build.add_argument('source_dir',
-                              help='source file/directory that contain the test result files to summarise')
+                              help='source file/directory/URL that contain the test result files to summarise')
     parser_build.add_argument('--branch', '-B', default='master', help="Branch to find commit in")
     parser_build.add_argument('--commit', help="Revision to report")
     parser_build.add_argument('-t', '--tag', default='',

scripts/lib/resulttool/resultutils.py

@@ -16,6 +16,8 @@ import os
 import json
 import scriptpath
 import copy
+import urllib
+import posixpath
 scriptpath.add_oe_lib_path()
 
 flatten_map = {
@@ -40,20 +42,33 @@ store_map = {
     "manual": ['TEST_TYPE', 'TEST_MODULE', 'MACHINE', 'IMAGE_BASENAME']
 }
 
+def is_url(p):
+    """
+    Helper for determining if the given path is a URL
+    """
+    return p.startswith('http://') or p.startswith('https://')
+
 #
 # Load the json file and append the results data into the provided results dict
 #
 def append_resultsdata(results, f, configmap=store_map):
     if type(f) is str:
-        with open(f, "r") as filedata:
-            data = json.load(filedata)
+        if is_url(f):
+            with urllib.request.urlopen(f) as response:
+                data = json.loads(response.read().decode('utf-8'))
+            url = urllib.parse.urlparse(f)
+            testseries = posixpath.basename(posixpath.dirname(url.path))
+        else:
+            with open(f, "r") as filedata:
+                data = json.load(filedata)
+            testseries = os.path.basename(os.path.dirname(f))
     else:
         data = f
     for res in data:
         if "configuration" not in data[res] or "result" not in data[res]:
             raise ValueError("Test results data without configuration or result section?")
         if "TESTSERIES" not in data[res]["configuration"]:
-            data[res]["configuration"]["TESTSERIES"] = os.path.basename(os.path.dirname(f))
+            data[res]["configuration"]["TESTSERIES"] = testseries
         testtype = data[res]["configuration"].get("TEST_TYPE")
         if testtype not in configmap:
             raise ValueError("Unknown test type %s" % testtype)
@@ -69,7 +84,7 @@ def append_resultsdata(results, f, configmap=store_map):
 #
 def load_resultsdata(source, configmap=store_map):
     results = {}
-    if os.path.isfile(source):
+    if is_url(source) or os.path.isfile(source):
         append_resultsdata(results, source, configmap)
         return results
     for root, dirs, files in os.walk(source):

scripts/lib/resulttool/store.py

@@ -29,7 +29,7 @@ def store(args, logger):
     try:
         results = {}
         logger.info('Reading files from %s' % args.source)
-        if os.path.isfile(args.source):
+        if resultutils.is_url(args.source) or os.path.isfile(args.source):
             resultutils.append_resultsdata(results, args.source)
         else:
             for root, dirs,  files in os.walk(args.source):
@@ -92,7 +92,7 @@ def register_commands(subparsers):
                                          group='setup')
     parser_build.set_defaults(func=store)
     parser_build.add_argument('source',
-                              help='source file or directory that contain the test result files to be stored')
+                              help='source file/directory/URL that contain the test result files to be stored')
     parser_build.add_argument('git_dir',
                               help='the location of the git repository to store the results in')
     parser_build.add_argument('-a', '--all', action='store_true',