validate-launcher: Allow running tests out-of-order

When the --shuffle option is used, the tests will be run out of order.
This optimizes CPU utilization since it allows running synchronized
and unsynchronized tests at the same.

validate/launcher/baseclasses.py

@@ -35,6 +35,7 @@ import threading
 import queue
 import configparser
 import xml
+import random
 
 from . import reporters
 from . import loggable
@@ -1609,6 +1610,12 @@ class _TestsLauncher(Loggable):
         max_num_jobs = min(self.options.num_jobs, len(tests))
         jobs_running = 0
 
+        # if order of test execution doesn't matter, shuffle
+        # the order to optimize cpu usage
+        if self.options.shuffle:
+            random.shuffle(tests)
+            random.shuffle(alone_tests)
+
         for num_jobs, tests in [(max_num_jobs, tests), (1, alone_tests)]:
             tests_left = list(tests)
             for i in range(num_jobs):

validate/launcher/main.py

@@ -467,6 +467,10 @@ Note that all testsuite should be inside python modules, so the directory should
     parser.add_argument('--xunit-file', dest='xunit_file',
                         action='store', metavar="FILE",
                         help=("Path to xml file to store the xunit report in."))
+    parser.add_argument('--shuffle', dest="shuffle", action="store_true",
+                        help="Runs the test in a random order. Can help speed up the overall"
+                             " test time by running synchronized and unsynchronized tests"
+                             " at the same time")
     dir_group = parser.add_argument_group(
         "Directories and files to be used by the launcher")
     dir_group.add_argument("-M", "--main-dir", dest="main_dir",

validate/launcher/reporters.py

@@ -97,7 +97,8 @@ class Reporter(Loggable):
     def final_report(self):
         print("\n")
         printc("Final Report:", title=True)
-        for test in sorted(self.results, key=lambda test: test.result):
+        sortedresults = sorted(self.results, key=lambda test: test.classname)
+        for test in sorted(sortedresults, key=lambda test: test.result):
             printc(test)
             if test.result != Result.PASSED:
                 print("\n")