parallel execute

parse-ci.py

@@ -88,6 +88,24 @@ def run_cli_sequential(job : dict) -> int:
                 job['image'], \
                 exec_location])
 
+def run_cli_parallel(job : dict) -> dict:
+    exec_location = os.path.join("/usr/src/project/",os.path.relpath(job['location'] , dir_path)).replace("\\", "/")
+    f = open(dir_path + os.sep + job['location'].replace("\\", os.sep).replace("/", os.sep) + ".log", 'w')
+    job["task"] = subprocess.Popen( \
+        args= \
+            [   "docker", \
+                "run", \
+                "--rm", \
+                "-v", \
+                f"{dir_path}:/usr/src/project", \
+                "--entrypoint", "sh", \
+                "-it", \
+                job['image'], \
+                exec_location],
+                stdout=f)
+    job["output"] = f
+    return job
+
 def remove_from_depends(process_info : dict, jobname : str) -> dict:
     if jobname in process_info["depends"]:
         process_info["depends"].remove(jobname)
@@ -98,14 +116,30 @@ def run_ci(compile_info : list[dict]) -> bool:
     print(compile_info)
     jobs = []
     failed_jobs = []
-    while len(jobs) != 0 or can_run_jon(compile_info):
-        new_job = get_new_job(compile_info)
-        print(f"compiling: {new_job['location']}")
-        code = run_cli_sequential(new_job)
-        compile_info.remove(new_job)
-        compile_info = [remove_from_depends(r, new_job['location']) for r in compile_info]
-        if code != 0:
-            failed_jobs.append(new_job['location'])
+    current_jobs = []
+    while len(jobs) != 0 or can_run_jon(compile_info) or len(current_jobs) != 0:
+        removed_job = None
+        for job in current_jobs:
+            poll = job["task"].poll()
+            if poll is not None:
+                if job["task"].returncode != 0:
+                    print("failed: " + job['location'])
+                    failed_jobs.append(job['location'])
+                else:
+                    print("completed: " + job['location'])
+                    compile_info = [remove_from_depends(r, job['location']) for r in compile_info]
+                    removed_job = job
+                    break
+        if removed_job is not None:
+            current_jobs.remove(removed_job)
+            job["output"].close()
+
+        if can_run_jon(compile_info):
+            new_job = get_new_job(compile_info)
+            print(f"compiling: {new_job['location']}")
+            code = run_cli_parallel(new_job)
+            compile_info.remove(new_job)
+            current_jobs.append(code)
     print(compile_info)
     if len(failed_jobs) != 0:
         print(f"failed {len(failed_jobs)} jobs:\n\t{failed_jobs}")