diff --git a/ci.py b/ci.py index 05f5e8c..a605836 100755 --- a/ci.py +++ b/ci.py @@ -66,6 +66,8 @@ def parse_args(): help='Ratch root directory.') ap.add_argument('-d', '--dry-run', action='store_true', default=False, help='Run it without uploading the result. default=False') + ap.add_argument('-j', '--jobs', action='store', type=str, default="4", + help='Number of make jobs (number or "auto"). default=4') # Positional parameter ap.add_argument('space', choices=['user', 'kernel'], @@ -396,6 +398,9 @@ def create_test_list_kernel(ci_data): # BuildKernel32 test_list.append(ci.BuildKernel32(ci_data, kernel_config=kernel_config)) + # CheckKernelLLVM + test_list.append(ci.CheckKernelLLVM(ci_data, kernel_config=kernel_config)) + # TestRunnerSetup tester_config = os.path.join(ci_data.config['bluez_dir'], "doc", "tester.config") @@ -501,6 +506,9 @@ def main(): log_error(f"Invalid parameter(space) {args.space}") sys.exit(1) + if args.jobs == "auto": + args.jobs = str(os.cpu_count()) + ci_data = Context(config_file=os.path.abspath(args.config), github_repo=args.repo, src_dir=main_src, @@ -508,7 +516,7 @@ def main(): branch=args.branch, dry_run=args.dry_run, bluez_dir=args.bluez_dir, ell_dir=args.ell_dir, kernel_dir=args.kernel_dir, pr_num=args.pr_num, - space=args.space) + space=args.space, jobs=args.jobs) # Setup Source for the test that needs to access the base like incremental # build. diff --git a/ci/__init__.py b/ci/__init__.py index e8d8db2..c501409 100755 --- a/ci/__init__.py +++ b/ci/__init__.py @@ -19,4 +19,5 @@ from .checksparse import CheckSparse from .checkallwarning import CheckAllWarning from .checksmatch import CheckSmatch +from .checkkernelllvm import CheckKernelLLVM diff --git a/ci/buildbluez.py b/ci/buildbluez.py index 9971f9a..ff3558a 100755 --- a/ci/buildbluez.py +++ b/ci/buildbluez.py @@ -29,7 +29,8 @@ def __init__(self, ci_data, src_dir=None, config_params=None, self.dry_run = dry_run super().__init__(config_params=config_params, work_dir=self.src_dir, - make_params=self.make_params) + make_params=self.make_params, + jobs=ci_data.config.get('jobs')) self.log_dbg("Initialization completed") diff --git a/ci/buildell.py b/ci/buildell.py index f5be597..e06f7aa 100755 --- a/ci/buildell.py +++ b/ci/buildell.py @@ -18,7 +18,8 @@ def __init__(self, ci_data, src_dir=None): self.desc = "Build and Install ELL" self.ci_data = ci_data - super().__init__(work_dir=ci_data.config['ell_dir'], install=True) + super().__init__(work_dir=ci_data.config['ell_dir'], install=True, + jobs=ci_data.config.get('jobs')) self.log_dbg("Initialization completed") diff --git a/ci/buildkernel.py b/ci/buildkernel.py index 96c2a27..e3be8f6 100755 --- a/ci/buildkernel.py +++ b/ci/buildkernel.py @@ -46,7 +46,8 @@ def __init__(self, ci_data, kernel_config=None, simple_build=True, self.stderr = None super().__init__(kernel_config=kernel_config, simple_build=simple_build, - make_params=make_params, work_dir=self.src_dir) + make_params=make_params, work_dir=self.src_dir, + jobs=ci_data.config.get('jobs')) self.log_dbg("Initialization completed") diff --git a/ci/buildkernel32.py b/ci/buildkernel32.py index dd33bb9..ab8c671 100755 --- a/ci/buildkernel32.py +++ b/ci/buildkernel32.py @@ -49,7 +49,8 @@ def __init__(self, ci_data, kernel_config=None, simple_build=True, self.log_dbg(f"New config for 32bit is created: {new_config}") super().__init__(kernel_config=new_config, simple_build=simple_build, - make_params=make_params, work_dir=self.src_dir) + make_params=make_params, work_dir=self.src_dir, + jobs=ci_data.config.get('jobs')) self.log_dbg("Initialization completed") diff --git a/ci/checkallwarning.py b/ci/checkallwarning.py index abfab2d..023b5fd 100755 --- a/ci/checkallwarning.py +++ b/ci/checkallwarning.py @@ -10,7 +10,8 @@ class CheckAllWarning(GenericKernelBuild): This class runs the kernel build with all warning enabled. """ - def __init__(self, ci_data, kernel_config=None, src_dir=None, dry_run=None): + def __init__(self, ci_data, kernel_config=None, src_dir=None, dry_run=None, + make_params=None): self.name = "CheckAllWarning" self.desc = "Run linux kernel with all warning enabled" @@ -30,8 +31,14 @@ def __init__(self, ci_data, kernel_config=None, src_dir=None, dry_run=None): self.log_dbg(f"Override the dry_run flag: {dry_run}") self.dry_run = dry_run + if make_params: + make_params = ['W=1'] + make_params + else: + make_params = ['W=1'] + super().__init__(kernel_config=kernel_config, simple_build=True, - make_params=['W=1'], work_dir=self.src_dir) + make_params=make_params, work_dir=self.src_dir, + jobs=ci_data.config.get('jobs')) self.log_dbg("Initialization completed") @@ -52,7 +59,7 @@ def run(self): if self.verdict == Verdict.FAIL: submit_pw_check(self.ci_data.pw, self.ci_data.patch_1, self.name, Verdict.FAIL, - "CheckAllWarning: FAIL: " + self.output, + f"{self.name}: FAIL: " + self.output, None, self.dry_run) # Test verdict and output is already set by the super().run(). # Just raising EndTest exception is enough here @@ -64,7 +71,7 @@ def run(self): # Build success submit_pw_check(self.ci_data.pw, self.ci_data.patch_1, self.name, Verdict.PASS, - "CheckAllWarning PASS", + f"{self.name} PASS", None, self.dry_run) # Actually no need to call success() here. But add it here just for # reference @@ -86,7 +93,7 @@ def run(self): # Found error and return warning submit_pw_check(self.ci_data.pw, self.ci_data.patch_1, self.name, Verdict.WARNING, - "CheckSparse WARNING " + output_str, + f"{self.name} WARNING " + output_str, None, self.dry_run) self.warning(output_str) return @@ -94,7 +101,7 @@ def run(self): # Build success submit_pw_check(self.ci_data.pw, self.ci_data.patch_1, self.name, Verdict.PASS, - "CheckSparse PASS", + f"{self.name} PASS", None, self.dry_run) # Actually no need to call success() here. But add it here just for # reference diff --git a/ci/checkkernelllvm.py b/ci/checkkernelllvm.py new file mode 100755 index 0000000..64b7ea0 --- /dev/null +++ b/ci/checkkernelllvm.py @@ -0,0 +1,33 @@ +import os +import sys +import re + +from ci import Verdict, EndTest, submit_pw_check +from ci import CheckAllWarning + +class CheckKernelLLVM(CheckAllWarning): + """Build kernel with LLVM + context analysis + """ + + def __init__(self, ci_data, kernel_config=None, src_dir=None, dry_run=None): + # Enable context analysis unconditionally + kernel_config = self.make_kernel_config(kernel_config) + + super().__init__(ci_data, kernel_config=kernel_config, src_dir=src_dir, + dry_run=dry_run, make_params=["LLVM=1"]) + + self.name = "CheckKernelLLVM" + self.desc = "Build kernel with LLVM + context analysis" + + def make_kernel_config(self, old_kernel_config): + kernel_config = "/build_kernel_llvm.config" + if not old_kernel_config: + old_kernel_config = '/bluetooth_build.config' + + with open(old_kernel_config, "r") as f: + config_text = f.read() + config_text += "\n\nCONFIG_WARN_CONTEXT_ANALYSIS=y" + with open(kernel_config, "w") as f: + f.write(config_text) + + return kernel_config diff --git a/ci/checksparse.py b/ci/checksparse.py index c2bb890..93c7438 100755 --- a/ci/checksparse.py +++ b/ci/checksparse.py @@ -31,7 +31,8 @@ def __init__(self, ci_data, kernel_config=None, src_dir=None, dry_run=None): self.dry_run = dry_run super().__init__(kernel_config=kernel_config, simple_build=True, - make_params=['C=1'], work_dir=self.src_dir) + make_params=['C=1'], work_dir=self.src_dir, + jobs=ci_data.config.get('jobs')) self.log_dbg("Initialization completed") diff --git a/ci/checkvalgrind.py b/ci/checkvalgrind.py index 1d0d69a..99723e3 100755 --- a/ci/checkvalgrind.py +++ b/ci/checkvalgrind.py @@ -27,7 +27,8 @@ def __init__(self, ci_data): config_params = ["--disable-lsan", "--disable-asan"] make_params = ["check"] super().__init__(config_params=config_params, make_params=make_params, - work_dir=ci_data.src_dir) + work_dir=ci_data.src_dir, + jobs=ci_data.config.get('jobs')) self.log_dbg("Initialization completed") diff --git a/ci/genericbuild.py b/ci/genericbuild.py index 0ccdb01..8689179 100755 --- a/ci/genericbuild.py +++ b/ci/genericbuild.py @@ -15,7 +15,7 @@ class GenericBuild(Base): def __init__(self, config_cmd=None, config_params=None, make_cmd=None, make_params=None, use_fakeroot=False, install=False, install_params=None, - work_dir=None): + work_dir=None, jobs=None): super().__init__() @@ -40,6 +40,8 @@ def __init__(self, config_cmd=None, config_params=None, self.stderr = None + self.jobs = jobs if jobs else "4" + self.log_dbg("Initialization completed") def run(self): @@ -58,7 +60,7 @@ def run(self): # Make # AR: Maybe read from /proc for job count - cmd = [self.make_cmd, "-j4"] + cmd = [self.make_cmd, "-j" + str(self.jobs)] if self.use_fakeroot: cmd = ["fakeroot"] + cmd if self.make_params: diff --git a/ci/generickernelbuild.py b/ci/generickernelbuild.py index 9fa15f8..951a716 100755 --- a/ci/generickernelbuild.py +++ b/ci/generickernelbuild.py @@ -18,7 +18,7 @@ class GenericKernelBuild(Base): """ def __init__(self, kernel_config=None, simple_build=True, - make_params=None, work_dir=None): + make_params=None, work_dir=None, jobs=None): super().__init__() @@ -37,6 +37,8 @@ def __init__(self, kernel_config=None, simple_build=True, # Save the error output self.stderr = None + self.jobs = str(jobs) if jobs else "4" + self.log_dbg("Initialization completed") def run(self): @@ -50,6 +52,8 @@ def run(self): # Update .config self.log_info("GenericKernelBuild: Run make olddefconfig") cmd = ["make", "olddefconfig"] + if self.make_params: + cmd += self.make_params (ret, stdout, stderr) = cmd_run(cmd, cwd=self.work_dir) if ret: self.log_err("GenericKernelBuild: Failed to config the kernel") @@ -58,7 +62,7 @@ def run(self): # make self.log_info("Run make") - base_cmd = ["make", "-j4"] + base_cmd = ["make", "-j" + self.jobs] if self.make_params: base_cmd += self.make_params self.log_dbg(f"GenericKernelBuild: Base Command: {base_cmd}") diff --git a/ci/incrementalbuild.py b/ci/incrementalbuild.py index 5c840a3..12b55b6 100755 --- a/ci/incrementalbuild.py +++ b/ci/incrementalbuild.py @@ -29,6 +29,9 @@ def __init__(self, ci_data, space, kernel_config=None): if self.space == "kernel" and not self.kernel_config: self.kernel_config = '/bluetooth_build.config' + jobs = ci_data.config.get('jobs') + self.jobs = str(jobs) if jobs else "4" + super().__init__() self.log_dbg("Initialization completed") @@ -57,7 +60,7 @@ def _initial_setup(self): def _incremental_make(self): """Run make without reconfiguring - truly incremental.""" - cmd = ["make", "-j4"] + cmd = ["make", "-j" + self.jobs] if self.space == "kernel": # Kernel simple build: only Bluetooth sources cmd.append('net/bluetooth/') diff --git a/ci/makedistcheck.py b/ci/makedistcheck.py index b81ea9e..a383711 100755 --- a/ci/makedistcheck.py +++ b/ci/makedistcheck.py @@ -25,7 +25,8 @@ def __init__(self, ci_data): config_params = ["--disable-lsan", "--disable-asan", "--disable-ubsan"] make_params = ["distcheck"] super().__init__(config_params=config_params, make_params=make_params, - use_fakeroot=True, work_dir=ci_data.src_dir) + use_fakeroot=True, work_dir=ci_data.src_dir, + jobs=ci_data.config.get('jobs')) self.log_dbg("Initialization completed") diff --git a/ci/makeextell.py b/ci/makeextell.py index 759bb2d..7a281a9 100755 --- a/ci/makeextell.py +++ b/ci/makeextell.py @@ -24,7 +24,8 @@ def __init__(self, ci_data): self.ci_data = ci_data config_params = ["--enable-external-ell", "--disable-lsan", "--disable-asan", "--disable-ubsan"] - super().__init__(config_params=config_params, work_dir=ci_data.src_dir) + super().__init__(config_params=config_params, work_dir=ci_data.src_dir, + jobs=ci_data.config.get('jobs')) self.log_dbg("Initialization completed") diff --git a/ci/scanbuild.py b/ci/scanbuild.py index dc736b6..179fc0d 100755 --- a/ci/scanbuild.py +++ b/ci/scanbuild.py @@ -19,6 +19,9 @@ def __init__(self, ci_data): self.desc = "Run Scan Build" self.ci_data = ci_data + jobs = ci_data.config.get('jobs') + self.jobs = str(jobs) if jobs else "4" + super().__init__() self.log_dbg("Initialization completed") @@ -40,7 +43,7 @@ def scan_build(self, error_filename): self.add_failure_end_test(stderr) # Scan Build Make - cmd = ["scan-build", "make", "-j4"] + cmd = ["scan-build", "make", "-j" + self.jobs] (ret, stdout, stderr) = cmd_run(cmd, cwd=self.ci_data.src_dir) if ret: self.log_err("Scan Build failed") diff --git a/entrypoint.sh b/entrypoint.sh index fdb47ae..9e51356 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -2,6 +2,8 @@ set -e +export PATH=/opt/llvm/bin:$PATH + echo "Environment Variables:" echo " Workflow: $GITHUB_WORKFLOW" echo " Action: $GITHUB_ACTION" @@ -171,9 +173,62 @@ case $TASK in exit 1 fi ;; + localci) + echo "Task: local CI" + mkdir -p /work + mkdir -p /work/base + GITHUB_WORKSPACE=/work + BASE_DIR=base + + SPACE="$2" + GITHUB_REPOSITORY="$3" + PR="$4" + + git config --global user.name "local" + git config --global user.email "local@users.noreply.github.com" + + echo "Target ($SPACE): $GITHUB_REPOSITORY PR: $PR" + + ls -l "$GITHUB_WORKSPACE/$BASE_DIR" + if ! test -d "$GITHUB_WORKSPACE/$BASE_DIR/src"; then + echo "Cloning https://github.com/$GITHUB_REPOSITORY" + git clone --depth 1 "https://github.com/$GITHUB_REPOSITORY" \ + $GITHUB_WORKSPACE/$BASE_DIR/src + fi + set_git_safe_dir $GITHUB_WORKSPACE/$BASE_DIR/src + + if ! test -d "$GITHUB_WORKSPACE/$BASE_DIR/ell"; then + clone_ell $GITHUB_WORKSPACE/$BASE_DIR/ell + fi + set_git_safe_dir $GITHUB_WORKSPACE/$BASE_DIR/ell + + mkdir $GITHUB_WORKSPACE/$BASE_DIR/patch + + if [ $SPACE == "kernel" ]; then + if ! test -d "$GITHUB_WORKSPACE/$BASE_DIR/bluez"; then + clone_bluez $GITHUB_WORKSPACE/$BASE_DIR/bluez + fi + set_git_safe_dir $GITHUB_WORKSPACE/$BASE_DIR/bluez + /ci.py -c /config.json -z $GITHUB_WORKSPACE/$BASE_DIR/bluez \ + -e $GITHUB_WORKSPACE/$BASE_DIR/ell \ + -k $GITHUB_WORKSPACE/$BASE_DIR/src \ + -p $GITHUB_WORKSPACE/$BASE_DIR/patch \ + kernel $GITHUB_REPOSITORY $PR \ + --dry-run -j auto + elif [ $SPACE == "user" ]; then + /ci.py -c /config.json -z $GITHUB_WORKSPACE/$BASE_DIR/src \ + -e $GITHUB_WORKSPACE/$BASE_DIR/ell \ + -p $GITHUB_WORKSPACE/$BASE_DIR/patch \ + user $GITHUB_REPOSITORY $PR \ + --dry-run -j auto + else + echo "Unknown SPACE: $SPACE" + exit 1 + fi + ;; *) echo "Unknown TASK: $TASK" - eixt 1 + exit 1 ;; esac diff --git a/libs/context.py b/libs/context.py index eb3bfae..e48a4ed 100755 --- a/libs/context.py +++ b/libs/context.py @@ -43,17 +43,15 @@ def __init__(self, config_file=None, github_repo=None, src_dir=None, # Init github log_info(f"Initialize Github: {github_repo}") if 'GITHUB_TOKEN' not in os.environ: - log_error("Set GITHUB_TOKEN environment variable") - raise ContextError + log_info("GITHUB_TOKEN environment variable not set") # Use a separate token for Check Runs API (requires GitHub App token, # not a PAT). Falls back to GITHUB_TOKEN if not set. - checks_token = os.environ.get('GITHUB_CHECKS_TOKEN', - os.environ['GITHUB_TOKEN']) + token = os.environ.get('GITHUB_TOKEN', None) + checks_token = os.environ.get('GITHUB_CHECKS_TOKEN', token) try: - self.gh = GithubTool(github_repo, os.environ['GITHUB_TOKEN'], - checks_token=checks_token) + self.gh = GithubTool(github_repo, token, checks_token=checks_token) except: log_error("Failed to initialize GithubTool class") raise ContextError