test_controllers.py

import json
import unittest
from importlib import reload
from unittest import mock
from unittest.mock import MagicMock

from flask import g

from mod_auth.models import Role
from mod_ci.controllers import (Workflow_builds, get_info_for_pr_comment,
                                is_valid_commit_hash, mark_test_failed,
                                progress_type_request, retry_with_backoff,
                                safe_db_commit, start_platforms)
from mod_ci.models import BlockedUsers
from mod_customized.models import CustomizedTest
from mod_home.models import CCExtractorVersion, GeneralData
from mod_regression.models import (RegressionTest, RegressionTestOutput,
                                   RegressionTestOutputFiles)
from mod_test.models import Test, TestPlatform, TestResultFile, TestType
from tests.base import (BaseTestCase, MockResponse, create_mock_db_query,
                        create_mock_regression_test, empty_github_token,
                        generate_git_api_header, generate_signature,
                        mock_api_request_github)


class MockGcpInstance:
    """Mock GcpInstance object."""

    def __init__(self, name):
        self.name = name


class MockPlatform:
    """Mock platform object."""

    def __init__(self, platform):
        self.platform = platform
        self.value = 'linux'


class MockStatus:
    """Mock GitHub commit status object."""

    def __init__(self, context):
        self.context = context


class MockFork:
    """Mock fork object."""

    def __init__(self, *args, **kwargs):
        self.github = None


class MockTest:
    """Mock test object."""

    def __init__(self):
        self.id = 1
        self.test_type = TestType.commit
        self.fork = MockFork()
        self.platform = MockPlatform(TestPlatform.linux)


WSGI_ENVIRONMENT = {'REMOTE_ADDR': "192.30.252.0"}


class TestControllers(BaseTestCase):
    """Test CI-related controllers."""

    def test_comment_info_handles_variant_files_correctly(self):
        """Test that allowed variants of output files are handled correctly in PR comments.

        Make sure that the info used for the PR comment treats tests as successes if they got one of the allowed
        variants of the output file instead of the original version
        """
        VARIANT_HASH = 'abcdefgh_this_is_a_hash'
        TEST_RUN_ID = 1
        # Setting up the database
        # create regression test with an output file that has an allowed variant
        regression_test: RegressionTest = RegressionTest.query.filter(RegressionTest.id == 1).first()
        output_file: RegressionTestOutput = regression_test.output_files[0]
        variant_output_file = RegressionTestOutputFiles(VARIANT_HASH, output_file.id)
        output_file.multiple_files.append(variant_output_file)
        g.db.add(output_file)

        test_run_output_file: TestResultFile = TestResultFile.query.filter(
            TestResultFile.regression_test_output_id == output_file.id,
            TestResultFile.test_id == TEST_RUN_ID
        ).first()
        # mark the test as getting the variant as output, not the expected file
        test_run_output_file.got = VARIANT_HASH
        g.db.add(test_run_output_file)
        g.db.commit()

        # The actual test
        test: Test = Test.query.get(TEST_RUN_ID)
        comment_info = get_info_for_pr_comment(test)
        # we got a valid variant, so should still pass
        self.assertEqual(comment_info.common_failed_tests, [])
        self.assertEqual(comment_info.extra_failed_tests, [])
        self.assertEqual(comment_info.fixed_tests, [])
        for stats in comment_info.category_stats:
            # make sure the stats for the category confirm that everything passed too
            self.assertEqual(stats.success, stats.total)

    def test_comment_info_handles_invalid_variants_correctly(self):
        """Test that invalid variants of output files are handled correctly in PR comments.

        Make sure that regression tests are correctly marked as not passing when an invalid file hash is found
        """
        INVALID_VARIANT_HASH = 'this_is_an_invalid_hash'
        TEST_RUN_ID = 1
        test_result_file: TestResultFile = TestResultFile.query.filter(TestResultFile.test_id == TEST_RUN_ID).first()
        test_result_file.got = INVALID_VARIANT_HASH
        g.db.add(test_result_file)
        g.db.commit()

        test: Test = Test.query.get(TEST_RUN_ID)
        comment_info = get_info_for_pr_comment(test)
        # all categories that this regression test applies to should fail because of the invalid hash
        for category in test_result_file.regression_test.categories:
            stats = [stat for stat in comment_info.category_stats if stat.category == category.name]
            for stat in stats:
                self.assertEqual(stat.success, 0)

    @mock.patch('mod_ci.controllers.get_compute_service_object')
    @mock.patch('mod_ci.controllers.delete_expired_instances')
    @mock.patch('mod_ci.controllers.gcp_instance')
    @mock.patch('run.log')
    def test_start_platform_none_specified(self, mock_log, mock_gcp_instance,
                                           mock_delete_expired_instances, mock_get_compute_service_object):
        """Test that both platforms run when no platform value is passed."""
        start_platforms(mock.ANY, 1)

        mock_delete_expired_instances.assert_called_once()
        mock_get_compute_service_object.assert_called_once()
        self.assertEqual(2, mock_gcp_instance.call_count)
        self.assertEqual(4, mock_log.info.call_count)

    @mock.patch('mod_ci.controllers.get_compute_service_object')
    @mock.patch('mod_ci.controllers.delete_expired_instances')
    @mock.patch('mod_ci.controllers.gcp_instance')
    @mock.patch('run.log')
    def test_start_platform_linux_specified(self, mock_log, mock_gcp_instance,
                                            mock_delete_expired_instances, mock_get_compute_service_object):
        """Test that only Linux platform runs when platform is specified as Linux."""
        start_platforms(mock.ANY, platform=TestPlatform.linux)

        self.assertEqual(1, mock_gcp_instance.call_count)
        self.assertEqual(2, mock_log.info.call_count)
        mock_log.info.assert_called_with("Linux GCP instances process kicked off")
        mock_delete_expired_instances.assert_called_once()
        mock_get_compute_service_object.assert_called_once()

    @mock.patch('mod_ci.controllers.get_compute_service_object')
    @mock.patch('mod_ci.controllers.delete_expired_instances')
    @mock.patch('mod_ci.controllers.gcp_instance')
    @mock.patch('run.log')
    def test_start_platform_windows_specified(self, mock_log, mock_gcp_instance,
                                              mock_delete_expired_instances, mock_get_compute_service_object):
        """Test that only Windows platform runs when platform is specified as Windows."""
        start_platforms(mock.ANY, platform=TestPlatform.windows)

        self.assertEqual(1, mock_gcp_instance.call_count)
        self.assertEqual(2, mock_log.info.call_count)
        mock_log.info.assert_called_with("Windows GCP instances process kicked off")
        mock_delete_expired_instances.assert_called_once()
        mock_get_compute_service_object.assert_called_once()

    @mock.patch('run.log')
    @mock.patch('mod_ci.controllers.MaintenanceMode')
    def test_gcp_instance_maintenance_mode(self, mock_maintenance, mock_log):
        """Test that gcp instance does not run when in maintainenace."""
        from mod_ci.controllers import gcp_instance

        class MockMaintenance:
            def __init__(self):
                self.disabled = True

        mock_maintenance.query.filter.return_value.first.return_value = MockMaintenance()

        gcp_instance(mock.ANY, mock.ANY, "test", mock.ANY, 1)

        mock_log.info.assert_called_once()
        mock_log.critical.assert_not_called()
        self.assertEqual(mock_log.debug.call_count, 2)

    @mock.patch('github.Github')
    @mock.patch('mod_ci.controllers.delete_expired_instances')
    @mock.patch('mod_ci.controllers.get_compute_service_object')
    @mock.patch('mod_ci.controllers.gcp_instance')
    def test_cron_job_testing_false(self, mock_gcp_instance, mock_get_compute_service_object,
                                    mock_delete_expired_instances, mock_github):
        """Test working of cron function when testing is disabled."""
        from mod_ci.cron import cron
        mock_delete_expired_instances.reset_mock()
        mock_get_compute_service_object.reset_mock()
        cron()
        self.assertEqual(mock_delete_expired_instances.call_count, 1)
        self.assertEqual(mock_get_compute_service_object.call_count, 1)

    @mock.patch('github.Github')
    @mock.patch('mod_ci.controllers.delete_expired_instances')
    @mock.patch('mod_ci.controllers.get_compute_service_object')
    @mock.patch('mod_ci.controllers.gcp_instance')
    def test_cron_job_testing_true(self, mock_gcp_instance, mock_get_compute_service_object,
                                   mock_delete_expired_instances, mock_github):
        """Test working of cron function when testing is enabled."""
        from mod_ci.cron import cron
        mock_delete_expired_instances.reset_mock()
        mock_get_compute_service_object.reset_mock()
        cron(testing=True)
        mock_delete_expired_instances.assert_not_called()
        mock_get_compute_service_object.assert_not_called()

    @mock.patch.dict('run.config', {'GITHUB_TOKEN': '', 'GITHUB_OWNER': 'test', 'GITHUB_REPOSITORY': 'test'})
    @mock.patch('run.log')
    def test_cron_job_empty_token(self, mock_log):
        """Test cron returns early when GitHub token is empty."""
        from mod_ci.cron import cron
        cron()
        mock_log.error.assert_called_with('GITHUB_TOKEN not configured, cannot run CI cron')

    @mock.patch('mod_ci.controllers.wait_for_operation')
    @mock.patch('mod_ci.controllers.create_instance')
    @mock.patch('builtins.open', new_callable=mock.mock_open())
    @mock.patch('mod_ci.controllers.g')
    @mock.patch('mod_ci.controllers.TestProgress')
    @mock.patch('mod_ci.controllers.GcpInstance')
    def test_start_test(self, mock_gcp_instance, mock_test_progress, mock_g, mock_open_file,
                        mock_create_instance, mock_wait_for_operation):
        """Test start_test function."""
        import zipfile

        import requests
        from github.Artifact import Artifact

        from mod_ci.controllers import Artifact_names, start_test

        # Mock locking checks to return None (no existing instances/progress)
        mock_gcp_instance.query.filter.return_value.first.return_value = None
        mock_test_progress.query.filter.return_value.first.return_value = None

        test = Test.query.first()
        repository = MagicMock()

        artifact1 = MagicMock(Artifact)
        artifact1.name = Artifact_names.windows
        artifact1.workflow_run.head_sha = '1978060bf7d2edd119736ba3ba88341f3bec3322'

        artifact2 = MagicMock(Artifact)
        artifact2.name = Artifact_names.linux
        artifact2.workflow_run.head_sha = '1978060bf7d2edd119736ba3ba88341f3bec3323'

        class mock_zip:
            def __enter__(self):
                return self

            def __exit__(self, *args):
                return False

            def extractall(*args, **kwargs):
                return None
        repository.get_artifacts.return_value = [artifact1, artifact2]
        response = requests.models.Response()
        response.status_code = 200
        requests.get = MagicMock(return_value=response)
        zipfile.ZipFile = MagicMock(return_value=mock_zip())
        customized_test = CustomizedTest(1, 1)
        g.db.add(customized_test)
        g.db.commit()

        # Set up mock db query chain to avoid AsyncMock behavior in Python 3.13+
        mock_query = create_mock_db_query(mock_g)
        mock_query.c.got = MagicMock()

        # Test when gcp create instance fails synchronously (error in operation response)
        mock_create_instance.return_value = {'error': {'errors': [{'code': 'TEST_ERROR'}]}}
        start_test(mock.ANY, self.app, mock_g.db, repository, test, mock.ANY)
        # Commit IS called to record the test failure in the database
        mock_g.db.commit.assert_called_once()
        mock_g.db.commit.reset_mock()
        mock_create_instance.reset_mock()

        # Test when gcp create instance is successful and verified
        mock_create_instance.return_value = {'name': 'test-operation-123', 'status': 'RUNNING'}
        mock_wait_for_operation.return_value = {'status': 'DONE'}  # Success
        start_test(mock.ANY, self.app, mock_g.db, repository, test, mock.ANY)
        mock_g.db.commit.assert_called_once()
        mock_create_instance.assert_called_once()

    @mock.patch('github.Github.get_repo')
    @mock.patch('mod_ci.controllers.start_test')
    @mock.patch('mod_ci.controllers.get_compute_service_object')
    @mock.patch('mod_ci.controllers.g')
    def test_gcp_instance(self, mock_g, mock_get_compute_service_object, mock_start_test, mock_repo):
        """Test gcp_instance function."""
        from mod_ci.controllers import gcp_instance

        repo = mock_repo()

        # Making test with id 1 invalid with pr_nr = 0
        test_1 = Test.query.get(1)
        test_1.pr_nr = 0

        # Test with id 2 has a different commit than PR head, but we still run it
        # (we no longer cancel tests just because a newer commit was pushed)
        test_2 = Test.query.get(2)
        pr_head_sha = test_2.commit + 'f'
        repo.get_pull.return_value.head.sha = pr_head_sha
        repo.get_pull.return_value.state = 'open'

        # Creating a test with pull_request type that is valid
        test_3 = Test(TestPlatform.linux, TestType.pull_request, 1, "pull_request", pr_head_sha, 2)
        g.db.add(test_3)

        # Creating a test with commit type that is valid
        test_4 = Test(TestPlatform.linux, TestType.commit, 1, "master", pr_head_sha)
        g.db.add(test_4)
        g.db.commit()

        # Set up mock db query chain to avoid AsyncMock behavior in Python 3.13+
        create_mock_db_query(mock_g)

        gcp_instance(self.app, mock_g.db, TestPlatform.linux, repo, None)

        # test_1 is descheduled (invalid pr_nr=0)
        # test_2, test_3, test_4 all run (3 total)
        self.assertEqual(mock_start_test.call_count, 3)
        mock_get_compute_service_object.assert_called_once()

    def test_get_compute_service_object(self):
        """Test get_compute_service_object function."""
        import googleapiclient
        from google.oauth2 import service_account

        from mod_ci.controllers import get_compute_service_object
        service_account.Credentials.from_service_account_file = MagicMock()
        compute = get_compute_service_object()
        self.assertEqual(type(compute), googleapiclient.discovery.Resource)

    @mock.patch('builtins.open', new_callable=mock.mock_open())
    def test_create_instance_linux(self, mock_open_file):
        """Test create_instance function for linux platform."""
        from mod_ci.controllers import create_instance
        compute = MagicMock()
        instance = create_instance(compute, "test", "test", Test.query.get(1), "")

        mock_open_file.assert_called()
        self.assertEqual(str(type(instance)), str(MagicMock))

    @mock.patch('builtins.open', new_callable=mock.mock_open())
    def test_create_instance_windows(self, mock_open_file):
        """Test create_instance function for windows platform."""
        from mod_ci.controllers import create_instance
        compute = MagicMock()
        new_test = Test(TestPlatform.windows, TestType.commit, 1, "test", "test")
        g.db.add(new_test)
        g.db.commit()
        instance = create_instance(compute, "test", "test", new_test, "")

        mock_open_file.assert_called()
        self.assertEqual(str(type(instance)), str(MagicMock))

    @mock.patch('mod_ci.controllers.GeneralData')
    @mock.patch('mod_ci.controllers.g')
    def test_set_avg_time_first(self, mock_g, mock_gd):
        """Test setting average time for the first time."""
        from mod_ci.controllers import set_avg_time

        mock_gd.query.filter.return_value.first.return_value = None

        set_avg_time(TestPlatform.linux, "build", 100)

        mock_gd.query.filter.assert_called_once_with(mock_gd.key == 'avg_build_count_linux')
        self.assertEqual(mock_gd.call_count, 2)
        self.assertEqual(mock_g.db.add.call_count, 2)
        mock_g.db.commit.assert_called_once()

    @mock.patch('mod_ci.controllers.int')
    @mock.patch('mod_ci.controllers.GeneralData')
    @mock.patch('mod_ci.controllers.g')
    def test_set_avg_time(self, mock_g, mock_gd, mock_int):
        """Test setting average time for NOT first time."""
        from mod_ci.controllers import set_avg_time

        mock_int.return_value = 5

        set_avg_time(TestPlatform.windows, "prep", 100)

        mock_gd.query.filter.assert_called_with(mock_gd.key == 'avg_prep_count_windows')
        self.assertEqual(mock_gd.call_count, 0)
        self.assertEqual(mock_g.db.add.call_count, 0)
        mock_g.db.commit.assert_called_once()

    @mock.patch('github.Github')
    def test_comments_successfully_in_passed_pr_test(self, mock_github):
        """Check comments in passed PR test."""
        import mod_ci.controllers
        reload(mod_ci.controllers)
        from github.IssueComment import IssueComment

        from mod_ci.controllers import Status, comment_pr
        from mod_test.models import Test
        pull_request = mock_github.return_value.get_repo.return_value.get_pull(number=1)
        pull_request.get_issue_comments.return_value = [MagicMock(IssueComment)]

        # Comment on test that fails some/all regression tests
        test = Test.query.get(2)
        comment_pr(test)
        pull_request.get_issue_comments.assert_called_with()
        args, kwargs = pull_request.create_issue_comment.call_args
        message = kwargs['body']
        if "passed" not in message:
            assert False, "Message not Correct"

    @mock.patch('mod_test.controllers.get_test_results')
    @mock.patch('github.Github')
    def test_comments_successfuly_in_failed_pr_test(self, mock_github, mock_get_test_results):
        """Check comments in failed PR test."""
        import mod_ci.controllers
        reload(mod_ci.controllers)
        from github.IssueComment import IssueComment

        from mod_ci.controllers import comment_pr
        from mod_regression.models import Category, RegressionTest
        from mod_test.models import Test
        pull_request = mock_github.return_value.get_repo.return_value.get_pull(number=1)
        pull_request.get_issue_comments.return_value = [MagicMock(IssueComment)]
        regression_test = RegressionTest.query.first()
        mock_get_test_results.return_value = [{'category': MagicMock(Category), 'tests': [{
            'test': regression_test, 'error': True}]}]
        # Comment on test that fails some/all regression tests
        test = Test.query.get(2)
        comment_pr(test)
        pull_request.get_issue_comments.assert_called_with()
        args, kwargs = pull_request.create_issue_comment.call_args
        message = kwargs['body']
        if regression_test.command not in message:
            assert False, "Message not Correct"

    def test_get_running_instances(self):
        """Test get_running_instances function."""
        from mod_ci.controllers import get_running_instances
        result = get_running_instances(MagicMock(), "test", "test")
        self.assertEqual(result, [])

    def test_check_main_repo_returns_in_false_url(self):
        """Test main repo checking."""
        from mod_ci.controllers import is_main_repo
        assert is_main_repo('random_user/random_repo') is False
        assert is_main_repo('test_owner/test_repo') is True

    @mock.patch('github.Github.get_repo')
    @mock.patch('mod_ci.controllers.update_status_on_github')
    @mock.patch('mod_ci.controllers.get_running_instances')
    def test_delete_expired_instances(self, mock_get_running_instances, mock_update_github_status, mock_repo):
        """Test working of delete_expired_instances function."""
        from datetime import datetime, timedelta, timezone

        from mod_ci.controllers import delete_expired_instances

        current_timestamp = datetime.now(timezone.utc)
        expired_instance_time = current_timestamp - timedelta(minutes=150)
        mock_get_running_instances.return_value = [{
            'name': 'windows-1',
            'creationTimestamp': current_timestamp.strftime("%Y-%m-%dT%H:%M:%S.%f%z"),
        }, {
            'name': 'linux-2',
            'creationTimestamp': expired_instance_time.strftime("%Y-%m-%dT%H:%M:%S.%f%z"),
        }, {
            'name': 'osx-3',
            'creationTimestamp': current_timestamp.strftime("%Y-%m-%dT%H:%M:%S.%f%z"),
        }]
        compute = MagicMock()
        pendingOperations = [
            {'status': "DONE"},
            {'status': "PENDING"}
        ]
        compute.zoneOperations.return_value.get.return_value.execute = pendingOperations.pop
        delete_expired_instances(compute, 120, 'a', 'a', MagicMock(), MagicMock())
        mock_get_running_instances.assert_called_once()
        mock_update_github_status.assert_called_once()

    def test_customizedtest_added_to_queue(self):
        """Test queue with a customized test addition."""
        regression_test = RegressionTest.query.filter(RegressionTest.id == 1).first()
        regression_test.active = False
        g.db.add(regression_test)
        g.db.commit()
        import mod_ci.controllers
        reload(mod_ci.controllers)
        from mod_ci.controllers import add_test_entry, queue_test

        # Use valid hex commit hash (customizedcommitcheck is not valid hex)
        test_commit = 'abc1234def567890abc1234def567890abcd1234'
        add_test_entry(g.db, test_commit, TestType.commit)
        queue_test(None, test_commit, TestType.commit, TestPlatform.linux)
        queue_test(None, test_commit, TestType.commit, TestPlatform.windows)
        test = Test.query.filter(Test.id == 3).first()
        customized_test = test.get_customized_regressiontests()
        self.assertIn(2, customized_test)
        self.assertNotIn(1, customized_test)

    @mock.patch('flask.g.log.error')
    @mock.patch('mailer.Mailer')
    @mock.patch('mod_ci.controllers.get_html_issue_body')
    def test_inform_mailing_list(self, mock_get_html_issue_body, mock_email, mock_log_error):
        """Test the inform_mailing_list function."""
        from mod_ci.controllers import inform_mailing_list

        mock_get_html_issue_body.return_value = """2430 - Some random string\n\n
                     Link to Issue: https://www.github.com/test_owner/test_repo/issues/matejmecka\n\n
                     Some random string(https://github.com/Some random string)\n\n\n
                     Lorem Ipsum sit dolor amet...\n        """
        inform_mailing_list(mock_email, "matejmecka", "2430", "Some random string", "Lorem Ipsum sit dolor amet...")

        mock_email.send_simple_message.assert_called_once_with(
            {
                'to': 'ccextractor-dev@googlegroups.com',
                'subject': 'GitHub Issue #matejmecka',
                'html': """2430 - Some random string\n\n
                     Link to Issue: https://www.github.com/test_owner/test_repo/issues/matejmecka\n\n
                     Some random string(https://github.com/Some random string)\n\n\n
                     Lorem Ipsum sit dolor amet...\n        """
            }
        )
        mock_get_html_issue_body.assert_called_once()
        mock_log_error.assert_not_called()

        mock_email.send_simple_message.return_value = False
        inform_mailing_list(mock_email, "matejmecka", "2430", "Some random string", "Lorem Ipsum sit dolor amet...")
        mock_log_error.assert_called_once()

    @staticmethod
    @mock.patch('mod_ci.controllers.markdown')
    def test_get_html_issue_body(mock_markdown):
        """Test the get_html_issue_body for correct email formatting."""
        from mod_ci.controllers import get_html_issue_body

        title = "[BUG] Test Title"
        author = "abcxyz"
        body = "i'm issue body"
        issue_number = 1
        url = "www.example.com"

        get_html_issue_body(title, author, body, issue_number, url)

        mock_markdown.assert_called_once_with(body, extras=["target-blank-links", "task_list", "code-friendly"])

    @mock.patch('mod_ci.controllers.update_status_on_github')
    @mock.patch('github.Github.get_repo')
    def test_add_blocked_users(self, mock_repo, mock_update_gh_status):
        """Check adding a user to block list."""
        from github.PullRequest import PullRequest
        self.create_user_with_role(self.user.name, self.user.email, self.user.password, Role.admin)
        mock_pr = MagicMock(PullRequest)
        mock_pr.user.id = 1
        mock_pr.number = 3
        mock_repo.return_value.get_pulls.return_value = [mock_pr]

        new_test = Test(TestPlatform.linux, TestType.pull_request, 1, "test", "test", 3)
        g.db.add(new_test)
        g.db.commit()

        with self.app.test_client() as c:
            c.post("/account/login", data=self.create_login_form_data(self.user.email, self.user.password))
            c.post("/blocked_users", data=dict(user_id=1, comment="Bad user", add=True))
            self.assertNotEqual(BlockedUsers.query.filter(BlockedUsers.user_id == 1).first(), None)
            with c.session_transaction() as session:
                flash_message = dict(session['_flashes']).get('message')
            self.assertEqual(flash_message, "User blocked successfully.")

    def test_add_blocked_users_wrong_id(self):
        """Check adding invalid user id to block list."""
        self.create_user_with_role(self.user.name, self.user.email, self.user.password, Role.admin)
        with self.app.test_client() as c:
            c.post("/account/login", data=self.create_login_form_data(self.user.email, self.user.password))
            response = c.post("/blocked_users", data=dict(user_id=0, comment="Bad user", add=True))
            self.assertEqual(BlockedUsers.query.filter(BlockedUsers.user_id == 0).first(), None)
            self.assertIn("GitHub User ID not filled in", str(response.data))

    def test_add_blocked_users_empty_id(self):
        """Check adding blank user id to block list."""
        self.create_user_with_role(
            self.user.name, self.user.email, self.user.password, Role.admin)
        with self.app.test_client() as c:
            c.post("/account/login", data=self.create_login_form_data(self.user.email, self.user.password))
            response = c.post("/blocked_users", data=dict(comment="Bad user", add=True))
            self.assertEqual(BlockedUsers.query.filter(BlockedUsers.user_id.is_(None)).first(), None)
            self.assertIn("GitHub User ID not filled in", str(response.data))

    @mock.patch('requests.get', return_value=MockResponse({"login": "test"}, 200))
    def test_add_blocked_users_already_exists(self, mock_request):
        """Check adding existing blocked user again."""
        self.create_user_with_role(
            self.user.name, self.user.email, self.user.password, Role.admin)
        with self.app.test_client() as c:
            c.post("/account/login", data=self.create_login_form_data(self.user.email, self.user.password))
            blocked_user = BlockedUsers(1, "Bad user")
            g.db.add(blocked_user)
            g.db.commit()
            c.post("/blocked_users", data=dict(user_id=1, comment="Bad user", add=True))
            with c.session_transaction() as session:
                flash_message = dict(session['_flashes']).get('message')
            self.assertEqual(flash_message, "User already blocked.")

    def test_remove_blocked_users(self):
        """Check removing user from block list."""
        self.create_user_with_role(
            self.user.name, self.user.email, self.user.password, Role.admin)
        with self.app.test_client() as c:
            c.post("/account/login", data=self.create_login_form_data(self.user.email, self.user.password))
            blocked_user = BlockedUsers(1, "Bad user")
            g.db.add(blocked_user)
            g.db.commit()
            self.assertNotEqual(BlockedUsers.query.filter(BlockedUsers.comment == "Bad user").first(), None)
            c.post("/blocked_users/1", data=dict(remove=True))
            self.assertEqual(BlockedUsers.query.filter(BlockedUsers.user_id == 1).first(), None)
            with c.session_transaction() as session:
                flash_message = dict(session['_flashes']).get('message')
            self.assertEqual(flash_message, "User removed successfully.")

    def test_remove_blocked_users_wrong_id(self):
        """Check removing non existing id from block list."""
        self.create_user_with_role(
            self.user.name, self.user.email, self.user.password, Role.admin)
        with self.app.test_client() as c:
            c.post("/account/login", data=self.create_login_form_data(self.user.email, self.user.password))
            c.post("/blocked_users/7355608", data=dict(remove=True))
            with c.session_transaction() as session:
                flash_message = dict(session['_flashes']).get('message')
            self.assertEqual(flash_message, "No such user in Blacklist")

    def test_remove_blocked_users_invalid_id(self):
        """Check invalid id for the blocked_user url."""
        self.create_user_with_role(
            self.user.name, self.user.email, self.user.password, Role.admin)
        with self.app.test_client() as c:
            c.post("/account/login", data=self.create_login_form_data(self.user.email, self.user.password))
            response = c.post("/blocked_users/hello", data=dict(remove=True))
            self.assertEqual(response.status_code, 404)

    @mock.patch('requests.get', side_effect=mock_api_request_github)
    def test_webhook_wrong_url(self, mock_request):
        """Check webhook fails when ping with wrong url."""
        with self.app.test_client() as c:
            # non GitHub ip address
            wsgi_environment = {'REMOTE_ADDR': '0.0.0.0'}
            data = {'action': "published",
                    'release': {'prerelease': False, 'published_at': "2018-05-30T20:18:44Z", 'tag_name': "0.0.1"}}
            response = c.post("/start-ci", environ_overrides=wsgi_environment,
                              data=json.dumps(data), headers=self.generate_header(data, "ping"))
            self.assertNotEqual(response.status_code, 200)

    @mock.patch('requests.get', side_effect=mock_api_request_github)
    def test_webhook_ping(self, mock_request):
        """Check webhook release update CCExtractor Version for ping."""
        with self.app.test_client() as c:
            data = {'action': 'published',
                    'release': {'prerelease': False, 'published_at': '2018-05-30T20:18:44Z', 'tag_name': '0.0.1'}}
            response = c.post(
                '/start-ci', environ_overrides=WSGI_ENVIRONMENT,
                data=json.dumps(data), headers=self.generate_header(data, 'ping'))
            self.assertEqual(response.status_code, 200)
            self.assertEqual(response.data, b'{"msg": "Hi!"}')

    @mock.patch('github.Github.get_repo')
    @mock.patch('requests.get', side_effect=mock_api_request_github)
    def test_webhook_release(self, mock_request, mock_repo):
        """Check webhook release update CCExtractor Version for release."""
        # Setup mock for tag ref (lightweight tag)
        mock_tag_ref = MagicMock()
        mock_tag_ref.object.type = "commit"
        mock_tag_ref.object.sha = "abcdef12345678901234567890123456789abcde"
        mock_repo.return_value.get_git_ref.return_value = mock_tag_ref

        with self.app.test_client() as c:
            # Full Release with version with 2.1
            data = {'action': 'published',
                    'release': {'prerelease': False, 'published_at': '2018-05-30T20:18:44Z', 'tag_name': 'v2.1'}}
            response = c.post(
                '/start-ci', environ_overrides=WSGI_ENVIRONMENT,
                data=json.dumps(data), headers=self.generate_header(data, 'release'))
            last_release = CCExtractorVersion.query.order_by(CCExtractorVersion.released.desc()).first()
            self.assertEqual(last_release.version, '2.1')

    @mock.patch('github.Github.get_repo')
    @mock.patch('requests.get', side_effect=mock_api_request_github)
    def test_webhook_release_edited(self, mock_request, mock_repo):
        """Check webhook action "edited" updates the specified version."""
        from datetime import datetime

        # Setup mock for tag ref
        mock_tag_ref = MagicMock()
        mock_tag_ref.object.type = "commit"
        mock_tag_ref.object.sha = "fedcba09876543210fedcba09876543210fedcba"
        mock_repo.return_value.get_git_ref.return_value = mock_tag_ref

        with self.app.test_client() as c:
            release = CCExtractorVersion('2.1', '2018-05-30T20:18:44Z', 'abcdefgh12345678abcdefgh12345678abcdefgh')
            g.db.add(release)
            g.db.commit()
            # Full Release with version with 2.1
            data = {'action': 'edited',
                    'release': {'prerelease': False, 'published_at': '2018-06-30T20:18:44Z', 'tag_name': 'v2.1'}}
            response = c.post(
                '/start-ci', environ_overrides=WSGI_ENVIRONMENT,
                data=json.dumps(data), headers=self.generate_header(data, 'release'))
            last_release = CCExtractorVersion.query.filter_by(version='2.1').first()
            self.assertEqual(last_release.released,
                             datetime.strptime('2018-06-30T20:18:44Z', '%Y-%m-%dT%H:%M:%SZ').date())

    @mock.patch('github.Github.get_repo')
    @mock.patch('requests.get', side_effect=mock_api_request_github)
    def test_webhook_release_deleted(self, mock_request, mock_repo):
        """Check webhook action "delete" removes the specified version."""
        with self.app.test_client() as c:
            # Use valid commit hash
            release = CCExtractorVersion('2.1', '2018-05-30T20:18:44Z', '1111111111111111111111111111111111111111')
            g.db.add(release)
            g.db.commit()
            # Delete full release with version with 2.1
            data = {'action': 'deleted',
                    'release': {'prerelease': False, 'published_at': '2018-05-30T20:18:44Z', 'tag_name': 'v2.1'}}
            response = c.post(
                '/start-ci', environ_overrides=WSGI_ENVIRONMENT,
                data=json.dumps(data), headers=self.generate_header(data, 'release'))
            last_release = CCExtractorVersion.query.order_by(CCExtractorVersion.released.desc()).first()
            self.assertNotEqual(last_release.version, '2.1')

    def test_webhook_prerelease(self):
        """Check webhook release update CCExtractor Version for prerelease."""
        with self.app.test_client() as c:
            # Full Release with version with 2.1 (prereleased action is ignored)
            data = {'action': 'prereleased',
                    'release': {'prerelease': True, 'published_at': '2018-05-30T20:18:44Z', 'tag_name': 'v2.1'}}
            sig = generate_signature(str(json.dumps(data)).encode('utf-8'), g.github['ci_key'])
            headers = generate_git_api_header('release', sig)
            response = c.post(
                '/start-ci', environ_overrides=WSGI_ENVIRONMENT,
                data=json.dumps(data), headers=self.generate_header(data, 'ping'))
            last_release = CCExtractorVersion.query.order_by(CCExtractorVersion.released.desc()).first()
            self.assertNotEqual(last_release.version, '2.1')

    @mock.patch('github.Github.get_repo')
    @mock.patch('requests.get', side_effect=mock_api_request_github)
    @mock.patch('run.log')
    def test_webhook_push_no_after(self, mock_log, mock_request, mock_repo):
        """Test webhook triggered with push event without 'after' in payload."""
        data = {'no_after': 'test'}
        with self.app.test_client() as c:
            response = c.post(
                '/start-ci', environ_overrides=WSGI_ENVIRONMENT,
                data=json.dumps(data), headers=self.generate_header(data, 'push'))

        mock_log.debug.assert_called_with("push event detected")
        mock_log.warning.assert_any_call("Unknown push type! Dumping payload for analysis")

    @mock.patch('github.Github.get_repo')
    @mock.patch('requests.get', side_effect=mock_api_request_github)
    @mock.patch('mod_ci.controllers.add_test_entry')
    @mock.patch('mod_ci.controllers.GeneralData')
    def test_webhook_push_valid(self, mock_gd, mock_add_test_entry, mock_request, mock_repo):
        """Test webhook triggered with push event with valid data."""
        data = {'after': 'abcdefgh', 'ref': 'refs/heads/master'}
        with self.app.test_client() as c:
            response = c.post(
                '/start-ci', environ_overrides=WSGI_ENVIRONMENT,
                data=json.dumps(data), headers=self.generate_header(data, 'push'))

        mock_gd.query.filter.assert_called()
        mock_add_test_entry.assert_called_once()

    @mock.patch('github.Github.get_repo')
    @mock.patch('mod_ci.controllers.Test')
    @mock.patch('requests.get', side_effect=mock_api_request_github)
    def test_webhook_pr_closed(self, mock_requests, mock_test, mock_repo):
        """Test webhook triggered with pull_request event with closed action."""
        platform_name = "platform"

        class MockTest:
            def __init__(self):
                self.id = 1
                self.progress = []
                self.platform = MockPlatform(platform_name)
                self.commit = "test"

        mock_test.query.filter.return_value.all.return_value = [MockTest()]
        # Use MockStatus object instead of dict - code expects .context attribute
        # Use 'linux' to match MockPlatform.value
        mock_repo.return_value.get_commit.return_value.get_statuses.return_value = [
            MockStatus("CI - linux")]

        data = {'action': 'closed',
                'pull_request': {'number': 1234, 'draft': False}}
        # one of ip address from GitHub web hook
        with self.app.test_client() as c:
            response = c.post(
                '/start-ci', environ_overrides=WSGI_ENVIRONMENT,
                data=json.dumps(data), headers=self.generate_header(data, 'pull_request'))

        mock_test.query.filter.assert_called_once()

    @mock.patch('mod_ci.controllers.BlockedUsers')
    @mock.patch('github.Github.get_repo')
    @mock.patch('requests.get', side_effect=mock_api_request_github)
    def test_webhook_pr_opened_blocked(self, mock_request, mock_repo, mock_blocked):
        """Test webhook triggered with pull_request event with opened action for blocked user."""
        data = {'action': 'opened',
                'pull_request': {'number': '1234', 'head': {'sha': 'abcd1234'}, 'user': {'id': 'test'}, 'draft': False}}
        with self.app.test_client() as c:
            response = c.post(
                '/start-ci', environ_overrides=WSGI_ENVIRONMENT,
                data=json.dumps(data), headers=self.generate_header(data, 'pull_request'))

        self.assertEqual(response.data, b'ERROR')
        mock_blocked.query.filter.assert_called_once()

    @mock.patch('mod_ci.controllers.BlockedUsers')
    @mock.patch('github.Github.get_repo')
    @mock.patch('mod_ci.controllers.add_test_entry')
    @mock.patch('requests.get', side_effect=mock_api_request_github)
    def test_webhook_pr_opened(self, mock_request, mock_add_test_entry, mock_repo, mock_blocked):
        """Test webhook triggered with pull_request event with opened action."""
        mock_blocked.query.filter.return_value.first.return_value = None

        data = {'action': 'opened',
                'pull_request': {'number': 1234, 'head': {'sha': 'abcd1234'}, 'user': {'id': 'test'}, 'draft': False}}
        with self.app.test_client() as c:
            response = c.post(
                '/start-ci', environ_overrides=WSGI_ENVIRONMENT,
                data=json.dumps(data), headers=self.generate_header(data, 'pull_request'))

        self.assertEqual(response.data, b'{"msg": "EOL"}')
        mock_blocked.query.filter.assert_called_once_with(mock_blocked.user_id == 'test')
        mock_add_test_entry.assert_called_once()

    @mock.patch('mod_ci.controllers.BlockedUsers')
    @mock.patch('github.Github.get_repo')
    @mock.patch('requests.get', side_effect=mock_api_request_github)
    def test_webhook_pr_invalid_action(self, mock_request, mock_repo, mock_blocked):
        """Test webhook triggered with pull_request event with an invalid action."""
        data = {'action': 'invalid',
                'pull_request': {'number': 1234, 'draft': False, 'head': {'sha': 'abcd1234'}, 'user': {'id': 'test'}}}
        with self.app.test_client() as c:
            response = c.post(
                '/start-ci', environ_overrides=WSGI_ENVIRONMENT,
                data=json.dumps(data), headers=self.generate_header(data, 'pull_request'))

        self.assertEqual(response.data, b'{"msg": "EOL"}')
        mock_blocked.query.filter.assert_not_called()

    @mock.patch('github.Github.get_repo')
    @mock.patch('requests.get', side_effect=mock_api_request_github)
    def test_webhook_pr_opened_draft(self, mock_request, mock_repo):
        """Test webhook triggered with pull_request event with open action, marked as draft."""
        data = {'action': 'opened',
                'pull_request': {'number': 1234, 'head': {'sha': 'abcd1234'}, 'user': {'id': 'test'}, 'draft': True}}
        with self.app.test_client() as c:
            response = c.post(
                '/start-ci', environ_overrides=WSGI_ENVIRONMENT,
                data=json.dumps(data), headers=self.generate_header(data, 'pull_request'))

        self.assertEqual(response.data, b'{"msg": "EOL"}')

    @mock.patch('github.Github.get_repo')
    @mock.patch('requests.get', side_effect=mock_api_request_github)
    def test_webhook_pr_synchronize_draft(self, mock_request, mock_repo):
        """Test webhook triggered with pull_request event with synchronize action, marked as draft."""
        data = {'action': 'synchronize',
                'pull_request': {'number': 1234, 'head': {'sha': 'abcd1234'}, 'user': {'id': 'test'}, 'draft': True}}
        with self.app.test_client() as c:
            response = c.post(
                '/start-ci', environ_overrides=WSGI_ENVIRONMENT,
                data=json.dumps(data), headers=self.generate_header(data, 'pull_request'))

        self.assertEqual(response.data, b'{"msg": "EOL"}')

    @mock.patch('github.Github.get_repo')
    @mock.patch('mod_ci.controllers.schedule_test')
    @mock.patch('requests.get', side_effect=mock_api_request_github)
    def test_webhook_workflow_run_requested_valid_workflow_name(self, mock_requests, mock_schedule_test, mock_repo):
        """Test webhook triggered with workflow run event with action requested with a valid workflow name."""
        data = {'action': 'requested', 'workflow_run': {
            'name': Workflow_builds.LINUX, 'head_sha': 'abcd1234'}}
        with self.app.test_client() as c:
            response = c.post(
                '/start-ci', environ_overrides=WSGI_ENVIRONMENT,
                data=json.dumps(data), headers=self.generate_header(data, 'workflow_run'))
        self.assertEqual(response.data, b'{"msg": "EOL"}')
        mock_schedule_test.assert_called_once()

    @mock.patch('mod_ci.controllers.verify_artifacts_exist', return_value={'linux': True, 'windows': True})
    @mock.patch('github.Github.get_repo')
    @mock.patch('mod_ci.controllers.queue_test')
    @mock.patch('requests.get', side_effect=mock_api_request_github)
    def test_webhook_workflow_run_completed_successful_linux(self, mock_request, mock_queue_test,
                                                             mock_repo, mock_verify):
        """Test webhook triggered with workflow run event with action completed and status success on linux."""
        data = {'action': 'completed',
                'workflow_run': {'event': 'push',
                                 'name': Workflow_builds.LINUX, 'head_sha': '1',
                                 'head_branch': 'master'}, 'sender': {'login': 'test_owner'}}
        from github.Workflow import Workflow
        workflow = MagicMock(Workflow)
        workflow.id = 1
        workflow.name = Workflow_builds.LINUX
        mock_repo.return_value.get_workflows.return_value = [workflow]

        from github.WorkflowRun import WorkflowRun
        workflow_run1 = MagicMock(WorkflowRun)
        workflow_run1.head_sha = '1'
        workflow_run1.workflow_id = 1
        workflow_run1.status = 'completed'
        workflow_run1.conclusion = 'success'
        workflow_run1.name = Workflow_builds.LINUX

        workflow_run2 = MagicMock(WorkflowRun)
        workflow_run2.head_sha = '2'

        mock_repo.return_value.get_workflow_runs.return_value = [workflow_run2, workflow_run1]

        with self.app.test_client() as c:
            response = c.post(
                '/start-ci', environ_overrides=WSGI_ENVIRONMENT,
                data=json.dumps(data), headers=self.generate_header(data, 'workflow_run'))
            mock_queue_test.assert_called_once()

    @mock.patch('mod_ci.controllers.verify_artifacts_exist', return_value={'linux': True, 'windows': True})
    @mock.patch('github.Github.get_repo')
    @mock.patch('mod_ci.controllers.queue_test')
    @mock.patch('requests.get', side_effect=mock_api_request_github)
    def test_webhook_workflow_run_completed_successful_windows(self, mock_request, mock_queue_test,
                                                               mock_repo, mock_verify):
        """Test webhook triggered with workflow run event with action completed and status success on windows."""
        data = {'action': 'completed',
                'workflow_run': {'event': 'push',
                                 'name': Workflow_builds.WINDOWS, 'head_sha': '1',
                                 'head_branch': 'master'}, 'sender': {'login': 'test_owner'}}
        from github.Workflow import Workflow
        workflow = MagicMock(Workflow)
        workflow.id = 1
        workflow.name = Workflow_builds.WINDOWS
        mock_repo.return_value.get_workflows.return_value = [workflow]

        from github.WorkflowRun import WorkflowRun
        workflow_run1 = MagicMock(WorkflowRun)
        workflow_run1.head_sha = '1'
        workflow_run1.workflow_id = 1
        workflow_run1.status = 'completed'
        workflow_run1.conclusion = 'success'
        workflow_run1.name = Workflow_builds.WINDOWS
        mock_repo.return_value.get_workflow_runs.return_value = [workflow_run1]

        with self.app.test_client() as c:
            response = c.post(
                '/start-ci', environ_overrides=WSGI_ENVIRONMENT,
                data=json.dumps(data), headers=self.generate_header(data, 'workflow_run'))
            mock_queue_test.assert_called_once()

    @mock.patch('requests.get', side_effect=mock_api_request_github)
    @mock.patch('github.Github.get_repo')
    @mock.patch('mod_ci.controllers.deschedule_test')
    def test_webhook_workflow_run_completed_failure(self, mock_deschedule_test, mock_repo, mock_request):
        """Test webhook triggered with workflow run event with action completed and status failure."""
        data = {'action': 'completed',
                'workflow_run': {'event': 'push',
                                 'name': Workflow_builds.WINDOWS, 'head_sha': '1',
                                 'head_branch': 'master'}, 'sender': {'login': 'test_owner'}}

        from github.Workflow import Workflow
        workflow = MagicMock(Workflow)
        workflow.id = 1
        workflow.name = Workflow_builds.WINDOWS
        mock_repo.return_value.get_workflows.return_value = [workflow]

        from github.WorkflowRun import WorkflowRun
        workflow_run1 = MagicMock(WorkflowRun)
        workflow_run1.head_sha = '1'
        workflow_run1.workflow_id = 1
        workflow_run1.status = 'completed'
        workflow_run1.conclusion = 'failure'
        workflow_run1.name = Workflow_builds.WINDOWS
        mock_repo.return_value.get_workflow_runs.return_value = [workflow_run1]

        from github.PullRequest import PullRequest
        pr = MagicMock(PullRequest)
        pr.head.sha = '1'
        pr.user.id = 1
        pr.number = 1
        mock_repo.return_value.get_pulls.return_value = [pr]

        with self.app.test_client() as c:
            response = c.post(
                '/start-ci', environ_overrides=WSGI_ENVIRONMENT,
                data=json.dumps(data), headers=self.generate_header(data, 'workflow_run'))
        mock_deschedule_test.assert_called()

    @mock.patch('mod_ci.controllers.deschedule_test')
    @mock.patch('mod_ci.controllers.schedule_test')
    @mock.patch('github.Github.get_repo')
    @mock.patch('requests.get', side_effect=mock_api_request_github)
    def test_webhook_workflow_run_running(self, mock_request, mock_repo, mock_schedule_test, mock_deschedule_test):
        """Test webhook triggered with workflow run event where github actions is still in progress."""
        data = {'action': 'completed',
                'workflow_run': {'event': 'push',
                                 'name': Workflow_builds.WINDOWS, 'head_sha': '1',
                                 'head_branch': 'master'}, 'sender': {'login': 'test_owner'}}

        from github.Workflow import Workflow
        workflow = MagicMock(Workflow)
        workflow.id = 1
        workflow.name = Workflow_builds.WINDOWS
        mock_repo.return_value.get_workflows.return_value = [workflow]

        from github.WorkflowRun import WorkflowRun
        workflow_run1 = MagicMock(WorkflowRun)
        workflow_run1.head_sha = '1'
        workflow_run1.workflow_id = 1
        workflow_run1.status = 'not_completed_yet'
        workflow_run1.name = Workflow_builds.WINDOWS
        mock_repo.return_value.get_workflow_runs.return_value = [workflow_run1]

        with self.app.test_client() as c:
            response = c.post(
                '/start-ci', environ_overrides=WSGI_ENVIRONMENT,
                data=json.dumps(data), headers=self.generate_header(data, 'workflow_run'))

        mock_schedule_test.assert_not_called()
        mock_deschedule_test.assert_not_called()

    @mock.patch('github.Github.get_repo')
    @mock.patch('mod_ci.controllers.schedule_test')
    @mock.patch('requests.get', side_effect=mock_api_request_github)
    def test_webhook_workflow_run_requested_invalid_workflow_name(self, mock_request, mock_schedule_test, mock_repo):
        """Test webhook triggered with workflow run event with an invalid action."""
        data = {'action': 'requested', 'workflow_run': {
            'name': 'Invalid', 'head_sha': 'abcdef'}}
        with self.app.test_client() as c:
            response = c.post(
                '/start-ci', environ_overrides=WSGI_ENVIRONMENT,
                data=json.dumps(data), headers=self.generate_header(data, 'workflow_run'))
        mock_schedule_test.assert_not_called()
        self.assertEqual(response.data, b'{"msg": "EOL"}')

    @mock.patch('github.Github.get_repo')
    @mock.patch('mod_ci.controllers.schedule_test')
    @mock.patch('mod_ci.controllers.deschedule_test')
    @mock.patch('mod_ci.controllers.add_test_entry')
    @mock.patch('requests.get', side_effect=mock_api_request_github)
    def test_webhook_with_unrecognized_event(self, mock_requests, mock_schedule_test,
                                             mock_deschedule_test, mock_add_test_entry, mock_repo):
        """Test webhook with an unrecognised event triggered via GitHub Actions."""
        with self.app.test_client() as c:
            response = c.post(
                '/start-ci', environ_overrides=WSGI_ENVIRONMENT,
                data=json.dumps({}), headers=self.generate_header({}, 'workflow_job'))
        mock_schedule_test.assert_not_called()
        mock_deschedule_test.assert_not_called()
        mock_add_test_entry.assert_not_called()
        self.assertEqual(response.data, b'{"msg": "EOL"}')

    @mock.patch('flask.g.log.warning')
    @mock.patch('requests.get', side_effect=mock_api_request_github)
    def test_webhook_with_invalid_ci_signature(self, mock_github, mock_warning):
        """Test webhook if an invalid X-Hub-Signature is passed within headers."""
        with self.app.test_client() as c:
            response = c.post(
                '/start-ci', environ_overrides=WSGI_ENVIRONMENT,
                data=json.dumps({}), headers=self.generate_header({}, 'workflow_run', "1"))
        mock_warning.assert_called_once()

    @mock.patch('mod_ci.controllers.verify_artifacts_exist', return_value={'linux': True, 'windows': True})
    @mock.patch('github.Github.get_repo')
    @mock.patch('mod_ci.controllers.BlockedUsers')
    @mock.patch('mod_ci.controllers.queue_test')
    @mock.patch('requests.get', side_effect=mock_api_request_github)
    def test_webhook_workflow_run_completed_successful_pr_linux(self, mock_request, mock_queue_test,
                                                                mock_blocked, mock_repo, mock_verify):
        """Test webhook triggered - workflow run event, action completed, status success for pull request on linux."""
        data = {'action': 'completed',
                'workflow_run': {'event': 'pull_request',
                                 'name': Workflow_builds.LINUX, 'head_sha': '1',
                                 'head_branch': 'master'}, 'sender': {'login': 'test_owner'}}
        from github.Workflow import Workflow
        workflow = MagicMock(Workflow)
        workflow.id = 1
        workflow.name = Workflow_builds.LINUX
        mock_repo.return_value.get_workflows.return_value = [workflow]

        from github.WorkflowRun import WorkflowRun
        workflow_run1 = MagicMock(WorkflowRun)
        workflow_run1.head_sha = '1'
        workflow_run1.workflow_id = 1
        workflow_run1.status = 'completed'
        workflow_run1.conclusion = 'success'
        workflow_run1.name = Workflow_builds.LINUX
        mock_repo.return_value.get_workflow_runs.return_value = [workflow_run1]

        from github.PullRequest import PullRequest
        pr = MagicMock(PullRequest)
        pr.head.sha = '1'
        pr.user.id = 1
        pr.number = 1
        mock_repo.return_value.get_pulls.return_value = [pr]
        with self.app.test_client() as c:
            mock_blocked.query.filter.return_value.first.return_value = None
            response = c.post(
                '/start-ci', environ_overrides=WSGI_ENVIRONMENT,
                data=json.dumps(data), headers=self.generate_header(data, 'workflow_run'))
            mock_queue_test.assert_called_once()
            mock_queue_test.reset_mock()

            mock_blocked.query.filter.return_value.first.return_value = 1
            response = c.post(
                '/start-ci', environ_overrides=WSGI_ENVIRONMENT,
                data=json.dumps(data), headers=self.generate_header(data, 'workflow_run'))
            mock_queue_test.assert_not_called()
            self.assertEqual(response.data, b'ERROR')

    @mock.patch('mod_ci.controllers.verify_artifacts_exist', return_value={'linux': True, 'windows': True})
    @mock.patch('github.Github.get_repo')
    @mock.patch('mod_ci.controllers.BlockedUsers')
    @mock.patch('mod_ci.controllers.queue_test')
    @mock.patch('requests.get', side_effect=mock_api_request_github)
    def test_webhook_workflow_run_completed_successful_pr_windows(self, mock_request,
                                                                  mock_queue_test, mock_blocked, mock_repo,
                                                                  mock_verify):
        """Test webhook triggered - workflow run event, action completed, status success for pull request on windows."""
        data = {'action': 'completed',
                'workflow_run': {'event': 'pull_request',
                                 'name': Workflow_builds.WINDOWS, 'head_sha': '1',
                                 'head_branch': 'master'}, 'sender': {'login': 'test_owner'}}
        from github.Workflow import Workflow
        workflow = MagicMock(Workflow)
        workflow.id = 1
        workflow.name = Workflow_builds.WINDOWS
        mock_repo.return_value.get_workflows.return_value = [workflow]

        from github.WorkflowRun import WorkflowRun
        workflow_run1 = MagicMock(WorkflowRun)
        workflow_run1.head_sha = '1'
        workflow_run1.workflow_id = 1
        workflow_run1.status = 'completed'
        workflow_run1.conclusion = 'success'
        workflow_run1.name = Workflow_builds.WINDOWS
        mock_repo.return_value.get_workflow_runs.return_value = [workflow_run1]

        from github.PullRequest import PullRequest
        pr = MagicMock(PullRequest)
        pr.head.sha = '1'
        pr.user.id = 1
        pr.number = 1
        mock_repo.return_value.get_pulls.return_value = [pr]

        with self.app.test_client() as c:
            mock_blocked.query.filter.return_value.first.return_value = None
            response = c.post(
                '/start-ci', environ_overrides=WSGI_ENVIRONMENT,
                data=json.dumps(data), headers=self.generate_header(data, 'workflow_run'))
            mock_queue_test.assert_called_once()
            mock_queue_test.reset_mock()

            mock_blocked.query.filter.return_value.first.return_value = 1
            response = c.post(
                '/start-ci', environ_overrides=WSGI_ENVIRONMENT,
                data=json.dumps(data), headers=self.generate_header(data, 'workflow_run'))
            mock_queue_test.assert_not_called()
            self.assertEqual(response.data, b'ERROR')

    @mock.patch('mod_ci.controllers.verify_artifacts_exist', return_value={'linux': True, 'windows': True})
    @mock.patch('github.Github.get_repo')
    @mock.patch('mod_ci.controllers.deschedule_test')
    @mock.patch('mod_ci.controllers.BlockedUsers')
    @mock.patch('mod_ci.controllers.queue_test')
    @mock.patch('requests.get', side_effect=mock_api_request_github)
    def test_webhook_workflow_run_completed_successful_pr_updated(self, mock_request, mock_queue_test,
                                                                  mock_blocked, mock_deschedule_test, mock_repo,
                                                                  mock_verify):
        """Test webhook triggered - workflow run event, action completed, for a pull request whose head was updated."""
        data = {'action': 'completed',
                'workflow_run': {'event': 'pull_request',
                                 'name': Workflow_builds.WINDOWS, 'head_sha': '1',
                                 'head_branch': 'master'}, 'sender': {'login': 'test_owner'}}

        from github.Workflow import Workflow
        workflow = MagicMock(Workflow)
        workflow.id = 1
        workflow.name = Workflow_builds.WINDOWS
        mock_repo.return_value.get_workflows.return_value = [workflow]

        from github.WorkflowRun import WorkflowRun
        workflow_run1 = MagicMock(WorkflowRun)
        workflow_run1.head_sha = '1'
        workflow_run1.workflow_id = 1
        workflow_run1.status = 'completed'
        workflow_run1.conclusion = 'success'
        workflow_run1.name = Workflow_builds.WINDOWS
        mock_repo.return_value.get_workflow_runs.return_value = [workflow_run1]

        from github.PullRequest import PullRequest
        pr = MagicMock(PullRequest)
        pr.head.sha = '2'
        pr.user.id = 1
        pr.number = 1
        mock_repo.return_value.get_pulls.return_value = [pr]

        with self.app.test_client() as c:
            mock_blocked.query.filter.return_value.first.return_value = None
            response = c.post(
                '/start-ci', environ_overrides=WSGI_ENVIRONMENT,
                data=json.dumps(data), headers=self.generate_header(data, 'workflow_run'))
            mock_queue_test.assert_not_called()
            mock_deschedule_test.assert_called()

    @mock.patch('utility.requests.get')
    def test_start_ci_with_a_get_request(self, mock_requests_get):
        """Test start_ci function with a request method other than post."""
        # Mock GitHub meta API response with webhook IP ranges using localhost
        mock_response = MagicMock()
        mock_response.json.return_value = {'hooks': ['127.0.0.0/8']}
        mock_requests_get.return_value = mock_response

        with self.app.test_client() as c:
            response = c.get(
                '/start-ci', environ_overrides={'REMOTE_ADDR': '127.0.0.1'},
                headers=self.generate_header({}, 'test'))
            self.assertEqual(response.data, b'OK')

    @mock.patch('github.Github')
    @mock.patch('run.log.debug')
    def test_queue_test_with_pull_request(self, mock_debug, mock_github):
        """Check queue_test function with pull request as test type."""
        from mod_ci.controllers import add_test_entry, queue_test
        repository = mock_github(g.github['bot_token']).get_repo(
            f"{g.github['repository_owner']}/{g.github['repository']}")
        # Use valid hex commit hash
        test_commit = 'def5678abc1234def5678abc1234def567890abc'
        add_test_entry(g.db, test_commit, TestType.pull_request)
        mock_debug.assert_called_once_with('pull request test type detected')
        queue_test(repository.get_commit("1"), test_commit, TestType.pull_request, TestPlatform.linux)
        mock_debug.assert_called_with('Created tests, waiting for cron...')

    @mock.patch('run.log.critical')
    @mock.patch('github.Github.get_repo')
    @mock.patch('run.log.debug')
    def test_queue_test_github_failure(self, mock_debug, mock_repo, mock_critical):
        """Check queue_test function when updating GitHub status fails."""
        from github import GithubException

        from mod_ci.controllers import add_test_entry, queue_test

        # Use valid hex commit hash
        test_commit = '1234567890abcdef1234567890abcdef12345678'
        add_test_entry(g.db, test_commit, TestType.pull_request)
        mock_debug.assert_called_once_with('pull request test type detected')
        mock_commit = mock_repo.get_commit("1")
        response_data = "mock 400 response"
        mock_commit.create_status = mock.MagicMock(
            side_effect=GithubException(status=400, data=response_data, headers={}))
        queue_test(mock_commit, test_commit, TestType.pull_request, TestPlatform.linux)
        mock_debug.assert_called_with('Created tests, waiting for cron...')
        mock_critical.assert_called_with(f"Could not post to GitHub! Response: {response_data}")

    @mock.patch('run.log')
    @mock.patch('github.Github')
    def test_schedule_test_function(self, git_mock, mock_log):
        """Check the functioning of schedule_test function."""
        from mod_ci.controllers import schedule_test
        repository = git_mock(g.github['bot_token']).get_repo(
            f"{g.github['repository_owner']}/{g.github['repository']}")
        schedule_test(repository.get_commit(1))
        mock_log.debug.assert_not_called()
        mock_log.critical.assert_not_called()

        mock_log.reset_mock()
        schedule_test(None)
        mock_log.debug.assert_not_called()
        mock_log.critical.assert_not_called()

    @mock.patch('run.log')
    @mock.patch('github.Github')
    def test_deschedule_test_function(self, git_mock, mock_log):
        """Check the functioning of deschedule_test function on linux platform."""
        from mod_ci.controllers import deschedule_test
        repository = git_mock(g.github['bot_token']).get_repo(
            f"{g.github['repository_owner']}/{g.github['repository']}")
        commit = Test.query.filter(Test.platform == TestPlatform.linux).first().commit
        deschedule_test(repository.get_commit(commit), commit, TestType.pull_request, TestPlatform.linux)
        mock_log.debug.assert_called_once_with("pull request test type detected")
        mock_log.critical.assert_not_called()

        # Test deschedule function with github commit None
        mock_log.reset_mock()
        deschedule_test(None, 1, TestType.commit, TestPlatform.linux)
        mock_log.debug.assert_not_called()
        mock_log.critical.assert_not_called()

    @mock.patch('run.log')
    @mock.patch('mod_ci.controllers.safe_db_commit')
    @mock.patch('github.Github')
    def test_deschedule_test_db_commit_failure(
            self, git_mock, mock_safe_commit, mock_log):
        """Check deschedule_test handles db commit failure gracefully."""
        from mod_ci.controllers import deschedule_test

        mock_safe_commit.return_value = False
        repository = git_mock(g.github['bot_token']).get_repo(
            f"{g.github['repository_owner']}/{g.github['repository']}")
        test = Test.query.filter(Test.platform == TestPlatform.linux).first()
        commit = test.commit
        deschedule_test(
            repository.get_commit(commit), commit,
            TestType.pull_request, TestPlatform.linux)

        mock_safe_commit.assert_called_once()
        mock_log.error.assert_called()

    @mock.patch('github.Github.get_repo')
    @mock.patch('requests.get', side_effect=mock_api_request_github)
    @mock.patch('mod_ci.controllers.retry_with_backoff')
    @mock.patch('run.log')
    def test_webhook_push_github_api_failure(
            self, mock_log, mock_retry, mock_request, mock_repo):
        """Test push webhook handles GitHub API retry failure."""
        from github import GithubException

        mock_retry.side_effect = GithubException(500, "API Error", None)

        data = {'after': 'abcdefgh', 'ref': 'refs/heads/master'}
        with self.app.test_client() as c:
            response = c.post(
                '/start-ci', environ_overrides=WSGI_ENVIRONMENT,
                data=json.dumps(data),
                headers=self.generate_header(data, 'push'))

        self.assertIn(b'ERROR', response.data)
        mock_log.error.assert_called()

    @mock.patch('github.Github.get_repo')
    @mock.patch('requests.get', side_effect=mock_api_request_github)
    @mock.patch('mod_ci.controllers.safe_db_commit')
    @mock.patch('mod_ci.controllers.retry_with_backoff')
    @mock.patch('run.log')
    def test_webhook_push_db_commit_failure(
            self, mock_log, mock_retry, mock_safe_commit,
            mock_request, mock_repo):
        """Test push webhook handles db commit failure."""
        mock_ref = MagicMock()
        mock_ref.object.sha = 'newcommithash'
        mock_retry.return_value = mock_ref
        mock_safe_commit.return_value = False

        data = {'after': 'abcdefgh', 'ref': 'refs/heads/master'}
        with self.app.test_client() as c:
            response = c.post(
                '/start-ci', environ_overrides=WSGI_ENVIRONMENT,
                data=json.dumps(data),
                headers=self.generate_header(data, 'push'))

        self.assertIn(b'ERROR', response.data)

    @mock.patch('github.Github.get_repo')
    @mock.patch('requests.get', side_effect=mock_api_request_github)
    @mock.patch('mod_ci.controllers.retry_with_backoff')
    @mock.patch('run.log')
    def test_webhook_pr_opened_github_api_failure(
            self, mock_log, mock_retry, mock_request, mock_repo):
        """Test PR opened webhook handles GitHub API retry failure."""
        from github import GithubException

        mock_retry.side_effect = GithubException(500, "API Error", None)

        data = {
            'action': 'opened',
            'pull_request': {
                'number': 1234,
                'head': {'sha': 'abcdef123456'},
                'user': {'id': 99999}
            }
        }
        with self.app.test_client() as c:
            response = c.post(
                '/start-ci', environ_overrides=WSGI_ENVIRONMENT,
                data=json.dumps(data),
                headers=self.generate_header(data, 'pull_request'))

        mock_log.error.assert_called()

    @mock.patch('github.Github.get_repo')
    @mock.patch('requests.get', side_effect=mock_api_request_github)
    @mock.patch('mod_ci.controllers.safe_db_commit')
    @mock.patch('run.log')
    def test_webhook_release_delete_db_failure(
            self, mock_log, mock_safe_commit, mock_request, mock_repo):
        """Test release delete webhook handles db commit failure."""
        mock_safe_commit.return_value = False

        # First add a release to delete
        release = CCExtractorVersion(
            '2.1', '2018-05-30T20:18:44Z', 'abcdefgh')
        g.db.add(release)
        g.db.commit()

        data = {
            'action': 'deleted',
            'release': {
                'prerelease': False,
                'published_at': '2018-05-30T20:18:44Z',
                'tag_name': 'v2.1'
            }
        }
        with self.app.test_client() as c:
            response = c.post(
                '/start-ci', environ_overrides=WSGI_ENVIRONMENT,
                data=json.dumps(data),
                headers=self.generate_header(data, 'release'))

        mock_log.error.assert_called()

    @mock.patch('github.Github.get_repo')
    @mock.patch('requests.get', side_effect=mock_api_request_github)
    @mock.patch('mod_ci.controllers.safe_db_commit')
    @mock.patch('run.log')
    def test_webhook_release_published_db_failure(
            self, mock_log, mock_safe_commit, mock_request, mock_repo):
        """Test release published webhook handles db commit failure."""
        mock_safe_commit.return_value = False

        last_commit = GeneralData.query.filter(
            GeneralData.key == 'last_commit').first()
        last_commit.value = 'abcdefgh'
        g.db.commit()

        data = {
            'action': 'published',
            'release': {
                'prerelease': False,
                'published_at': '2018-05-30T20:18:44Z',
                'tag_name': 'v2.2'
            }
        }
        with self.app.test_client() as c:
            response = c.post(
                '/start-ci', environ_overrides=WSGI_ENVIRONMENT,
                data=json.dumps(data),
                headers=self.generate_header(data, 'release'))

        mock_log.error.assert_called()

    @mock.patch('github.Github.get_repo')
    @mock.patch('requests.get', side_effect=mock_api_request_github)
    @mock.patch('run.log')
    def test_webhook_release_unsupported_action(
            self, mock_log, mock_request, mock_repo):
        """Test release webhook handles unsupported action."""
        data = {
            'action': 'unknown_action',
            'release': {
                'prerelease': False,
                'published_at': '2018-05-30T20:18:44Z',
                'tag_name': 'v2.1'
            }
        }
        with self.app.test_client() as c:
            response = c.post(
                '/start-ci', environ_overrides=WSGI_ENVIRONMENT,
                data=json.dumps(data),
                headers=self.generate_header(data, 'release'))

        mock_log.warning.assert_called()

    @mock.patch('mod_ci.controllers.safe_db_commit')
    @mock.patch('run.log')
    def test_add_test_entry_db_commit_failure(
            self, mock_log, mock_safe_commit):
        """Test add_test_entry handles db commit failure."""
        from mod_ci.controllers import add_test_entry

        mock_safe_commit.return_value = False

        # Use valid hex commit hash (is_valid_commit_hash validates before db commit)
        add_test_entry(g.db, 'abcdef1234567890abcdef1234567890abcdef12', TestType.commit)

        mock_safe_commit.assert_called_once()
        mock_log.error.assert_called()

    @mock.patch('mod_ci.controllers.delete_instance')
    @mock.patch('mod_ci.controllers.safe_db_commit')
    @mock.patch('mod_ci.controllers.is_instance_testing')
    @mock.patch('run.log')
    def test_delete_expired_instances_db_commit_failure(
            self, mock_log, mock_is_testing, mock_safe_commit,
            mock_delete):
        """Test delete_expired_instances handles db commit failure."""
        from mod_ci.controllers import delete_expired_instances

        mock_is_testing.return_value = True
        mock_safe_commit.return_value = False

        # Create a mock compute service
        mock_compute = MagicMock()
        mock_compute.instances().list().execute.return_value = {
            'items': [{
                'name': 'linux-1',
                'creationTimestamp': '2020-01-01T00:00:00.000+00:00'
            }]
        }

        mock_repo = MagicMock()

        delete_expired_instances(
            mock_compute, 60, 'project', 'zone', g.db, mock_repo)

        # Should continue to next instance after commit failure
        mock_delete.assert_not_called()

    @mock.patch('mod_ci.controllers.delete_instance_with_tracking')
    @mock.patch('mod_ci.controllers.update_status_on_github')
    @mock.patch('mod_ci.controllers.safe_db_commit')
    @mock.patch('mod_ci.controllers.is_instance_testing')
    @mock.patch('run.log')
    def test_delete_expired_instances_includes_target_url(
            self, mock_log, mock_is_testing, mock_safe_commit,
            mock_update_github, mock_delete_tracking):
        """Test delete_expired_instances includes target_url in GitHub status."""
        from mod_ci.controllers import delete_expired_instances

        mock_is_testing.return_value = True
        mock_safe_commit.return_value = True
        mock_delete_tracking.return_value = {'name': 'op-123'}

        # Create a mock compute service with expired instance
        mock_compute = MagicMock()
        mock_compute.instances().list().execute.return_value = {
            'items': [{
                'name': 'linux-1',
                'creationTimestamp': '2020-01-01T00:00:00.000+00:00'
            }]
        }

        mock_repo = MagicMock()
        mock_gh_commit = MagicMock()
        mock_repo.get_commit.return_value = mock_gh_commit

        delete_expired_instances(
            mock_compute, 60, 'project', 'zone', g.db, mock_repo)

        # Verify update_status_on_github was called with target_url
        mock_update_github.assert_called_once()
        call_args = mock_update_github.call_args
        # Check that target_url (5th argument) contains the test ID
        self.assertEqual(len(call_args[0]), 5)  # 5 positional args
        target_url = call_args[0][4]
        self.assertIn('/test/1', target_url)

    @mock.patch('github.Github.get_repo')
    @mock.patch('requests.get', side_effect=mock_api_request_github)
    @mock.patch('mod_ci.controllers.inform_mailing_list')
    @mock.patch('mod_sample.models.Issue.query')
    def test_webhook_issue_opened(self, mock_issue, mock_mailing, mock_request, mock_github):
        """Test webhook triggered with issues event with opened action."""
        from mod_sample.models import Issue
        data = {'action': 'opened',
                'issue': {'number': '1234', 'title': 'testTitle', 'body': 'testing', 'state': 'opened',
                          'user': {'login': 'testAuthor'}}}
        with self.app.test_client() as c:
            response = c.post(
                '/start-ci', environ_overrides=WSGI_ENVIRONMENT,
                data=json.dumps(data), headers=self.generate_header(data, 'issues'))

        self.assertEqual(response.data, b'{"msg": "EOL"}')
        mock_mailing.assert_called_once_with(mock.ANY, '1234', 'testTitle', 'testAuthor', 'testing')

    @mock.patch('github.Github')
    @mock.patch('run.log.critical')
    def test_github_api_error(self, mock_critical, mock_github):
        """Test functions with GitHub API error."""
        from github import Github, GithubException

        from mod_ci.controllers import deschedule_test, schedule_test
        github_status = Github('1').get_repo(
            f"{g.github['repository_owner']}/{g.github['repository']}").get_commit('abcdef')
        github_status.create_status.side_effect = GithubException(status=500, data="", headers={})
        schedule_test(github_status)
        mock_critical.assert_called()
        mock_critical.reset_mock()
        test = Test.query.first()
        deschedule_test(github_status, test=test, db=g.db)
        mock_critical.assert_called()

    @mock.patch('mod_ci.controllers.get_test_results')
    @mock.patch('mod_ci.controllers.is_main_repo')
    @mock.patch('mod_ci.controllers.shutil')
    def test_update_build_badge(self, mock_shutil, mock_check_repo, mock_get_test_results):
        """Test update_build_badge function."""
        from mod_ci.controllers import update_build_badge

        update_build_badge('pass', MockTest())

        mock_check_repo.assert_called_once_with(None)
        mock_get_test_results.assert_called_once()
        mock_shutil.copyfile.assert_called_once_with(mock.ANY, mock.ANY)

    @mock.patch('mod_ci.controllers.request')
    @mock.patch('mod_ci.controllers.Test')
    def test_progress_reporter_no_test(self, mock_test, mock_request):
        """Test progress_reporter with no test found."""
        from mod_ci.controllers import progress_reporter

        mock_test.query.filter.return_value.first.return_value = None

        expected_ret = "FAIL"

        ret_val = progress_reporter(1, "token")

        self.assertEqual(expected_ret, ret_val)
        mock_test.query.filter.assert_called_once()
        mock_request.assert_not_called()

    @mock.patch('mod_ci.controllers.request')
    @mock.patch('mod_ci.controllers.Test')
    @mock.patch('mod_ci.controllers.progress_type_request')
    def test_progress_reporter_progress_type_fail(self, mock_progress_type, mock_test, mock_request):
        """Test progress_reporter with failing of request type progress."""
        from mod_ci.controllers import progress_reporter

        mock_test_obj = MagicMock()
        mock_test_obj.token = "token"
        mock_test.query.filter.return_value.first.return_value = mock_test_obj
        mock_request.form = {'type': 'progress'}
        mock_progress_type.return_value = False

        expected_ret = "FAIL"

        ret_val = progress_reporter(1, "token")

        self.assertEqual(expected_ret, ret_val)
        mock_test.query.filter.assert_called_once()
        mock_request.assert_not_called()
        mock_progress_type.assert_called_once_with(mock.ANY, mock.ANY, 1, mock.ANY)

    @mock.patch('mod_ci.controllers.request')
    @mock.patch('mod_ci.controllers.Test')
    @mock.patch('mod_ci.controllers.progress_type_request')
    def test_progress_reporter_progress_type(self, mock_progress_type, mock_test, mock_request):
        """Test progress_reporter with request type progress."""
        from mod_ci.controllers import progress_reporter

        mock_test_obj = MagicMock()
        mock_test_obj.token = "token"
        mock_test.query.filter.return_value.first.return_value = mock_test_obj
        mock_request.form = {'type': 'progress'}
        mock_progress_type.return_value = "OK"

        expected_ret = "OK"

        ret_val = progress_reporter(1, "token")

        self.assertEqual(expected_ret, ret_val)
        mock_test.query.filter.assert_called_once()
        mock_request.assert_not_called()
        mock_progress_type.assert_called_once_with(mock.ANY, mock.ANY, 1, mock.ANY)

    @mock.patch('mod_ci.controllers.request')
    @mock.patch('mod_ci.controllers.Test')
    @mock.patch('mod_ci.controllers.equality_type_request')
    def test_progress_reporter_equality_type(self, mock_equality_type, mock_test, mock_request):
        """Test progress_reporter with request type equality."""
        from mod_ci.controllers import progress_reporter

        mock_test_obj = MagicMock()
        mock_test_obj.token = "token"
        mock_test.query.filter.return_value.first.return_value = mock_test_obj
        mock_request.form = {'type': 'equality'}
        mock_equality_type.return_value = "OK"

        expected_ret = "OK"

        ret_val = progress_reporter(1, "token")

        self.assertEqual(expected_ret, ret_val)
        mock_test.query.filter.assert_called_once()
        mock_request.assert_not_called()
        mock_equality_type.assert_called_once_with(mock.ANY, 1, mock.ANY, mock.ANY)

    @mock.patch('mod_ci.controllers.request')
    @mock.patch('mod_ci.controllers.Test')
    @mock.patch('mod_ci.controllers.upload_log_type_request')
    def test_progress_reporter_logupload_type_empty(self, mock_logupload_type, mock_test, mock_request):
        """Test progress_reporter with request type logupload returning 'EMPTY'."""
        from mod_ci.controllers import progress_reporter

        mock_test_obj = MagicMock()
        mock_test_obj.token = "token"
        mock_test.query.filter.return_value.first.return_value = mock_test_obj
        mock_request.form = {'type': 'logupload'}
        mock_logupload_type.return_value = False

        expected_ret = "EMPTY"

        ret_val = progress_reporter(1, "token")

        self.assertEqual(expected_ret, ret_val)
        mock_test.query.filter.assert_called_once()
        mock_request.assert_not_called()
        mock_logupload_type.assert_called_once_with(mock.ANY, 1, mock.ANY, mock.ANY, mock.ANY)

    @mock.patch('mod_ci.controllers.request')
    @mock.patch('mod_ci.controllers.Test')
    @mock.patch('mod_ci.controllers.upload_log_type_request')
    def test_progress_reporter_logupload_type(self, mock_logupload_type, mock_test, mock_request):
        """Test progress_reporter with request type logupload."""
        from mod_ci.controllers import progress_reporter

        mock_test_obj = MagicMock()
        mock_test_obj.token = "token"
        mock_test.query.filter.return_value.first.return_value = mock_test_obj
        mock_request.form = {'type': 'logupload'}
        mock_logupload_type.return_value = "OK"

        expected_ret = "OK"

        ret_val = progress_reporter(1, "token")

        self.assertEqual(expected_ret, ret_val)
        mock_test.query.filter.assert_called_once()
        mock_request.assert_not_called()
        mock_logupload_type.assert_called_once_with(mock.ANY, 1, mock.ANY, mock.ANY, mock.ANY)

    @mock.patch('mod_ci.controllers.request')
    @mock.patch('mod_ci.controllers.Test')
    @mock.patch('mod_ci.controllers.upload_type_request')
    def test_progress_reporter_upload_type_empty(self, mock_upload_type, mock_test, mock_request):
        """Test progress_reporter with request type upload with returning 'EMPTY'."""
        from mod_ci.controllers import progress_reporter

        mock_test_obj = MagicMock()
        mock_test_obj.token = "token"
        mock_test.query.filter.return_value.first.return_value = mock_test_obj
        mock_request.form = {'type': 'upload'}
        mock_upload_type.return_value = False

        expected_ret = "EMPTY"

        ret_val = progress_reporter(1, "token")

        self.assertEqual(expected_ret, ret_val)
        mock_test.query.filter.assert_called_once()
        mock_request.assert_not_called()
        mock_upload_type.assert_called_once_with(mock.ANY, 1, mock.ANY, mock.ANY, mock.ANY)

    @mock.patch('mod_ci.controllers.request')
    @mock.patch('mod_ci.controllers.Test')
    @mock.patch('mod_ci.controllers.upload_type_request')
    def test_progress_reporter_upload_type(self, mock_upload_type, mock_test, mock_request):
        """Test progress_reporter with request type upload."""
        from mod_ci.controllers import progress_reporter

        mock_test_obj = MagicMock()
        mock_test_obj.token = "token"
        mock_test.query.filter.return_value.first.return_value = mock_test_obj
        mock_request.form = {'type': 'upload'}
        mock_upload_type.return_value = "OK"

        expected_ret = "OK"

        ret_val = progress_reporter(1, "token")

        self.assertEqual(expected_ret, ret_val)
        mock_test.query.filter.assert_called_once()
        mock_request.assert_not_called()
        mock_upload_type.assert_called_once_with(mock.ANY, 1, mock.ANY, mock.ANY, mock.ANY)

    @mock.patch('mod_ci.controllers.request')
    @mock.patch('mod_ci.controllers.Test')
    @mock.patch('mod_ci.controllers.finish_type_request')
    def test_progress_reporter_finish_type(self, mock_finish_type, mock_test, mock_request):
        """Test progress_reporter with request type finish."""
        from mod_ci.controllers import progress_reporter

        mock_test_obj = MagicMock()
        mock_test_obj.token = "token"
        mock_test.query.filter.return_value.first.return_value = mock_test_obj
        mock_request.form = {'type': 'finish'}
        mock_finish_type.return_value = "OK"

        expected_ret = "OK"

        ret_val = progress_reporter(1, "token")

        self.assertEqual(expected_ret, ret_val)
        mock_test.query.filter.assert_called_once()
        mock_request.assert_not_called()
        mock_finish_type.assert_called_once_with(mock.ANY, 1, mock.ANY, mock.ANY)

    @mock.patch('mod_ci.controllers.RegressionTestOutput')
    def test_equality_type_request_rto_none(self, mock_rto):
        """Test function equality_type_request when rto is None."""
        from mod_ci.controllers import equality_type_request

        mock_request = MagicMock()
        mock_request.form = {
            'test_id': 1,
            'test_file_id': 1
        }
        mock_rto.query.filter.return_value.first.return_value = None
        mock_log = MagicMock()

        equality_type_request(mock_log, 1, MagicMock(), mock_request)

        mock_log.debug.assert_called_once()
        mock_rto.query.filter.assert_called_once_with(mock_rto.id == 1)
        mock_log.info.assert_called_once()

    @mock.patch('mod_ci.controllers.delete_instance')
    @mock.patch('mod_ci.controllers.get_compute_service_object')
    @mock.patch('mod_ci.controllers.update_build_badge')
    @mock.patch('github.Github.get_repo')
    def test_progress_type_request(self, mock_repo, mock_update_build_badge, mock_get_compute_service_object,
                                   mock_delete_instance):
        """Test progress_type_request function."""
        from mod_ci.models import GcpInstance
        from run import log

        self.create_user_with_role(
            self.user.name, self.user.email, self.user.password, Role.tester)
        self.create_forktest("own-fork-commit", TestPlatform.linux, regression_tests=[2])
        request = MagicMock()
        request.form = {'status': 'completed', 'message': 'Ran all tests'}
        gcp_instance = GcpInstance(name='test_instance', test_id=3)
        g.db.add(gcp_instance)
        g.db.commit()

        test = Test.query.filter(Test.id == 3).first()

        response = progress_type_request(log, test, test.id, request)
        mock_update_build_badge.assert_called_once()
        mock_get_compute_service_object.assert_called()
        self.assertTrue(response)

    @mock.patch('mod_ci.controllers.g')
    @mock.patch('mod_ci.controllers.TestResultFile')
    @mock.patch('mod_ci.controllers.RegressionTestOutput')
    def test_equality_type_request_rto_exists(self, mock_rto, mock_result_file, mock_g):
        """Test function equality_type_request when rto exists."""
        from mod_ci.controllers import equality_type_request

        mock_request = MagicMock()
        mock_request.form = {
            'test_id': 1,
            'test_file_id': 1
        }
        mock_log = MagicMock()

        equality_type_request(mock_log, 1, MagicMock(), mock_request)

        mock_log.debug.assert_called_once()
        mock_rto.query.filter.assert_called_once_with(mock_rto.id == 1)
        mock_log.info.assert_not_called()
        mock_result_file.assert_called_once_with(mock.ANY, 1, mock.ANY, mock.ANY)
        mock_g.db.add.assert_called_once()
        mock_g.db.commit.assert_called_once()

    @mock.patch('mod_ci.controllers.secure_filename')
    def test_logupload_type_request_empty(self, mock_filename):
        """Test function logupload_type_request when filename is empty."""
        from mod_ci.controllers import upload_log_type_request

        mock_log = MagicMock()
        mock_request = MagicMock()
        mock_request.files = {'file': MagicMock()}
        mock_filename.return_value = ''

        self.assertFalse(upload_log_type_request(mock_log, 1, MagicMock(), MagicMock(), mock_request))

        mock_log.debug.assert_called_once()
        mock_filename.assert_called_once()

    @mock.patch('mod_ci.controllers.os')
    @mock.patch('mod_ci.controllers.secure_filename')
    def test_logupload_type_request(self, mock_filename, mock_os):
        """Test function logupload_type_request."""
        from mod_ci.controllers import upload_log_type_request

        mock_request = MagicMock()
        mock_log = MagicMock()
        mock_uploadfile = MagicMock()
        mock_request.files = {'file': mock_uploadfile}

        upload_log_type_request(mock_log, 1, MagicMock(), MagicMock(), mock_request)

        self.assertEqual(2, mock_log.debug.call_count)
        mock_filename.assert_called_once()
        self.assertEqual(2, mock_os.path.join.call_count)
        mock_uploadfile.save.assert_called_once()
        mock_os.rename.assert_called_once()

    @mock.patch('mod_ci.controllers.secure_filename')
    def test_upload_type_request_empty(self, mock_filename):
        """Test function upload_type_request when filename is empty."""
        from mod_ci.controllers import upload_type_request

        mock_request = MagicMock()
        mock_log = MagicMock()
        mock_request.files = {
            'file': MagicMock(),
            'test_id': 1,
            'test_file_id': 1
        }
        mock_filename.return_value = ''

        self.assertFalse(upload_type_request(mock_log, 1, MagicMock(), MagicMock(), mock_request))
        mock_log.debug.assert_called_once()
        mock_filename.assert_called_once()

    @mock.patch('mod_ci.controllers.hashlib')
    @mock.patch('mod_ci.controllers.TestResultFile')
    @mock.patch('mod_ci.controllers.RegressionTestOutput')
    @mock.patch('mod_ci.controllers.g')
    @mock.patch('mod_ci.controllers.iter')
    @mock.patch('mod_ci.controllers.open')
    @mock.patch('mod_ci.controllers.os')
    @mock.patch('mod_ci.controllers.secure_filename')
    def test_upload_type_request(self, mock_filename, mock_os, mock_open, mock_iter,
                                 mock_g, mock_rto, mock_result_file, mock_hashlib):
        """Test function upload_type_request."""
        from mod_ci.controllers import upload_type_request

        mock_upload_file = MagicMock()
        mock_log = MagicMock()
        mock_request = MagicMock()
        mock_request.files = {
            'file': mock_upload_file
        }
        mock_request.form = {
            'test_id': 1,
            'test_file_id': 1
        }
        mock_iter.return_value = ['chunk']
        mock_os.path.splitext.return_value = "a", "b"

        upload_type_request(mock_log, 1, MagicMock(), MagicMock(), mock_request)

        mock_log.debug.assert_called_once()
        mock_filename.assert_called_once()
        # 4 calls: temp_dir, temp_path, results_dir, final_path
        self.assertEqual(4, mock_os.path.join.call_count)
        # 2 calls: makedirs for TempFiles and TestResults directories
        self.assertEqual(2, mock_os.makedirs.call_count)
        mock_upload_file.save.assert_called_once()
        mock_open.assert_called_once_with(mock.ANY, "rb")
        mock_os.path.splitext.assert_called_once_with(mock.ANY)
        mock_os.rename.assert_called_once_with(mock.ANY, mock.ANY)
        mock_rto.query.filter.assert_called_once_with(mock_rto.id == 1)
        mock_result_file.assert_called_once_with(mock.ANY, 1, mock.ANY, mock.ANY, mock.ANY)
        mock_g.db.add.assert_called_once_with(mock.ANY)
        mock_g.db.commit.assert_called_once_with()
        mock_hashlib.sha256.assert_called_once_with()
        mock_iter.assert_called_once_with(mock.ANY, b"")

    @mock.patch('mod_ci.controllers.RegressionTest')
    @mock.patch('mod_ci.controllers.TestResult')
    @mock.patch('mod_ci.controllers.g')
    def test_finish_type_request(self, mock_g, mock_result, mock_rt):
        """Test function finish_type_request without exception occurring."""
        from mod_ci.controllers import finish_type_request

        mock_log = MagicMock()
        mock_request = MagicMock()
        mock_request.form = {
            'test_id': 1,
            'runTime': 1,
            'exitCode': 0
        }

        # Configure mock regression test to avoid AsyncMock in Python 3.13+
        create_mock_regression_test(mock_rt)

        finish_type_request(mock_log, 1, MagicMock(), mock_request)

        mock_log.debug.assert_called_once()
        mock_rt.query.filter.assert_called_once_with(mock_rt.id == 1)
        mock_result.assert_called_once_with(mock.ANY, mock.ANY, 1, 0, mock.ANY)
        mock_g.db.add.assert_called_once_with(mock.ANY)
        mock_g.db.commit.assert_called_once_with()

    @mock.patch('mod_ci.controllers.RegressionTest')
    @mock.patch('mod_ci.controllers.TestResult')
    @mock.patch('mod_ci.controllers.g')
    def test_finish_type_request_with_error(self, mock_g, mock_result, mock_rt):
        """Test function finish_type_request with error in database commit."""
        from pymysql.err import IntegrityError

        from mod_ci.controllers import finish_type_request

        mock_log = MagicMock()
        mock_request = MagicMock()
        mock_request.form = {
            'test_id': 1,
            'runTime': 1,
            'exitCode': 0
        }

        # Configure mock regression test to avoid AsyncMock in Python 3.13+
        create_mock_regression_test(mock_rt)

        # Set up mock db with IntegrityError on commit
        create_mock_db_query(mock_g)
        mock_g.db.commit.side_effect = IntegrityError("test error", "test")

        finish_type_request(mock_log, 1, MagicMock(), mock_request)

        mock_log.debug.assert_called_once()
        mock_rt.query.filter.assert_called_once_with(mock_rt.id == 1)
        mock_result.assert_called_once_with(mock.ANY, mock.ANY, 1, 0, mock.ANY)
        mock_g.db.add.assert_called_once_with(mock.ANY)
        mock_g.db.commit.assert_called_once_with()
        mock_log.error.assert_called_once()

    def test_in_maintenance_mode_ValueError(self):
        """Test in_maintenance_mode function with invalid platform."""
        with self.app.test_client() as c:
            response = c.post(
                '/maintenance/invalid')

        self.assertIsNotNone(response.data, b'ERROR')

    def test_in_maintenance_mode_linux(self):
        """Test in_maintenance_mode function with linux platform."""
        with self.app.test_client() as c:
            response = c.post(
                '/maintenance/linux')

        self.assertIsNotNone(response.data)

    def test_in_maintenance_mode_windows(self):
        """Test in_maintenance_mode function with windows platform."""
        with self.app.test_client() as c:
            response = c.post(
                '/maintenance/windows')

        self.assertIsNotNone(response.data)

    @mock.patch('run.log')
    @mock.patch('run.config')
    def test_start_platforms_with_empty_zone(self, mock_config, mock_log):
        """Test start_platforms function if GCP zone is not provided in config."""
        def config_get(key, *args, **kwargs):
            if key == "ZONE":
                return ""
            return "test"
        mock_config.get.side_effect = config_get
        start_platforms(mock.ANY)

        mock_log.critical.assert_called_with('GCP zone name is empty!')

    @mock.patch('run.log')
    @mock.patch('run.config')
    def test_start_platforms_with_empty_project_name(self, mock_config, mock_log):
        """Test start_platforms function if GCP project name is not provided in config."""
        def config_get(key, *args, **kwargs):
            if key == "PROJECT_NAME":
                return ""
            return "test"
        mock_config.get.side_effect = config_get
        start_platforms(mock.ANY)

        mock_log.critical.assert_called_with('GCP project name is empty!')

    @mock.patch('run.log')
    @mock.patch('mod_ci.controllers.update_status_on_github')
    @mock.patch('mod_ci.controllers.TestProgress')
    @mock.patch('mod_ci.controllers.g')
    def test_mark_test_failed_success(self, mock_g, mock_test_progress, mock_update_status, mock_log):
        """Test mark_test_failed function successfully marks a test as failed."""
        from mod_ci.controllers import mark_test_failed

        test = Test.query.first()
        repository = MagicMock()
        mock_commit = MagicMock()
        repository.get_commit.return_value = mock_commit

        mark_test_failed(mock_g.db, test, repository, "Test error message")

        mock_test_progress.assert_called_once()
        mock_g.db.add.assert_called_once()
        mock_g.db.commit.assert_called_once()
        mock_update_status.assert_called_once()
        mock_log.info.assert_called()

    @mock.patch('run.log')
    @mock.patch('mod_ci.controllers.update_status_on_github')
    @mock.patch('mod_ci.controllers.TestProgress')
    @mock.patch('mod_ci.controllers.g')
    def test_mark_test_failed_exception(self, mock_g, mock_test_progress, mock_update_status, mock_log):
        """Test mark_test_failed function handles exceptions gracefully."""
        from mod_ci.controllers import mark_test_failed

        test = Test.query.first()
        repository = MagicMock()
        repository.get_commit.side_effect = Exception("GitHub API error")

        # Should not raise, just log the error
        mark_test_failed(mock_g.db, test, repository, "Test error message")

        mock_log.error.assert_called()

    @mock.patch('github.Github.get_repo')
    @mock.patch('mod_ci.controllers.start_test')
    @mock.patch('mod_ci.controllers.get_compute_service_object')
    @mock.patch('mod_ci.controllers.g')
    @mock.patch('run.log')
    def test_gcp_instance_github_exception(self, mock_log, mock_g, mock_get_compute,
                                           mock_start_test, mock_repo):
        """Test gcp_instance handles GithubException gracefully."""
        from github import GithubException

        from mod_ci.controllers import gcp_instance

        repo = mock_repo()
        # Make get_commit raise GithubException
        repo.get_commit.side_effect = GithubException(404, "Not found", None)

        # Create a PR type test that will trigger the exception
        test = Test(TestPlatform.linux, TestType.pull_request, 1, "test", "abc123", 1)
        g.db.add(test)
        g.db.commit()

        gcp_instance(self.app, g.db, TestPlatform.linux, repo, None)

        # Should log error and continue
        mock_log.error.assert_called()

    @mock.patch('github.Github.get_repo')
    @mock.patch('mod_ci.controllers.start_test')
    @mock.patch('mod_ci.controllers.get_compute_service_object')
    @mock.patch('mod_ci.controllers.g')
    @mock.patch('run.log')
    def test_gcp_instance_unexpected_exception(self, mock_log, mock_g, mock_get_compute,
                                               mock_start_test, mock_repo):
        """Test gcp_instance handles unexpected exceptions gracefully."""
        from mod_ci.controllers import gcp_instance

        repo = mock_repo()
        # Make get_commit raise unexpected exception
        repo.get_commit.side_effect = RuntimeError("Unexpected error")

        # Create a PR type test that will trigger the exception
        test = Test(TestPlatform.linux, TestType.pull_request, 1, "test", "def456", 2)
        g.db.add(test)
        g.db.commit()

        gcp_instance(self.app, g.db, TestPlatform.linux, repo, None)

        # Should log error and continue
        mock_log.error.assert_called()

    @mock.patch('mod_ci.controllers.create_instance')
    @mock.patch('builtins.open', new_callable=mock.mock_open())
    @mock.patch('mod_ci.controllers.g')
    @mock.patch('mod_ci.controllers.TestProgress')
    @mock.patch('mod_ci.controllers.GcpInstance')
    @mock.patch('run.log')
    def test_start_test_duplicate_instance_check(
            self, mock_log, mock_gcp_instance, mock_test_progress,
            mock_g, mock_open_file, mock_create_instance):
        """Test start_test skips if GCP instance already exists for test."""
        from mod_ci.controllers import start_test

        test = Test.query.first()
        repository = MagicMock()

        # Mock that an instance already exists
        gcp_filter = mock_gcp_instance.query.filter.return_value
        gcp_filter.first.return_value = MagicMock()

        start_test(mock.ANY, self.app, mock_g.db, repository, test, mock.ANY)

        # Should log warning and return early
        mock_log.warning.assert_called()
        mock_create_instance.assert_not_called()

    @mock.patch('mod_ci.controllers.create_instance')
    @mock.patch('builtins.open', new_callable=mock.mock_open())
    @mock.patch('mod_ci.controllers.g')
    @mock.patch('mod_ci.controllers.TestProgress')
    @mock.patch('mod_ci.controllers.GcpInstance')
    @mock.patch('run.log')
    def test_start_test_duplicate_progress_check(
            self, mock_log, mock_gcp_instance, mock_test_progress,
            mock_g, mock_open_file, mock_create_instance):
        """Test start_test skips if test already has progress entries."""
        from mod_ci.controllers import start_test

        test = Test.query.first()
        repository = MagicMock()

        # Mock no instance but progress exists
        gcp_filter = mock_gcp_instance.query.filter.return_value
        gcp_filter.first.return_value = None
        prog_filter = mock_test_progress.query.filter.return_value
        prog_filter.first.return_value = MagicMock()

        start_test(mock.ANY, self.app, mock_g.db, repository, test, mock.ANY)

        # Should log warning and return early
        mock_log.warning.assert_called()
        mock_create_instance.assert_not_called()

    @mock.patch('mod_ci.controllers.mark_test_failed')
    @mock.patch('mod_ci.controllers.create_instance')
    @mock.patch('builtins.open', new_callable=mock.mock_open())
    @mock.patch('mod_ci.controllers.g')
    @mock.patch('mod_ci.controllers.TestProgress')
    @mock.patch('mod_ci.controllers.GcpInstance')
    @mock.patch('run.log')
    @mock.patch('requests.get')
    def test_start_test_artifact_timeout(
            self, mock_requests_get, mock_log, mock_gcp_instance,
            mock_test_progress, mock_g, mock_open_file,
            mock_create_instance, mock_mark_failed):
        """Test start_test handles artifact download timeout."""
        import requests
        from github.Artifact import Artifact

        from mod_ci.controllers import Artifact_names, start_test

        test = Test.query.first()
        repository = MagicMock()

        # Mock locking checks
        gcp_filter = mock_gcp_instance.query.filter.return_value
        gcp_filter.first.return_value = None
        prog_filter = mock_test_progress.query.filter.return_value
        prog_filter.first.return_value = None

        # Mock artifact
        artifact = MagicMock(Artifact)
        artifact.name = Artifact_names.linux
        artifact.workflow_run.head_sha = test.commit
        repository.get_artifacts.return_value = [artifact]

        # Mock timeout exception
        mock_requests_get.side_effect = requests.exceptions.Timeout()

        customized_test = CustomizedTest(1, 1)
        g.db.add(customized_test)
        g.db.commit()

        start_test(mock.ANY, self.app, g.db, repository, test, mock.ANY)

        # Should log critical and mark test failed
        mock_log.critical.assert_called()
        mock_mark_failed.assert_called()
        mock_create_instance.assert_not_called()

    @mock.patch('mod_ci.controllers.mark_test_failed')
    @mock.patch('mod_ci.controllers.create_instance')
    @mock.patch('builtins.open', new_callable=mock.mock_open())
    @mock.patch('mod_ci.controllers.g')
    @mock.patch('mod_ci.controllers.TestProgress')
    @mock.patch('mod_ci.controllers.GcpInstance')
    @mock.patch('run.log')
    @mock.patch('requests.get')
    def test_start_test_artifact_http_error(
            self, mock_requests_get, mock_log, mock_gcp_instance,
            mock_test_progress, mock_g, mock_open_file,
            mock_create_instance, mock_mark_failed):
        """Test start_test handles artifact download HTTP errors."""
        import requests
        from github.Artifact import Artifact

        from mod_ci.controllers import Artifact_names, start_test

        test = Test.query.first()
        repository = MagicMock()

        # Mock locking checks
        gcp_filter = mock_gcp_instance.query.filter.return_value
        gcp_filter.first.return_value = None
        prog_filter = mock_test_progress.query.filter.return_value
        prog_filter.first.return_value = None

        # Mock artifact
        artifact = MagicMock(Artifact)
        artifact.name = Artifact_names.linux
        artifact.workflow_run.head_sha = test.commit
        repository.get_artifacts.return_value = [artifact]

        # Mock HTTP 500 response
        response = requests.models.Response()
        response.status_code = 500
        mock_requests_get.return_value = response

        customized_test = CustomizedTest(1, 1)
        g.db.add(customized_test)
        g.db.commit()

        start_test(mock.ANY, self.app, g.db, repository, test, mock.ANY)

        # Should log critical and mark test failed
        mock_log.critical.assert_called()
        mock_mark_failed.assert_called()
        mock_create_instance.assert_not_called()

    @mock.patch('run.log')
    @mock.patch('time.sleep')
    @mock.patch('time.time')
    def test_wait_for_operation_timeout(self, mock_time, mock_sleep, mock_log):
        """Test wait_for_operation returns timeout error after max wait."""
        from mod_ci.controllers import wait_for_operation

        compute = MagicMock()
        # Simulate time passing beyond max_wait
        # Start at 0, then jump to 100 seconds
        mock_time.side_effect = [0, 100]

        result = wait_for_operation(
            compute, "project", "zone", "operation", max_wait=50)

        self.assertEqual(result['status'], 'TIMEOUT')
        self.assertIn('error', result)
        mock_log.error.assert_called()

    @mock.patch('run.log')
    @mock.patch('time.sleep')
    @mock.patch('time.time')
    def test_wait_for_operation_api_error(
            self, mock_time, mock_sleep, mock_log):
        """Test wait_for_operation handles API errors gracefully."""
        from mod_ci.controllers import wait_for_operation

        compute = MagicMock()
        # Make the API call raise an exception
        zone_ops = compute.zoneOperations.return_value.get.return_value
        zone_ops.execute.side_effect = Exception("API Error")
        mock_time.return_value = 0

        result = wait_for_operation(
            compute, "project", "zone", "operation", max_wait=100)

        self.assertEqual(result['status'], 'ERROR')
        self.assertIn('error', result)
        mock_log.error.assert_called()

    @mock.patch('run.log')
    @mock.patch('time.sleep')
    @mock.patch('time.time')
    def test_wait_for_operation_success(self, mock_time, mock_sleep, mock_log):
        """Test wait_for_operation returns successfully on completion."""
        from mod_ci.controllers import wait_for_operation

        compute = MagicMock()
        # First call returns RUNNING, second returns DONE
        zone_ops = compute.zoneOperations.return_value.get.return_value
        zone_ops.execute.side_effect = [
            {'status': 'RUNNING'},
            {'status': 'DONE'}
        ]
        # Various time readings during the loop
        mock_time.side_effect = [0, 0, 5, 5]

        result = wait_for_operation(
            compute, "project", "zone", "operation", max_wait=100)

        self.assertEqual(result['status'], 'DONE')
        mock_log.info.assert_called()

    @mock.patch('mod_ci.controllers.Github')
    @mock.patch('mod_ci.controllers.delete_instance')
    @mock.patch('mod_ci.controllers.get_compute_service_object')
    @mock.patch('mod_ci.controllers.update_build_badge')
    def test_progress_type_request_empty_token(
            self, mock_update_build_badge, mock_get_compute_service_object,
            mock_delete_instance, mock_github):
        """Test progress_type_request returns True when GitHub token is empty."""
        from mod_ci.models import GcpInstance
        from run import log

        self.create_user_with_role(
            self.user.name, self.user.email, self.user.password, Role.tester)
        self.create_forktest("own-fork-commit", TestPlatform.linux, regression_tests=[2])
        request = MagicMock()
        request.form = {'status': 'completed', 'message': 'Ran all tests'}
        gcp_instance = GcpInstance(name='test_instance', test_id=3)
        g.db.add(gcp_instance)
        g.db.commit()

        test = Test.query.filter(Test.id == 3).first()

        with empty_github_token():
            response = progress_type_request(log, test, test.id, request)
            self.assertTrue(response)
            mock_github.assert_not_called()

    def test_blocked_users_empty_token(self):
        """Test blocked_users handles empty GitHub token gracefully."""
        self.create_user_with_role(self.user.name, self.user.email, self.user.password, Role.admin)

        with empty_github_token():
            with self.app.test_client() as c:
                c.post("/account/login", data=self.create_login_form_data(self.user.email, self.user.password))
                c.post("/blocked_users", data=dict(user_id=999, comment="Test user", add=True))
                self.assertIsNotNone(BlockedUsers.query.filter(BlockedUsers.user_id == 999).first())

    @mock.patch('requests.get', side_effect=mock_api_request_github)
    @mock.patch('run.get_github_config')
    def test_start_ci_empty_token(self, mock_get_github_config, mock_requests):
        """Test start_ci returns 500 when GitHub token is empty."""
        payload = {'ref': 'refs/heads/master', 'after': 'abc123'}
        headers = self.generate_header(payload, 'push')

        mock_get_github_config.return_value = {
            'ci_key': g.github['ci_key'],
            'bot_token': '',
            'repository_owner': g.github['repository_owner'],
            'repository': g.github['repository']
        }

        with self.app.test_client() as c:
            response = c.post('/start-ci', environ_overrides=WSGI_ENVIRONMENT,
                              headers=headers,
                              data=json.dumps(payload),
                              content_type='application/json')
            self.assertEqual(response.status_code, 500)
            self.assertIn(b'GitHub token not configured', response.data)

    def test_comment_pr_empty_token(self):
        """Test comment_pr returns FAILURE when GitHub token is empty."""
        from mod_ci.controllers import comment_pr
        from mod_ci.models import Status

        test = Test.query.first()

        with empty_github_token():
            result = comment_pr(test)
            self.assertEqual(result, Status.FAILURE)

    def test_is_valid_commit_hash_valid_full(self):
        """Test is_valid_commit_hash with valid full SHA."""
        self.assertTrue(is_valid_commit_hash('1978060bf7d2edd119736ba3ba88341f3bec3323'))

    def test_is_valid_commit_hash_valid_short(self):
        """Test is_valid_commit_hash with valid short SHA."""
        self.assertTrue(is_valid_commit_hash('1978060'))
        self.assertTrue(is_valid_commit_hash('abcdef1'))

    def test_is_valid_commit_hash_null_sha(self):
        """Test is_valid_commit_hash rejects null SHA."""
        self.assertFalse(is_valid_commit_hash('0000000000000000000000000000000000000000'))
        self.assertFalse(is_valid_commit_hash('0000000'))

    def test_is_valid_commit_hash_empty(self):
        """Test is_valid_commit_hash rejects empty/None."""
        self.assertFalse(is_valid_commit_hash(''))
        self.assertFalse(is_valid_commit_hash(None))
        self.assertFalse(is_valid_commit_hash('   '))

    def test_is_valid_commit_hash_too_short(self):
        """Test is_valid_commit_hash rejects too short hashes."""
        self.assertFalse(is_valid_commit_hash('abc'))
        self.assertFalse(is_valid_commit_hash('123456'))

    def test_is_valid_commit_hash_invalid_chars(self):
        """Test is_valid_commit_hash rejects non-hex characters."""
        self.assertFalse(is_valid_commit_hash('xyz1234567890'))
        self.assertFalse(is_valid_commit_hash('ghijklm'))

    def test_is_valid_commit_hash_with_whitespace(self):
        """Test is_valid_commit_hash strips whitespace correctly."""
        # Valid hash with whitespace should be accepted (whitespace stripped)
        self.assertTrue(is_valid_commit_hash('  1978060bf7d2edd119736ba3ba88341f3bec3323  '))
        self.assertTrue(is_valid_commit_hash('\t1978060\n'))

    def test_is_valid_commit_hash_too_long(self):
        """Test is_valid_commit_hash rejects hashes > 40 characters."""
        # 41 characters - too long
        self.assertFalse(is_valid_commit_hash('1978060bf7d2edd119736ba3ba88341f3bec33231'))
        # 50 characters - way too long
        self.assertFalse(is_valid_commit_hash('1978060bf7d2edd119736ba3ba88341f3bec332312345678'))

    @mock.patch('mod_ci.controllers.add_test_entry')
    @mock.patch('github.Github.get_repo')
    @mock.patch('requests.get', side_effect=mock_api_request_github)
    def test_webhook_release_gets_commit_from_tag(self, mock_request, mock_repo, mock_add_test):
        """Test that release webhook gets commit from tag, not last_commit."""
        # Setup mock for tag ref (lightweight tag)
        commit_from_tag = "aaaa111122223333444455556666777788889999"
        mock_tag_ref = MagicMock()
        mock_tag_ref.object.type = "commit"
        mock_tag_ref.object.sha = commit_from_tag
        mock_repo.return_value.get_git_ref.return_value = mock_tag_ref

        with self.app.test_client() as c:
            data = {
                'action': 'published',
                'release': {
                    'prerelease': False,
                    'published_at': '2018-05-30T20:18:44Z',
                    'tag_name': 'v2.2'
                }
            }
            c.post(
                '/start-ci', environ_overrides=WSGI_ENVIRONMENT,
                data=json.dumps(data), headers=self.generate_header(data, 'release'))

            # Verify the release was created with the commit from tag
            release = CCExtractorVersion.query.filter_by(version='2.2').first()
            self.assertIsNotNone(release)
            self.assertEqual(release.commit, commit_from_tag)

    @mock.patch('mod_ci.controllers.add_test_entry')
    @mock.patch('github.Github.get_repo')
    @mock.patch('requests.get', side_effect=mock_api_request_github)
    def test_webhook_release_annotated_tag(self, mock_request, mock_repo, mock_add_test):
        """Test release webhook handles annotated tags correctly."""
        # Setup mock for annotated tag
        actual_commit = "bbbb222233334444555566667777888899990000"
        mock_tag_ref = MagicMock()
        mock_tag_ref.object.type = "tag"  # Annotated tag
        mock_tag_ref.object.sha = "cccc333344445555666677778888999900001111"

        mock_tag_obj = MagicMock()
        mock_tag_obj.object.sha = actual_commit

        mock_repo.return_value.get_git_ref.return_value = mock_tag_ref
        mock_repo.return_value.get_git_tag.return_value = mock_tag_obj

        with self.app.test_client() as c:
            data = {
                'action': 'published',
                'release': {
                    'prerelease': False,
                    'published_at': '2018-05-30T20:18:44Z',
                    'tag_name': 'v2.3'
                }
            }
            c.post(
                '/start-ci', environ_overrides=WSGI_ENVIRONMENT,
                data=json.dumps(data), headers=self.generate_header(data, 'release'))

            release = CCExtractorVersion.query.filter_by(version='2.3').first()
            self.assertIsNotNone(release)
            self.assertEqual(release.commit, actual_commit)

    @mock.patch('github.Github.get_repo')
    @mock.patch('requests.get', side_effect=mock_api_request_github)
    def test_webhook_release_invalid_commit_fallback(self, mock_request, mock_repo):
        """Test release webhook falls back to last_commit when tag lookup fails."""
        from github import GithubException

        # Setup: tag lookup fails
        mock_repo.return_value.get_git_ref.side_effect = GithubException(404, "Not Found", None)

        # Set a valid last_commit as fallback (must be valid hex)
        fallback_commit = 'dddd444455556666777788889999aaaa0000bbbb'
        last_commit = GeneralData.query.filter(GeneralData.key == 'last_commit').first()
        last_commit.value = fallback_commit
        g.db.commit()

        with self.app.test_client() as c:
            data = {
                'action': 'published',
                'release': {
                    'prerelease': False,
                    'published_at': '2018-05-30T20:18:44Z',
                    'tag_name': 'v2.4'
                }
            }
            c.post(
                '/start-ci', environ_overrides=WSGI_ENVIRONMENT,
                data=json.dumps(data), headers=self.generate_header(data, 'release'))

            release = CCExtractorVersion.query.filter_by(version='2.4').first()
            self.assertIsNotNone(release)
            self.assertEqual(release.commit, fallback_commit)

    @mock.patch('github.Github.get_repo')
    @mock.patch('requests.get', side_effect=mock_api_request_github)
    def test_webhook_release_with_existing_test(self, mock_request, mock_repo):
        """Test release webhook updates baseline when test entry exists for commit."""
        # Setup mock for tag ref (lightweight tag)
        commit_hash = "eeee555566667777888899990000aaaabbbbcccc"
        mock_tag_ref = MagicMock()
        mock_tag_ref.object.type = "commit"
        mock_tag_ref.object.sha = commit_hash
        mock_repo.return_value.get_git_ref.return_value = mock_tag_ref

        # Create a test entry for this commit
        from mod_test.models import Fork
        fork = Fork.query.first()
        test = Test(TestPlatform.linux, TestType.commit, fork.id, "master", commit_hash, 0)
        g.db.add(test)
        g.db.commit()

        with self.app.test_client() as c:
            data = {
                'action': 'published',
                'release': {
                    'prerelease': False,
                    'published_at': '2018-05-30T20:18:44Z',
                    'tag_name': 'v2.6'
                }
            }
            c.post(
                '/start-ci', environ_overrides=WSGI_ENVIRONMENT,
                data=json.dumps(data), headers=self.generate_header(data, 'release'))

            # Release should be created with the commit from tag
            release = CCExtractorVersion.query.filter_by(version='2.6').first()
            self.assertIsNotNone(release)
            self.assertEqual(release.commit, commit_hash)

    @mock.patch('github.Github.get_repo')
    @mock.patch('requests.get', side_effect=mock_api_request_github)
    def test_webhook_release_no_valid_commit_fails(self, mock_request, mock_repo):
        """Test release webhook fails gracefully when no valid commit available."""
        from github import GithubException

        # Setup: tag lookup fails
        mock_repo.return_value.get_git_ref.side_effect = GithubException(404, "Not Found", None)

        # Set an invalid last_commit (null SHA)
        last_commit = GeneralData.query.filter(GeneralData.key == 'last_commit').first()
        last_commit.value = '0000000000000000000000000000000000000000'
        g.db.commit()

        with self.app.test_client() as c:
            data = {
                'action': 'published',
                'release': {
                    'prerelease': False,
                    'published_at': '2018-05-30T20:18:44Z',
                    'tag_name': 'v2.5'
                }
            }
            c.post(
                '/start-ci', environ_overrides=WSGI_ENVIRONMENT,
                data=json.dumps(data), headers=self.generate_header(data, 'release'))

            # Release should NOT be created
            release = CCExtractorVersion.query.filter_by(version='2.5').first()
            self.assertIsNone(release)

    @mock.patch('mod_ci.controllers.time.sleep')
    @mock.patch('run.log')
    def test_retry_with_backoff_success_first_try(self, mock_log, mock_sleep):
        """Test retry_with_backoff succeeds on first attempt."""
        mock_func = MagicMock(return_value="success")

        result = retry_with_backoff(mock_func, max_retries=3)

        self.assertEqual(result, "success")
        mock_func.assert_called_once()
        mock_sleep.assert_not_called()

    @mock.patch('mod_ci.controllers.time.sleep')
    @mock.patch('run.log')
    def test_retry_with_backoff_success_after_retry(
            self, mock_log, mock_sleep):
        """Test retry_with_backoff succeeds after retries."""
        from github import GithubException

        mock_func = MagicMock(side_effect=[
            GithubException(500, "Server Error", None),
            GithubException(500, "Server Error", None),
            "success"
        ])

        result = retry_with_backoff(
            mock_func, max_retries=3, initial_backoff=1)

        self.assertEqual(result, "success")
        self.assertEqual(mock_func.call_count, 3)
        self.assertEqual(mock_sleep.call_count, 2)
        mock_log.warning.assert_called()

    @mock.patch('mod_ci.controllers.time.sleep')
    @mock.patch('run.log')
    def test_retry_with_backoff_all_retries_fail(self, mock_log, mock_sleep):
        """Test retry_with_backoff raises exception when retries fail."""
        from github import GithubException

        exc = GithubException(500, "Server Error", None)
        mock_func = MagicMock(side_effect=exc)

        with self.assertRaises(GithubException):
            retry_with_backoff(mock_func, max_retries=2, initial_backoff=1)

        self.assertEqual(mock_func.call_count, 3)  # Initial + 2 retries
        self.assertEqual(mock_sleep.call_count, 2)
        mock_log.error.assert_called()

    @mock.patch('mod_ci.controllers.time.sleep')
    @mock.patch('run.log')
    def test_retry_with_backoff_exponential_backoff(
            self, mock_log, mock_sleep):
        """Test retry_with_backoff uses exponential backoff."""
        from github import GithubException

        mock_func = MagicMock(side_effect=[
            GithubException(500, "Error", None),
            GithubException(500, "Error", None),
            GithubException(500, "Error", None),
            "success"
        ])

        retry_with_backoff(
            mock_func, max_retries=3, initial_backoff=1, max_backoff=30)

        # Check backoff times: 1s, 2s, 4s
        sleep_calls = [call[0][0] for call in mock_sleep.call_args_list]
        self.assertEqual(sleep_calls, [1, 2, 4])

    @mock.patch('run.log')
    def test_safe_db_commit_success(self, mock_log):
        """Test safe_db_commit returns True on success."""
        mock_db = MagicMock()

        result = safe_db_commit(mock_db, "test operation")

        self.assertTrue(result)
        mock_db.commit.assert_called_once()
        mock_db.rollback.assert_not_called()

    @mock.patch('run.log')
    def test_safe_db_commit_failure_with_rollback(self, mock_log):
        """Test safe_db_commit rolls back and returns False on failure."""
        mock_db = MagicMock()
        mock_db.commit.side_effect = Exception("DB Error")

        result = safe_db_commit(mock_db, "test operation")

        self.assertFalse(result)
        mock_db.commit.assert_called_once()
        mock_db.rollback.assert_called_once()
        mock_log.error.assert_called()
        mock_log.info.assert_called()  # Rollback success message

    @mock.patch('run.log')
    def test_safe_db_commit_failure_with_rollback_failure(self, mock_log):
        """Test safe_db_commit handles rollback failure gracefully."""
        mock_db = MagicMock()
        mock_db.commit.side_effect = Exception("DB Error")
        mock_db.rollback.side_effect = Exception("Rollback Error")

        result = safe_db_commit(mock_db, "test operation")

        self.assertFalse(result)
        mock_db.commit.assert_called_once()
        mock_db.rollback.assert_called_once()
        # Two error logs: one for commit failure, one for rollback failure
        self.assertEqual(mock_log.error.call_count, 2)

    @mock.patch('mod_ci.controllers.update_status_on_github')
    @mock.patch('run.log')
    def test_mark_test_failed_with_db_commit_failure(
            self, mock_log, mock_update_github):
        """Test mark_test_failed returns False when DB commit fails but GitHub succeeds."""
        from mod_test.models import Test

        test = Test.query.first()
        mock_db = MagicMock()
        mock_db.commit.side_effect = Exception("DB commit failed")
        mock_repo = MagicMock()

        result = mark_test_failed(mock_db, test, mock_repo, "Test failed")

        # DB failed but GitHub succeeded, so result is False
        self.assertFalse(result)
        mock_db.add.assert_called_once()
        mock_db.commit.assert_called_once()
        # GitHub update should still be attempted
        mock_update_github.assert_called_once()

    @mock.patch('mod_ci.controllers.time.sleep')
    @mock.patch('run.log')
    def test_retry_with_backoff_max_backoff_cap(self, mock_log, mock_sleep):
        """Test retry_with_backoff caps backoff at max_backoff."""
        from github import GithubException

        mock_func = MagicMock(side_effect=[
            GithubException(500, "Error", None),
            GithubException(500, "Error", None),
            GithubException(500, "Error", None),
            GithubException(500, "Error", None),
            "success"
        ])

        # With initial_backoff=8 and max_backoff=10,
        # backoff should be: 8, 10, 10, 10
        retry_with_backoff(
            mock_func, max_retries=4, initial_backoff=8, max_backoff=10)

        sleep_calls = [call[0][0] for call in mock_sleep.call_args_list]
        self.assertEqual(sleep_calls, [8, 10, 10, 10])

    @mock.patch('mod_ci.controllers.time.sleep')
    @mock.patch('run.log')
    def test_retry_with_backoff_custom_exceptions(self, mock_log, mock_sleep):
        """Test retry_with_backoff with custom exception types."""
        mock_func = MagicMock(side_effect=[
            ValueError("Custom error"),
            "success"
        ])

        result = retry_with_backoff(
            mock_func,
            max_retries=2,
            initial_backoff=1,
            retryable_exceptions=(ValueError,)
        )

        self.assertEqual(result, "success")
        self.assertEqual(mock_func.call_count, 2)

    @mock.patch('github.Github.get_repo')
    @mock.patch('mod_ci.controllers.start_test')
    @mock.patch('mod_ci.controllers.get_compute_service_object')
    @mock.patch('mod_ci.controllers.g')
    @mock.patch('run.log')
    def test_gcp_instance_pr_closed(self, mock_log, mock_g, mock_get_compute, mock_start_test, mock_repo):
        """Test gcp_instance handles closed PR correctly."""
        from mod_ci.controllers import gcp_instance

        repo = mock_repo()
        mock_commit = MagicMock()
        repo.get_commit.return_value = mock_commit

        # Mock a closed PR
        mock_pr = MagicMock()
        mock_pr.state = 'closed'
        mock_pr.head.sha = '1978060bf7d2edd119736ba3ba88341f3bec3322'
        repo.get_pull.return_value = mock_pr

        # Setup mock config
        mock_g.db = g.db

        gcp_instance(self.app, mock_g.db, TestPlatform.linux, repo, None)

        # start_test should not be called for closed PRs
        # test_1 has pr_nr=0 (descheduled)
        # test_2 is closed PR (descheduled)
        # test_3 and test_4 are open PRs
        # But since we mocked all PRs as closed, only test_1 (invalid) triggers, rest are skipped
        mock_log.info.assert_any_call(mock.ANY)

    @mock.patch('mod_ci.controllers.update_status_on_github')
    @mock.patch('mod_ci.controllers.retry_with_backoff')
    @mock.patch('mod_ci.controllers.safe_db_commit')
    @mock.patch('run.log')
    def test_mark_test_failed_returns_true_on_success(self, mock_log, mock_safe_commit, mock_retry, mock_update):
        """Test mark_test_failed returns True on successful execution."""
        from mod_test.models import Test

        mock_safe_commit.return_value = True
        mock_retry.return_value = MagicMock()

        test = Test.query.first()
        mock_db = MagicMock()
        mock_repo = MagicMock()

        result = mark_test_failed(mock_db, test, mock_repo, "Test failed")

        self.assertTrue(result)
        mock_log.info.assert_called()

    @staticmethod
    def generate_header(data, event, ci_key=None):
        """
        Generate headers for various REST methods.

        :param data: payload for the event
        :type data: dict
        :param event: the GitHub event to be triggered
        :type event: str
        """
        sig = generate_signature(str(json.dumps(data)).encode(
            'utf-8'), g.github['ci_key'] if ci_key is None else ci_key)
        headers = generate_git_api_header(event, sig)
        return headers


class TestFindArtifactForCommit(BaseTestCase):
    """Test the find_artifact_for_commit function."""

    def test_find_artifact_success(self):
        """Test finding an artifact successfully."""
        from mod_ci.controllers import Artifact_names, find_artifact_for_commit

        repository = MagicMock()
        mock_artifact = MagicMock()
        mock_artifact.name = Artifact_names.linux
        mock_artifact.workflow_run.head_sha = "abc123def456"

        # Return the matching artifact
        repository.get_artifacts.return_value = [mock_artifact]

        log = MagicMock()
        result = find_artifact_for_commit(repository, "abc123def456", TestPlatform.linux, log)

        self.assertIsNotNone(result)
        self.assertEqual(result, mock_artifact)
        log.debug.assert_called()

    def test_find_artifact_not_found(self):
        """Test when artifact is not found."""
        from mod_ci.controllers import Artifact_names, find_artifact_for_commit

        repository = MagicMock()
        mock_artifact = MagicMock()
        mock_artifact.name = Artifact_names.linux
        mock_artifact.workflow_run.head_sha = "different_commit"

        repository.get_artifacts.return_value = [mock_artifact]

        log = MagicMock()
        result = find_artifact_for_commit(repository, "abc123def456", TestPlatform.linux, log)

        self.assertIsNone(result)

    def test_find_artifact_wrong_platform(self):
        """Test when artifact exists but for wrong platform."""
        from mod_ci.controllers import Artifact_names, find_artifact_for_commit

        repository = MagicMock()
        mock_artifact = MagicMock()
        mock_artifact.name = Artifact_names.windows  # Wrong platform
        mock_artifact.workflow_run.head_sha = "abc123def456"

        repository.get_artifacts.return_value = [mock_artifact]

        log = MagicMock()
        result = find_artifact_for_commit(repository, "abc123def456", TestPlatform.linux, log)

        self.assertIsNone(result)

    def test_find_artifact_handles_exception(self):
        """Test that exceptions are handled gracefully."""
        from mod_ci.controllers import find_artifact_for_commit

        repository = MagicMock()
        repository.get_artifacts.side_effect = Exception("API error")

        log = MagicMock()
        result = find_artifact_for_commit(repository, "abc123def456", TestPlatform.linux, log)

        self.assertIsNone(result)
        log.error.assert_called()

    def test_find_artifact_respects_max_search_limit(self):
        """Test that artifact search respects the maximum search limit."""
        from mod_ci.controllers import (MAX_ARTIFACTS_TO_SEARCH,
                                        Artifact_names,
                                        find_artifact_for_commit)

        repository = MagicMock()

        # Create many artifacts that don't match
        def generate_artifacts():
            for i in range(MAX_ARTIFACTS_TO_SEARCH + 100):
                artifact = MagicMock()
                artifact.name = Artifact_names.linux
                artifact.workflow_run.head_sha = f"commit_{i}"
                yield artifact

        repository.get_artifacts.return_value = generate_artifacts()

        log = MagicMock()
        result = find_artifact_for_commit(repository, "target_commit", TestPlatform.linux, log)

        self.assertIsNone(result)
        # Verify warning was logged about reaching limit
        log.warning.assert_called()


class TestVerifyArtifactsExist(BaseTestCase):
    """Test the verify_artifacts_exist function."""

    @mock.patch('mod_ci.controllers.find_artifact_for_commit')
    def test_verify_both_artifacts_exist(self, mock_find):
        """Test verification when both artifacts exist."""
        from mod_ci.controllers import verify_artifacts_exist

        # Both artifacts found
        mock_find.side_effect = [MagicMock(), MagicMock()]

        repository = MagicMock()
        log = MagicMock()

        result = verify_artifacts_exist(repository, "abc123", log)

        self.assertTrue(result['linux'])
        self.assertTrue(result['windows'])

    @mock.patch('mod_ci.controllers.find_artifact_for_commit')
    def test_verify_only_linux_exists(self, mock_find):
        """Test verification when only Linux artifact exists."""
        from mod_ci.controllers import verify_artifacts_exist

        # Only Linux found
        mock_find.side_effect = [MagicMock(), None]

        repository = MagicMock()
        log = MagicMock()

        result = verify_artifacts_exist(repository, "abc123", log)

        self.assertTrue(result['linux'])
        self.assertFalse(result['windows'])

    @mock.patch('mod_ci.controllers.find_artifact_for_commit')
    def test_verify_neither_exists(self, mock_find):
        """Test verification when neither artifact exists."""
        from mod_ci.controllers import verify_artifacts_exist

        # Neither found
        mock_find.side_effect = [None, None]

        repository = MagicMock()
        log = MagicMock()

        result = verify_artifacts_exist(repository, "abc123", log)

        self.assertFalse(result['linux'])
        self.assertFalse(result['windows'])
        # Warning should be logged for both
        self.assertEqual(log.warning.call_count, 2)


class TestMarkTestFailedImproved(BaseTestCase):
    """Test the improved mark_test_failed function."""

    @mock.patch('mod_ci.controllers.update_status_on_github')
    @mock.patch('mod_ci.controllers.TestProgress')
    def test_mark_test_failed_updates_both_db_and_github(self, mock_progress, mock_update_github):
        """Test that mark_test_failed updates both database and GitHub."""
        from mod_ci.controllers import mark_test_failed

        db = MagicMock()
        test = MagicMock()
        test.id = 123
        test.commit = "abc123def456"
        test.platform.value = "linux"

        repository = MagicMock()
        gh_commit = MagicMock()
        repository.get_commit.return_value = gh_commit

        with mock.patch('run.log'):
            mark_test_failed(db, test, repository, "Test error message")

        # Verify database was updated
        mock_progress.assert_called_once()
        db.add.assert_called_once()
        db.commit.assert_called_once()

        # Verify GitHub was updated
        mock_update_github.assert_called_once()

    @mock.patch('mod_ci.controllers.update_status_on_github')
    @mock.patch('mod_ci.controllers.TestProgress')
    def test_mark_test_failed_github_updated_even_if_db_fails(self, mock_progress, mock_update_github):
        """Test that GitHub is updated even if database update fails."""
        from mod_ci.controllers import mark_test_failed

        db = MagicMock()
        db.commit.side_effect = Exception("DB error")

        test = MagicMock()
        test.id = 123
        test.commit = "abc123def456"
        test.platform.value = "linux"

        repository = MagicMock()
        gh_commit = MagicMock()
        repository.get_commit.return_value = gh_commit

        with mock.patch('run.log') as mock_log:
            mark_test_failed(db, test, repository, "Test error message")

        # GitHub should still be updated even though DB failed
        mock_update_github.assert_called_once()
        # Error should be logged for DB failure
        mock_log.error.assert_called()

    @mock.patch('mod_ci.controllers.update_status_on_github')
    @mock.patch('mod_ci.controllers.TestProgress')
    def test_mark_test_failed_logs_critical_when_both_fail(self, mock_progress, mock_update_github):
        """Test that critical error is logged when both DB and GitHub updates fail."""
        from mod_ci.controllers import mark_test_failed

        db = MagicMock()
        db.commit.side_effect = Exception("DB error")

        test = MagicMock()
        test.id = 123
        test.commit = "abc123def456"
        test.platform.value = "linux"

        repository = MagicMock()
        repository.get_commit.side_effect = Exception("GitHub error")

        with mock.patch('run.log') as mock_log:
            mark_test_failed(db, test, repository, "Test error message")

        # Critical error should be logged when both fail
        mock_log.critical.assert_called()

    @mock.patch('mod_ci.controllers.update_status_on_github')
    @mock.patch('mod_ci.controllers.TestProgress')
    def test_mark_test_failed_includes_target_url(self, mock_progress, mock_update_github):
        """Test that mark_test_failed includes target_url in GitHub status."""
        from mod_ci.controllers import Status, mark_test_failed

        db = MagicMock()
        test = MagicMock()
        test.id = 456
        test.commit = "abc123def456"
        test.platform.value = "linux"

        repository = MagicMock()
        gh_commit = MagicMock()
        repository.get_commit.return_value = gh_commit

        with mock.patch('run.log'):
            mark_test_failed(db, test, repository, "Test error")

        # Verify target_url was passed to update_status_on_github
        call_args = mock_update_github.call_args
        self.assertEqual(len(call_args[0]), 5)  # 5 positional args including target_url
        self.assertIn("456", call_args[0][4])  # target_url contains test ID

    @mock.patch('mod_ci.controllers.update_status_on_github')
    @mock.patch('mod_ci.controllers.TestProgress')
    @mock.patch('mod_ci.controllers.retry_with_backoff')
    def test_mark_test_failed_uses_retry_for_github(self, mock_retry, mock_progress, mock_update_github):
        """Test that mark_test_failed uses retry_with_backoff for GitHub status update."""
        from mod_ci.controllers import mark_test_failed

        db = MagicMock()
        test = MagicMock()
        test.id = 789
        test.commit = "abc123"
        test.platform.value = "linux"

        repository = MagicMock()
        gh_commit = MagicMock()
        repository.get_commit.return_value = gh_commit

        # Make retry_with_backoff call the function it receives
        mock_retry.side_effect = lambda func, **kwargs: func()

        with mock.patch('run.log'):
            mark_test_failed(db, test, repository, "Test error")

        # Verify retry_with_backoff was called with correct parameters
        mock_retry.assert_called_once()
        call_kwargs = mock_retry.call_args[1]
        self.assertEqual(call_kwargs['max_retries'], 3)
        self.assertEqual(call_kwargs['initial_backoff'], 2.0)


class TestParseGcpError(unittest.TestCase):
    """Tests for the parse_gcp_error helper function."""

    def test_parse_gcp_error_zone_resource_exhausted(self):
        """Test that ZONE_RESOURCE_POOL_EXHAUSTED returns user-friendly message."""
        from mod_ci.controllers import parse_gcp_error

        mock_log = MagicMock()
        result = {
            'status': 'DONE',
            'error': {
                'errors': [{
                    'code': 'ZONE_RESOURCE_POOL_EXHAUSTED',
                    'message': "The zone 'projects/test/zones/us-central1-a' does not "
                               "have enough resources available to fulfill the request."
                }]
            }
        }

        error_msg = parse_gcp_error(result, log=mock_log)
        self.assertIn("GCP resources temporarily unavailable", error_msg)
        self.assertIn("retried automatically", error_msg)
        # Should NOT contain raw technical details
        self.assertNotIn("us-central1-a", error_msg)

    def test_parse_gcp_error_quota_exceeded(self):
        """Test that QUOTA_EXCEEDED returns user-friendly message."""
        from mod_ci.controllers import parse_gcp_error

        mock_log = MagicMock()
        result = {
            'error': {
                'errors': [{
                    'code': 'QUOTA_EXCEEDED',
                    'message': 'Quota exceeded for resource.'
                }]
            }
        }

        error_msg = parse_gcp_error(result, log=mock_log)
        self.assertIn("quota limit reached", error_msg)

    def test_parse_gcp_error_timeout(self):
        """Test that TIMEOUT returns user-friendly message."""
        from mod_ci.controllers import parse_gcp_error

        mock_log = MagicMock()
        result = {
            'status': 'TIMEOUT',
            'error': {
                'errors': [{
                    'code': 'TIMEOUT',
                    'message': 'Operation timed out after 1800 seconds'
                }]
            }
        }

        error_msg = parse_gcp_error(result, log=mock_log)
        self.assertIn("timed out", error_msg)
        self.assertIn("retried automatically", error_msg)

    def test_parse_gcp_error_unknown_code(self):
        """Test that unknown error codes return generic message and log details."""
        from mod_ci.controllers import parse_gcp_error

        mock_log = MagicMock()
        result = {
            'error': {
                'errors': [{
                    'code': 'SOME_NEW_ERROR',
                    'message': 'Something unexpected happened.'
                }]
            }
        }

        error_msg = parse_gcp_error(result, log=mock_log)
        # Should include error code but not the full message (security)
        self.assertIn("SOME_NEW_ERROR", error_msg)
        self.assertIn("contact the administrator", error_msg)
        # Should NOT expose the raw error message
        self.assertNotIn("Something unexpected happened", error_msg)
        # Should log the full details server-side
        mock_log.error.assert_called_once()
        self.assertIn("SOME_NEW_ERROR", mock_log.error.call_args[0][0])
        self.assertIn("Something unexpected happened", mock_log.error.call_args[0][0])

    def test_parse_gcp_error_logs_sensitive_info(self):
        """Test that sensitive info is logged but not returned to user."""
        from mod_ci.controllers import parse_gcp_error

        mock_log = MagicMock()
        result = {
            'error': {
                'errors': [{
                    'code': 'UNKNOWN_ERROR',
                    'message': 'Error in project my-secret-project zone us-central1-a'
                }]
            }
        }

        error_msg = parse_gcp_error(result, log=mock_log)
        # Should NOT expose project/zone names
        self.assertNotIn("my-secret-project", error_msg)
        self.assertNotIn("us-central1-a", error_msg)
        # But should log them server-side
        mock_log.error.assert_called_once()
        self.assertIn("my-secret-project", mock_log.error.call_args[0][0])

    def test_parse_gcp_error_no_error_key(self):
        """Test handling when 'error' key is missing."""
        from mod_ci.controllers import parse_gcp_error

        mock_log = MagicMock()
        result = {'status': 'DONE'}

        error_msg = parse_gcp_error(result, log=mock_log)
        self.assertIn("VM creation failed", error_msg)
        mock_log.error.assert_called_once()

    def test_parse_gcp_error_empty_errors_list(self):
        """Test handling when 'errors' list is empty."""
        from mod_ci.controllers import parse_gcp_error

        mock_log = MagicMock()
        result = {'error': {'errors': []}}

        error_msg = parse_gcp_error(result, log=mock_log)
        self.assertIn("VM creation failed", error_msg)
        mock_log.error.assert_called_once()

    def test_parse_gcp_error_not_a_dict(self):
        """Test handling when result is not a dictionary."""
        from mod_ci.controllers import parse_gcp_error

        mock_log = MagicMock()
        error_msg = parse_gcp_error("some string error", log=mock_log)
        self.assertIn("VM creation failed", error_msg)
        # Should NOT expose the raw input
        self.assertNotIn("some string error", error_msg)
        # But should log it
        mock_log.error.assert_called_once()
        self.assertIn("some string error", mock_log.error.call_args[0][0])

    def test_parse_gcp_error_error_not_a_dict(self):
        """Test handling when 'error' value is not a dictionary."""
        from mod_ci.controllers import parse_gcp_error

        mock_log = MagicMock()
        result = {'error': 'just a string'}

        error_msg = parse_gcp_error(result, log=mock_log)
        self.assertIn("VM creation failed", error_msg)
        mock_log.error.assert_called_once()


class TestDiagnoseMissingArtifact(BaseTestCase):
    """Test the _diagnose_missing_artifact function and retry behavior."""

    def test_build_in_progress_is_retryable(self):
        """Test that build in progress returns retryable=True."""
        from mod_ci.controllers import _diagnose_missing_artifact

        repository = MagicMock()
        log = MagicMock()

        # Mock workflow
        workflow = MagicMock()
        workflow.id = 1
        workflow.name = "Build CCExtractor on Linux"
        repository.get_workflows.return_value = [workflow]

        # Mock workflow run - in progress
        workflow_run = MagicMock()
        workflow_run.workflow_id = 1
        workflow_run.status = "in_progress"
        repository.get_workflow_runs.return_value = [workflow_run]

        message, is_retryable = _diagnose_missing_artifact(
            repository, "abc123", TestPlatform.linux, log
        )

        self.assertTrue(is_retryable)
        self.assertIn("Build still in progress", message)
        self.assertIn("Will retry", message)

    def test_build_queued_is_retryable(self):
        """Test that queued build returns retryable=True."""
        from mod_ci.controllers import _diagnose_missing_artifact

        repository = MagicMock()
        log = MagicMock()

        # Mock workflow
        workflow = MagicMock()
        workflow.id = 1
        workflow.name = "Build CCExtractor on Windows"
        repository.get_workflows.return_value = [workflow]

        # Mock workflow run - queued
        workflow_run = MagicMock()
        workflow_run.workflow_id = 1
        workflow_run.status = "queued"
        repository.get_workflow_runs.return_value = [workflow_run]

        message, is_retryable = _diagnose_missing_artifact(
            repository, "abc123", TestPlatform.windows, log
        )

        self.assertTrue(is_retryable)
        self.assertIn("Build still in progress", message)

    def test_build_failed_is_not_retryable(self):
        """Test that failed build returns retryable=False."""
        from mod_ci.controllers import _diagnose_missing_artifact

        repository = MagicMock()
        log = MagicMock()

        # Mock workflow
        workflow = MagicMock()
        workflow.id = 1
        workflow.name = "Build CCExtractor on Linux"
        repository.get_workflows.return_value = [workflow]

        # Mock workflow run - completed but failed
        workflow_run = MagicMock()
        workflow_run.workflow_id = 1
        workflow_run.status = "completed"
        workflow_run.conclusion = "failure"
        repository.get_workflow_runs.return_value = [workflow_run]

        message, is_retryable = _diagnose_missing_artifact(
            repository, "abc123", TestPlatform.linux, log
        )

        self.assertFalse(is_retryable)
        self.assertIn("Build failed", message)

    def test_artifact_expired_is_not_retryable(self):
        """Test that expired artifact returns retryable=False."""
        from mod_ci.controllers import _diagnose_missing_artifact

        repository = MagicMock()
        log = MagicMock()

        # Mock workflow
        workflow = MagicMock()
        workflow.id = 1
        workflow.name = "Build CCExtractor on Linux"
        repository.get_workflows.return_value = [workflow]

        # Mock workflow run - completed successfully but artifact gone
        workflow_run = MagicMock()
        workflow_run.workflow_id = 1
        workflow_run.status = "completed"
        workflow_run.conclusion = "success"
        repository.get_workflow_runs.return_value = [workflow_run]

        message, is_retryable = _diagnose_missing_artifact(
            repository, "abc123", TestPlatform.linux, log
        )

        self.assertFalse(is_retryable)
        self.assertIn("Artifact not found", message)
        self.assertIn("expired", message)

    def test_recently_completed_build_is_retryable(self):
        """Test that recently completed build (within grace period) returns retryable=True."""
        from datetime import datetime, timedelta, timezone

        from mod_ci.controllers import _diagnose_missing_artifact

        repository = MagicMock()
        log = MagicMock()

        # Mock workflow
        workflow = MagicMock()
        workflow.id = 1
        workflow.name = "Build CCExtractor on Linux"
        repository.get_workflows.return_value = [workflow]

        # Mock workflow run - completed successfully but very recently (1 minute ago)
        workflow_run = MagicMock()
        workflow_run.workflow_id = 1
        workflow_run.status = "completed"
        workflow_run.conclusion = "success"
        workflow_run.updated_at = datetime.now(timezone.utc) - timedelta(seconds=60)
        repository.get_workflow_runs.return_value = [workflow_run]

        message, is_retryable = _diagnose_missing_artifact(
            repository, "abc123", TestPlatform.linux, log
        )

        self.assertTrue(is_retryable)
        self.assertIn("Build completed recently", message)
        self.assertIn("Will retry", message)

    def test_old_completed_build_is_not_retryable(self):
        """Test that build completed more than grace period ago returns retryable=False."""
        from datetime import datetime, timedelta, timezone

        from mod_ci.controllers import _diagnose_missing_artifact

        repository = MagicMock()
        log = MagicMock()

        # Mock workflow
        workflow = MagicMock()
        workflow.id = 1
        workflow.name = "Build CCExtractor on Linux"
        repository.get_workflows.return_value = [workflow]

        # Mock workflow run - completed 10 minutes ago (beyond grace period)
        workflow_run = MagicMock()
        workflow_run.workflow_id = 1
        workflow_run.status = "completed"
        workflow_run.conclusion = "success"
        workflow_run.updated_at = datetime.now(timezone.utc) - timedelta(seconds=600)
        repository.get_workflow_runs.return_value = [workflow_run]

        message, is_retryable = _diagnose_missing_artifact(
            repository, "abc123", TestPlatform.linux, log
        )

        self.assertFalse(is_retryable)
        self.assertIn("Artifact not found", message)

    def test_no_workflow_run_recent_commit_is_retryable(self):
        """Test that missing workflow run for a recent commit returns retryable=True."""
        from datetime import datetime, timezone

        from mod_ci.controllers import _diagnose_missing_artifact

        repository = MagicMock()
        log = MagicMock()

        # Mock workflow
        workflow = MagicMock()
        workflow.id = 1
        workflow.name = "Build CCExtractor on Linux"
        repository.get_workflows.return_value = [workflow]

        # No workflow runs for this commit
        repository.get_workflow_runs.return_value = []

        # Mock a recent commit (1 hour ago)
        commit_obj = MagicMock()
        commit_obj.commit.author.date = datetime.now(timezone.utc)
        repository.get_commit.return_value = commit_obj

        message, is_retryable = _diagnose_missing_artifact(
            repository, "abc123", TestPlatform.linux, log
        )

        self.assertTrue(is_retryable)
        self.assertIn("No workflow run found", message)
        self.assertIn("Will retry", message)

    def test_no_workflow_run_old_commit_not_retryable(self):
        """Test that missing workflow run for an old commit returns retryable=False."""
        from datetime import datetime, timedelta, timezone

        from mod_ci.controllers import _diagnose_missing_artifact

        repository = MagicMock()
        log = MagicMock()

        # Mock workflow
        workflow = MagicMock()
        workflow.id = 1
        workflow.name = "Build CCExtractor on Linux"
        repository.get_workflows.return_value = [workflow]

        # No workflow runs for this commit
        repository.get_workflow_runs.return_value = []

        # Mock an old commit (5 hours ago - beyond the 3 hour limit)
        commit_obj = MagicMock()
        commit_obj.commit.author.date = datetime.now(timezone.utc) - timedelta(hours=5)
        repository.get_commit.return_value = commit_obj

        message, is_retryable = _diagnose_missing_artifact(
            repository, "abc123", TestPlatform.linux, log
        )

        self.assertFalse(is_retryable)
        self.assertIn("No build available", message)
        self.assertIn("never ran", message)
        self.assertIn("5.0 hours old", message)

    def test_diagnostic_exception_is_retryable(self):
        """Test that diagnostic failures default to retryable=True."""
        from mod_ci.controllers import _diagnose_missing_artifact

        repository = MagicMock()
        log = MagicMock()

        # Make get_workflows throw an exception
        repository.get_workflows.side_effect = Exception("API error")

        message, is_retryable = _diagnose_missing_artifact(
            repository, "abc123", TestPlatform.linux, log
        )

        self.assertTrue(is_retryable)
        self.assertIn("diagnostic check failed", message)

    @mock.patch('mod_ci.controllers._will_workflow_run_for_commit')
    def test_workflow_will_not_run_not_retryable(self, mock_will_run):
        """Test that when workflow won't run due to path filters, it's not retryable."""
        from mod_ci.controllers import _diagnose_missing_artifact

        repository = MagicMock()
        log = MagicMock()

        # Workflow will NOT run due to path filters
        mock_will_run.return_value = False

        message, is_retryable = _diagnose_missing_artifact(
            repository, "abc123", TestPlatform.linux, log
        )

        self.assertFalse(is_retryable)
        self.assertIn("No build will be created", message)
        self.assertIn("does not modify any files", message)


class TestWorkflowPathFilters(BaseTestCase):
    """Test the workflow path filter checking functions."""

    @mock.patch('mod_ci.controllers._get_workflow_path_filters')
    def test_will_workflow_run_matches_c_file(self, mock_get_filters):
        """Test that changing a .c file triggers the workflow."""
        from mod_ci.controllers import _will_workflow_run_for_commit

        repository = MagicMock()
        log = MagicMock()

        # Mock path filters
        mock_get_filters.return_value = ['**.c', '**.h', 'linux/**']

        # Mock commit with a .c file changed
        mock_file = MagicMock()
        mock_file.filename = 'src/ccextractor.c'
        mock_commit = MagicMock()
        mock_commit.files = [mock_file]
        repository.get_commit.return_value = mock_commit

        result = _will_workflow_run_for_commit(
            repository, "abc123", "Build CCExtractor on Linux", log
        )

        self.assertTrue(result)

    @mock.patch('mod_ci.controllers._get_workflow_path_filters')
    def test_will_workflow_run_no_match_docs_only(self, mock_get_filters):
        """Test that changing only docs doesn't trigger the workflow."""
        from mod_ci.controllers import _will_workflow_run_for_commit

        repository = MagicMock()
        log = MagicMock()

        # Mock path filters
        mock_get_filters.return_value = ['**.c', '**.h', 'linux/**']

        # Mock commit with only README changed
        mock_file = MagicMock()
        mock_file.filename = 'README.md'
        mock_commit = MagicMock()
        mock_commit.files = [mock_file]
        repository.get_commit.return_value = mock_commit

        result = _will_workflow_run_for_commit(
            repository, "abc123", "Build CCExtractor on Linux", log
        )

        self.assertFalse(result)

    @mock.patch('mod_ci.controllers._get_workflow_path_filters')
    def test_will_workflow_run_no_filters_returns_none(self, mock_get_filters):
        """Test that workflows without path filters return None."""
        from mod_ci.controllers import _will_workflow_run_for_commit

        repository = MagicMock()
        log = MagicMock()

        # No path filters (workflow runs on everything)
        mock_get_filters.return_value = None

        result = _will_workflow_run_for_commit(
            repository, "abc123", "Build CCExtractor on Linux", log
        )

        self.assertIsNone(result)

    @mock.patch('mod_ci.controllers._get_workflow_path_filters')
    def test_will_workflow_run_matches_subdirectory(self, mock_get_filters):
        """Test that ** patterns match subdirectories."""
        from mod_ci.controllers import _will_workflow_run_for_commit

        repository = MagicMock()
        log = MagicMock()

        # Mock path filters with ** pattern
        mock_get_filters.return_value = ['src/rust/**']

        # Mock commit with a file in src/rust subdirectory
        mock_file = MagicMock()
        mock_file.filename = 'src/rust/lib_ccxr/src/main.rs'
        mock_commit = MagicMock()
        mock_commit.files = [mock_file]
        repository.get_commit.return_value = mock_commit

        result = _will_workflow_run_for_commit(
            repository, "abc123", "Build CCExtractor on Linux", log
        )

        self.assertTrue(result)


class TestStartTestRetryBehavior(BaseTestCase):
    """Test that start_test correctly handles retryable vs permanent failures."""

    @mock.patch('mod_ci.controllers.find_artifact_for_commit')
    @mock.patch('mod_ci.controllers._diagnose_missing_artifact')
    @mock.patch('mod_ci.controllers.mark_test_failed')
    @mock.patch('mod_ci.controllers.GcpInstance')
    @mock.patch('mod_ci.controllers.TestProgress')
    @mock.patch('run.log')
    def test_retryable_failure_does_not_mark_test_failed(
            self, mock_log, mock_progress, mock_gcp, mock_mark_failed,
            mock_diagnose, mock_find_artifact):
        """Test that retryable failures don't mark the test as failed."""
        from mod_ci.controllers import start_test

        # Setup: artifact not found, but build is in progress (retryable)
        mock_find_artifact.return_value = None
        mock_diagnose.return_value = ("Build still in progress", True)

        # Mock locking checks
        mock_gcp.query.filter.return_value.first.return_value = None
        mock_progress.query.filter.return_value.first.return_value = None

        test = Test.query.first()
        repository = MagicMock()

        start_test(MagicMock(), self.app, g.db, repository, test, "token")

        # Should NOT call mark_test_failed for retryable failure
        mock_mark_failed.assert_not_called()
        # Should log info, not critical
        mock_log.info.assert_called()

    @mock.patch('mod_ci.controllers.find_artifact_for_commit')
    @mock.patch('mod_ci.controllers._diagnose_missing_artifact')
    @mock.patch('mod_ci.controllers.mark_test_failed')
    @mock.patch('mod_ci.controllers.GcpInstance')
    @mock.patch('mod_ci.controllers.TestProgress')
    @mock.patch('run.log')
    def test_permanent_failure_marks_test_failed(
            self, mock_log, mock_progress, mock_gcp, mock_mark_failed,
            mock_diagnose, mock_find_artifact):
        """Test that permanent failures mark the test as failed."""
        from mod_ci.controllers import start_test

        # Setup: artifact not found, build failed (not retryable)
        mock_find_artifact.return_value = None
        mock_diagnose.return_value = ("Build failed with conclusion 'failure'", False)

        # Mock locking checks
        mock_gcp.query.filter.return_value.first.return_value = None
        mock_progress.query.filter.return_value.first.return_value = None

        test = Test.query.first()
        repository = MagicMock()

        start_test(MagicMock(), self.app, g.db, repository, test, "token")

        # SHOULD call mark_test_failed for permanent failure
        mock_mark_failed.assert_called_once()
        # Should log critical
        mock_log.critical.assert_called()


class TestPendingDeletionTracking(BaseTestCase):
    """Tests for VM deletion tracking and verification functionality."""

    @mock.patch('run.log')
    def test_check_operation_status_success(self, mock_log):
        """Test check_operation_status returns status correctly."""
        from mod_ci.controllers import check_operation_status

        compute = MagicMock()
        zone_ops = compute.zoneOperations.return_value.get.return_value
        zone_ops.execute.return_value = {'status': 'DONE'}

        result = check_operation_status(compute, "project", "zone", "op-123")

        self.assertEqual(result['status'], 'DONE')

    @mock.patch('run.log')
    def test_check_operation_status_error(self, mock_log):
        """Test check_operation_status handles API errors."""
        from mod_ci.controllers import check_operation_status

        compute = MagicMock()
        zone_ops = compute.zoneOperations.return_value.get.return_value
        zone_ops.execute.side_effect = Exception("API Error")

        result = check_operation_status(compute, "project", "zone", "op-123")

        self.assertEqual(result['status'], 'ERROR')
        self.assertIn('error', result)

    @mock.patch('mod_ci.controllers.safe_db_commit')
    @mock.patch('mod_ci.controllers.delete_instance')
    @mock.patch('run.log')
    def test_delete_instance_with_tracking_success(
            self, mock_log, mock_delete, mock_commit):
        """Test delete_instance_with_tracking records pending deletion."""
        from mod_ci.controllers import delete_instance_with_tracking
        from mod_ci.models import PendingDeletion

        mock_delete.return_value = {'name': 'op-123'}
        mock_commit.return_value = True

        db = MagicMock()
        result = delete_instance_with_tracking(
            MagicMock(), "project", "zone", "linux-42", db)

        self.assertEqual(result['name'], 'op-123')
        # Verify PendingDeletion was added
        db.add.assert_called_once()
        added_obj = db.add.call_args[0][0]
        self.assertIsInstance(added_obj, PendingDeletion)
        self.assertEqual(added_obj.vm_name, 'linux-42')
        self.assertEqual(added_obj.operation_name, 'op-123')

    @mock.patch('mod_ci.controllers.safe_db_commit')
    @mock.patch('mod_ci.controllers.delete_instance')
    @mock.patch('run.log')
    def test_delete_instance_with_tracking_failure(
            self, mock_log, mock_delete, mock_commit):
        """Test delete_instance_with_tracking handles failures."""
        from mod_ci.controllers import delete_instance_with_tracking
        from mod_ci.models import PendingDeletion

        mock_delete.side_effect = Exception("GCP Error")
        mock_commit.return_value = True

        db = MagicMock()
        with self.assertRaises(Exception):
            delete_instance_with_tracking(
                MagicMock(), "project", "zone", "linux-42", db)

        # Should still record the failed attempt
        db.add.assert_called_once()
        added_obj = db.add.call_args[0][0]
        self.assertIsInstance(added_obj, PendingDeletion)
        self.assertTrue(added_obj.operation_name.startswith('failed-'))

    @mock.patch('mod_ci.controllers.PendingDeletion')
    @mock.patch('mod_ci.controllers.check_operation_status')
    @mock.patch('mod_ci.controllers.safe_db_commit')
    @mock.patch('run.log')
    def test_verify_pending_deletions_success(
            self, mock_log, mock_commit, mock_check, mock_pending_class):
        """Test verify_pending_deletions removes successful deletions."""
        from mod_ci.controllers import verify_pending_deletions

        # Create mock pending deletion
        mock_pending = MagicMock()
        mock_pending.vm_name = 'linux-42'
        mock_pending.operation_name = 'op-123'
        mock_pending.retry_count = 0
        mock_pending_class.query.all.return_value = [mock_pending]

        # Operation completed successfully
        mock_check.return_value = {'status': 'DONE'}
        mock_commit.return_value = True

        db = MagicMock()
        verify_pending_deletions(MagicMock(), "project", "zone", db)

        # Should delete the pending record
        db.delete.assert_called_once_with(mock_pending)
        mock_commit.assert_called()

    @mock.patch('mod_ci.controllers.PendingDeletion')
    @mock.patch('mod_ci.controllers.check_operation_status')
    @mock.patch('mod_ci.controllers._retry_deletion')
    @mock.patch('run.log')
    def test_verify_pending_deletions_retries_on_error(
            self, mock_log, mock_retry, mock_check, mock_pending_class):
        """Test verify_pending_deletions retries failed deletions."""
        from mod_ci.controllers import verify_pending_deletions

        # Create mock pending deletion
        mock_pending = MagicMock()
        mock_pending.vm_name = 'linux-42'
        mock_pending.operation_name = 'op-123'
        mock_pending.retry_count = 0
        mock_pending_class.query.all.return_value = [mock_pending]

        # Operation failed
        mock_check.return_value = {
            'status': 'DONE',
            'error': {'errors': [{'message': 'Permission denied'}]}
        }

        db = MagicMock()
        verify_pending_deletions(MagicMock(), "project", "zone", db)

        # Should retry
        mock_retry.assert_called_once()

    @mock.patch('mod_ci.controllers.GcpInstance')
    @mock.patch('mod_ci.controllers.get_running_instances')
    @mock.patch('mod_ci.controllers.is_instance_testing')
    @mock.patch('mod_ci.controllers.delete_instance')
    @mock.patch('mod_ci.controllers.safe_db_commit')
    @mock.patch('run.log')
    def test_scan_for_orphaned_vms_finds_orphan(
            self, mock_log, mock_commit, mock_delete, mock_is_testing,
            mock_get_running, mock_gcp_class):
        """Test scan_for_orphaned_vms finds and deletes orphaned VMs."""
        from mod_ci.controllers import scan_for_orphaned_vms
        from mod_ci.models import PendingDeletion

        # Mock running VM
        mock_get_running.return_value = [{'name': 'linux-42'}]
        mock_is_testing.return_value = True

        # No GcpInstance record (orphaned)
        mock_gcp_class.query.filter.return_value.first.return_value = None
        # Not tracked in PendingDeletion - use real query
        with mock.patch.object(PendingDeletion, 'query') as mock_query:
            mock_query.filter.return_value.first.return_value = None

            mock_delete.return_value = {'name': 'op-orphan-42'}
            mock_commit.return_value = True

            db = MagicMock()
            scan_for_orphaned_vms(MagicMock(), "project", "zone", db)

            # Should delete the orphan
            mock_delete.assert_called_once()
            # Should track it
            db.add.assert_called_once()
            added_obj = db.add.call_args[0][0]
            self.assertIsInstance(added_obj, PendingDeletion)
            self.assertEqual(added_obj.vm_name, 'linux-42')

    @mock.patch('mod_ci.controllers.PendingDeletion')
    @mock.patch('mod_ci.controllers.GcpInstance')
    @mock.patch('mod_ci.controllers.get_running_instances')
    @mock.patch('mod_ci.controllers.is_instance_testing')
    @mock.patch('run.log')
    def test_scan_for_orphaned_vms_ignores_active(
            self, mock_log, mock_is_testing, mock_get_running,
            mock_gcp_class, mock_pending_class):
        """Test scan_for_orphaned_vms ignores VMs with GcpInstance records."""
        from mod_ci.controllers import scan_for_orphaned_vms

        # Mock running VM
        mock_get_running.return_value = [{'name': 'linux-42'}]
        mock_is_testing.return_value = True

        # Has GcpInstance record (active test)
        mock_gcp_class.query.filter.return_value.first.return_value = MagicMock()

        db = MagicMock()
        scan_for_orphaned_vms(MagicMock(), "project", "zone", db)

        # Should NOT track or delete
        db.add.assert_not_called()

    @mock.patch('mod_ci.controllers.delete_instance')
    @mock.patch('mod_ci.controllers.safe_db_commit')
    @mock.patch('run.log')
    def test_retry_deletion_success(self, mock_log, mock_commit, mock_delete):
        """Test _retry_deletion successfully retries."""
        from mod_ci.controllers import _retry_deletion
        from mod_ci.models import PendingDeletion

        mock_delete.return_value = {'name': 'op-retry-123'}
        mock_commit.return_value = True

        pending = PendingDeletion('linux-42', 'op-failed')
        pending.retry_count = 0

        db = MagicMock()
        _retry_deletion(MagicMock(), "project", "zone", "linux-42", pending, db, mock_log)

        mock_delete.assert_called_once()
        self.assertEqual(pending.retry_count, 1)
        self.assertEqual(pending.operation_name, 'op-retry-123')

    @mock.patch('mod_ci.controllers.delete_instance')
    @mock.patch('run.log')
    def test_retry_deletion_max_retries(self, mock_log, mock_delete):
        """Test _retry_deletion stops after max retries."""
        from mod_ci.controllers import _retry_deletion
        from mod_ci.models import PendingDeletion

        pending = PendingDeletion('linux-42', 'op-failed')
        pending.retry_count = PendingDeletion.MAX_RETRIES

        db = MagicMock()
        _retry_deletion(MagicMock(), "project", "zone", "linux-42", pending, db, mock_log)

        # Should NOT attempt delete
        mock_delete.assert_not_called()
        mock_log.error.assert_called()

    @mock.patch('mod_ci.controllers.delete_instance')
    @mock.patch('mod_ci.controllers.safe_db_commit')
    @mock.patch('run.log')
    def test_retry_deletion_vm_not_found(self, mock_log, mock_commit, mock_delete):
        """Test _retry_deletion handles 404 (VM already deleted)."""
        from mod_ci.controllers import _retry_deletion
        from mod_ci.models import PendingDeletion

        mock_delete.side_effect = Exception("notFound: VM not found")
        mock_commit.return_value = True

        pending = PendingDeletion('linux-42', 'op-failed')
        pending.retry_count = 0

        db = MagicMock()
        _retry_deletion(MagicMock(), "project", "zone", "linux-42", pending, db, mock_log)

        # Should remove the pending record (VM is gone)
        db.delete.assert_called_once_with(pending)


class TestVMCreationVerification(BaseTestCase):
    """Tests for VM creation verification to prevent stuck tests.

    These tests verify the fix for the issue where tests would get stuck forever
    when VM creation failed (e.g., QUOTA_EXCEEDED) but a GcpInstance record was
    still created in the database.

    Root cause investigated: Test #7768 for CCExtractor PR #2014 was stuck in
    "Preparation" phase for 12+ hours because:
    1. GCP VM creation failed with QUOTA_EXCEEDED (8 IP addresses limit)
    2. The platform created a gcp_instance DB record before verifying success
    3. The cron job saw the record and assumed the test was running
    4. The test was stuck forever with no way to recover

    The fix: Wait for the GCP operation to complete (or timeout) before creating
    the gcp_instance record. If the operation fails, mark the test as failed.
    """

    def _setup_start_test_mocks(self, mock_g, mock_gcp_instance, mock_test_progress,
                                mock_find_artifact, mock_requests_get, mock_zipfile):
        """Set up common mocks for start_test VM creation verification tests."""
        # Mock locking checks to return None (no existing instances/progress)
        mock_gcp_instance.query.filter.return_value.first.return_value = None
        mock_test_progress.query.filter.return_value.first.return_value = None
        mock_query = create_mock_db_query(mock_g)
        mock_query.c.got = MagicMock()

        # Mock successful artifact download
        mock_artifact = MagicMock()
        mock_artifact.name = 'CCExtractor Linux build'
        mock_artifact.archive_download_url = 'https://example.com/artifact.zip'
        mock_find_artifact.return_value = mock_artifact

        mock_response = MagicMock()
        mock_response.status_code = 200
        mock_response.content = b'fake zip content'
        mock_requests_get.return_value = mock_response

        mock_zip = MagicMock()
        mock_zipfile.return_value.__enter__ = MagicMock(return_value=mock_zip)
        mock_zipfile.return_value.__exit__ = MagicMock(return_value=False)

    @mock.patch('run.log')
    @mock.patch('mod_ci.controllers.mark_test_failed')
    @mock.patch('mod_ci.controllers.wait_for_operation')
    @mock.patch('mod_ci.controllers.create_instance')
    @mock.patch('mod_ci.controllers.find_artifact_for_commit')
    @mock.patch('mod_ci.controllers.requests.get')
    @mock.patch('mod_ci.controllers.zipfile.ZipFile')
    @mock.patch('builtins.open', new_callable=mock.mock_open())
    @mock.patch('mod_ci.controllers.g')
    @mock.patch('mod_ci.controllers.TestProgress')
    @mock.patch('mod_ci.controllers.GcpInstance')
    def test_start_test_quota_exceeded_retries_instead_of_failing(
            self, mock_gcp_instance, mock_test_progress, mock_g, mock_open_file,
            mock_zipfile, mock_requests_get, mock_find_artifact,
            mock_create_instance, mock_wait_for_operation, mock_mark_failed, mock_log):
        """Test that QUOTA_EXCEEDED error leaves test pending for retry.

        QUOTA_EXCEEDED is a transient error - the test should remain pending
        and be retried on the next cron run, not marked as failed.
        """
        from mod_ci.controllers import start_test

        self._setup_start_test_mocks(mock_g, mock_gcp_instance, mock_test_progress,
                                     mock_find_artifact, mock_requests_get, mock_zipfile)

        # VM creation returns operation ID, but wait_for_operation returns error
        mock_create_instance.return_value = {'name': 'op-123', 'status': 'RUNNING'}
        mock_wait_for_operation.return_value = {
            'status': 'DONE',
            'error': {'errors': [{'code': 'QUOTA_EXCEEDED', 'message': 'Quota exceeded'}]},
            'httpErrorStatusCode': 403
        }

        start_test(MagicMock(), self.app, mock_g.db, MagicMock(), Test.query.first(), "token")

        # QUOTA_EXCEEDED is retryable, so mark_test_failed should NOT be called
        mock_mark_failed.assert_not_called()
        # No GcpInstance record should be created
        mock_g.db.add.assert_not_called()

    @mock.patch('run.log')
    @mock.patch('mod_ci.controllers.mark_test_failed')
    @mock.patch('mod_ci.controllers.wait_for_operation')
    @mock.patch('mod_ci.controllers.create_instance')
    @mock.patch('mod_ci.controllers.find_artifact_for_commit')
    @mock.patch('mod_ci.controllers.requests.get')
    @mock.patch('mod_ci.controllers.zipfile.ZipFile')
    @mock.patch('builtins.open', new_callable=mock.mock_open())
    @mock.patch('mod_ci.controllers.g')
    @mock.patch('mod_ci.controllers.TestProgress')
    @mock.patch('mod_ci.controllers.GcpInstance')
    def test_start_test_non_retryable_error_marks_failed(
            self, mock_gcp_instance, mock_test_progress, mock_g, mock_open_file,
            mock_zipfile, mock_requests_get, mock_find_artifact,
            mock_create_instance, mock_wait_for_operation, mock_mark_failed, mock_log):
        """Test that non-retryable errors (like RESOURCE_NOT_FOUND) mark test as failed.

        Only transient errors like QUOTA_EXCEEDED should be retried. Permanent
        errors like RESOURCE_NOT_FOUND should immediately mark the test as failed.
        """
        from mod_ci.controllers import start_test

        self._setup_start_test_mocks(mock_g, mock_gcp_instance, mock_test_progress,
                                     mock_find_artifact, mock_requests_get, mock_zipfile)

        # VM creation returns operation ID, but wait_for_operation returns non-retryable error
        mock_create_instance.return_value = {'name': 'op-123', 'status': 'RUNNING'}
        mock_wait_for_operation.return_value = {
            'status': 'DONE',
            'error': {'errors': [{'code': 'RESOURCE_NOT_FOUND', 'message': 'Resource not found'}]},
            'httpErrorStatusCode': 404
        }

        start_test(MagicMock(), self.app, mock_g.db, MagicMock(), Test.query.first(), "token")

        # RESOURCE_NOT_FOUND is NOT retryable, so mark_test_failed SHOULD be called
        mock_mark_failed.assert_called_once()
        # No GcpInstance record should be created
        mock_g.db.add.assert_not_called()

    @mock.patch('run.log')
    @mock.patch('mod_ci.controllers.safe_db_commit')
    @mock.patch('mod_ci.controllers.wait_for_operation')
    @mock.patch('mod_ci.controllers.create_instance')
    @mock.patch('mod_ci.controllers.find_artifact_for_commit')
    @mock.patch('mod_ci.controllers.requests.get')
    @mock.patch('mod_ci.controllers.zipfile.ZipFile')
    @mock.patch('builtins.open', new_callable=mock.mock_open())
    @mock.patch('mod_ci.controllers.g')
    @mock.patch('mod_ci.controllers.TestProgress')
    @mock.patch('mod_ci.controllers.GcpInstance')
    def test_start_test_vm_verified_creates_db_record(
            self, mock_gcp_instance, mock_test_progress, mock_g, mock_open_file,
            mock_zipfile, mock_requests_get, mock_find_artifact,
            mock_create_instance, mock_wait_for_operation, mock_commit, mock_log):
        """Test that successful VM creation creates gcp_instance record."""
        from mod_ci.controllers import start_test

        self._setup_start_test_mocks(mock_g, mock_gcp_instance, mock_test_progress,
                                     mock_find_artifact, mock_requests_get, mock_zipfile)

        mock_create_instance.return_value = {'name': 'op-123', 'status': 'RUNNING'}
        mock_wait_for_operation.return_value = {'status': 'DONE'}
        mock_commit.return_value = True

        start_test(MagicMock(), self.app, mock_g.db, MagicMock(), Test.query.first(), "token")

        mock_g.db.add.assert_called_once()

    @mock.patch('run.log')
    @mock.patch('mod_ci.controllers.safe_db_commit')
    @mock.patch('mod_ci.controllers.wait_for_operation')
    @mock.patch('mod_ci.controllers.create_instance')
    @mock.patch('mod_ci.controllers.find_artifact_for_commit')
    @mock.patch('mod_ci.controllers.requests.get')
    @mock.patch('mod_ci.controllers.zipfile.ZipFile')
    @mock.patch('builtins.open', new_callable=mock.mock_open())
    @mock.patch('mod_ci.controllers.g')
    @mock.patch('mod_ci.controllers.TestProgress')
    @mock.patch('mod_ci.controllers.GcpInstance')
    def test_start_test_operation_timeout_creates_db_record(
            self, mock_gcp_instance, mock_test_progress, mock_g, mock_open_file,
            mock_zipfile, mock_requests_get, mock_find_artifact,
            mock_create_instance, mock_wait_for_operation, mock_commit, mock_log):
        """Test that operation timeout still creates record (VM may be starting slowly)."""
        from mod_ci.controllers import start_test

        self._setup_start_test_mocks(mock_g, mock_gcp_instance, mock_test_progress,
                                     mock_find_artifact, mock_requests_get, mock_zipfile)

        mock_create_instance.return_value = {'name': 'op-123', 'status': 'RUNNING'}
        mock_wait_for_operation.return_value = {'status': 'TIMEOUT'}
        mock_commit.return_value = True

        start_test(MagicMock(), self.app, mock_g.db, MagicMock(), Test.query.first(), "token")

        mock_g.db.add.assert_called_once()