diff --git a/treeherder/etl/management/commands/ingest.py b/treeherder/etl/management/commands/ingest.py index 3da21a06c20..e3f32208a11 100644 --- a/treeherder/etl/management/commands/ingest.py +++ b/treeherder/etl/management/commands/ingest.py @@ -22,7 +22,7 @@ from treeherder.etl.taskcluster_pulse.handler import EXCHANGE_EVENT_MAP, handle_message from treeherder.model.models import Repository from treeherder.utils import github -from treeherder.utils.github import fetch_api, fetch_api_full_url +from treeherder.utils.github import compare_shas, get_commit logger = logging.getLogger(__name__) logger.setLevel(logging.INFO) @@ -276,29 +276,42 @@ def query_data(repo_meta, commit): event_base_sha = repo_meta["branch"] # First we try with `master` being the base sha # e.g. https://api.github.com/repos/servo/servo/compare/master...1418c0555ff77e5a3d6cf0c6020ba92ece36be2e - compare_response = fetch_api( - f"repos/{repo_meta['owner']}/{repo_meta['repo']}/compare/{event_base_sha}...{commit}" + # compare_response = fetch_api( + # f"repos/{repo_meta['owner']}/{repo_meta['repo']}/compare/{event_base_sha}...{commit}" + # ) + compare_response = compare_shas( + repo_meta["owner"], repo_meta["repo"], event_base_sha, commit, get_comparison_object=True ) - merge_base_commit = compare_response.get("merge_base_commit") + merge_base_commit = ( + compare_response.merge_base_commit + ) # compare_response.get("merge_base_commit") if merge_base_commit: - commiter_date = merge_base_commit["commit"]["committer"]["date"] + commiter_date = ( + merge_base_commit.commit.committer.date + ) # merge_base_commit["commit"]["committer"]["date"] # Since we don't use PushEvents that contain the "before" or "event.base.sha" fields [1] # we need to discover the right parent which existed in the base branch. # [1] https://github.com/taskcluster/taskcluster/blob/3dda0adf85619d18c5dcf255259f3e274d2be346/services/github/src/api.js#L55 - parents = compare_response["merge_base_commit"]["parents"] + parents = ( + compare_response.merge_base_commit.parents + ) # compare_response["merge_base_commit"]["parents"] if len(parents) == 1: parent = parents[0] - commit_info = fetch_api_full_url(parent["url"]) + commit_info = get_commit( + repo_meta["owner"], repo_meta["repo"], parent.sha + ) # fetch_api_full_url(parent["url"]) committer_date = commit_info["commit"]["committer"]["date"] # All commits involved in a PR share the same committer's date - if merge_base_commit["commit"]["committer"]["date"] == committer_date: + if merge_base_commit.commit.committer.date == committer_date: # Recursively find the forking parent - event_base_sha, _ = query_data(repo_meta, parent["sha"]) + event_base_sha, _ = query_data(repo_meta, parent.sha) else: - event_base_sha = parent["sha"] + event_base_sha = parent.sha else: for parent in parents: - _commit = fetch_api_full_url(parent["url"]) + _commit = get_commit( + repo_meta["owner"], repo_meta["repo"], parent.sha + ) # fetch_api_full_url(parent["url"]) # All commits involved in a merge share the same committer's date if commiter_date != _commit["commit"]["committer"]["date"]: event_base_sha = _commit["sha"] @@ -307,12 +320,16 @@ def query_data(repo_meta, commit): assert event_base_sha != repo_meta["branch"] logger.info("We have a new base: %s", event_base_sha) # When using the correct event_base_sha the "commits" field will be correct - compare_response = fetch_api( - f"repos/{repo_meta['owner']}/{repo_meta['repo']}/compare/{event_base_sha}...{commit}" - ) + compare_response = compare_shas( + repo_meta["owner"], + repo_meta["repo"], + event_base_sha, + commit, + get_comparison_object=True, + ) # fetch_api(f"repos/{repo_meta['owner']}/{repo_meta['repo']}/compare/{event_base_sha}...{commit}") commits = [] - for _commit in compare_response["commits"]: + for _commit in compare_response.commits: commits.append( { "message": _commit["commit"]["message"], diff --git a/treeherder/etl/push_loader.py b/treeherder/etl/push_loader.py index d170ce722b2..a88371bf6c2 100644 --- a/treeherder/etl/push_loader.py +++ b/treeherder/etl/push_loader.py @@ -103,8 +103,17 @@ def get_info(self): return info def process_push(self, push_data): - commits = push_data - head_commit = commits[-1] + revisions = [] + head_commit = None + for commit in push_data: + revisions.append( + { + "comment": commit.commit.message, + "author": f"{commit.commit.author.name} <{commit.commit.author.email}>", + "revision": commit.sha, + } + ) + head_commit = commit push = { "revision": head_commit.sha, # A push can be co-authored @@ -113,19 +122,8 @@ def process_push(self, push_data): "push_timestamp": head_commit.commit.committer.date.timestamp(), # We want the original author's email to show up in the UI "author": head_commit.commit.author.email, + "revisions": revisions, } - - revisions = [] - for commit in commits: - revisions.append( - { - "comment": commit.commit.message, - "author": f"{commit.commit.author.name} <{commit.commit.author.email}>", - "revision": commit.sha, - } - ) - - push["revisions"] = revisions return push diff --git a/treeherder/utils/github.py b/treeherder/utils/github.py index 4ab9ec9878c..43a950263fd 100644 --- a/treeherder/utils/github.py +++ b/treeherder/utils/github.py @@ -78,10 +78,12 @@ def get_releases(owner, repo, params=None): return releases -def compare_shas(owner, repo, base, head): +def compare_shas(owner, repo, base, head, get_comparison_object=False): repo = pygithub_get_repo(owner, repo) comparison = repo.compare(base, head) - return [commit for commit in comparison.commits] + if get_comparison_object: + return comparison + return (commit for commit in comparison.commits) def get_all_commits(owner, repo, params=None): @@ -119,4 +121,4 @@ def get_pull_request(owner, repo, pr_id): def get_pull_request_commits(owner, repo, pr_id): pr = get_pull_request(owner, repo, pr_id) - return [commit for commit in pr.get_commits()] + return (commit for commit in pr.get_commits())