Bug-2038705: Remove usage of fetch_api and fetch_api_full_url

treeherder/etl/management/commands/ingest.py

@@ -22,7 +22,7 @@
 from treeherder.etl.taskcluster_pulse.handler import EXCHANGE_EVENT_MAP, handle_message
 from treeherder.model.models import Repository
 from treeherder.utils import github
-from treeherder.utils.github import fetch_api, fetch_api_full_url
+from treeherder.utils.github import compare_shas, get_commit
 
 logger = logging.getLogger(__name__)
 logger.setLevel(logging.INFO)
@@ -276,29 +276,42 @@ def query_data(repo_meta, commit):
     event_base_sha = repo_meta["branch"]
     # First we try with `master` being the base sha
     # e.g. https://api.github.com/repos/servo/servo/compare/master...1418c0555ff77e5a3d6cf0c6020ba92ece36be2e
-    compare_response = fetch_api(
-        f"repos/{repo_meta['owner']}/{repo_meta['repo']}/compare/{event_base_sha}...{commit}"
+    # compare_response = fetch_api(
+    #     f"repos/{repo_meta['owner']}/{repo_meta['repo']}/compare/{event_base_sha}...{commit}"
+    # )
+    compare_response = compare_shas(
+        repo_meta["owner"], repo_meta["repo"], event_base_sha, commit, get_comparison_object=True
     )
-    merge_base_commit = compare_response.get("merge_base_commit")
+    merge_base_commit = (
+        compare_response.merge_base_commit
+    )  # compare_response.get("merge_base_commit")
     if merge_base_commit:
-        commiter_date = merge_base_commit["commit"]["committer"]["date"]
+        commiter_date = (
+            merge_base_commit.commit.committer.date
+        )  # merge_base_commit["commit"]["committer"]["date"]
         # Since we don't use PushEvents that contain the "before" or "event.base.sha" fields [1]
         # we need to discover the right parent which existed in the base branch.
         # [1] https://github.com/taskcluster/taskcluster/blob/3dda0adf85619d18c5dcf255259f3e274d2be346/services/github/src/api.js#L55
-        parents = compare_response["merge_base_commit"]["parents"]
+        parents = (
+            compare_response.merge_base_commit.parents
+        )  # compare_response["merge_base_commit"]["parents"]
         if len(parents) == 1:
             parent = parents[0]
-            commit_info = fetch_api_full_url(parent["url"])
+            commit_info = get_commit(
+                repo_meta["owner"], repo_meta["repo"], parent.sha
+            )  # fetch_api_full_url(parent["url"])
             committer_date = commit_info["commit"]["committer"]["date"]
             # All commits involved in a PR share the same committer's date
-            if merge_base_commit["commit"]["committer"]["date"] == committer_date:
+            if merge_base_commit.commit.committer.date == committer_date:
                 # Recursively find the forking parent
-                event_base_sha, _ = query_data(repo_meta, parent["sha"])
+                event_base_sha, _ = query_data(repo_meta, parent.sha)
             else:
-                event_base_sha = parent["sha"]
+                event_base_sha = parent.sha
         else:
             for parent in parents:
-                _commit = fetch_api_full_url(parent["url"])
+                _commit = get_commit(
+                    repo_meta["owner"], repo_meta["repo"], parent.sha
+                )  # fetch_api_full_url(parent["url"])
                 # All commits involved in a merge share the same committer's date
                 if commiter_date != _commit["commit"]["committer"]["date"]:
                     event_base_sha = _commit["sha"]
@@ -307,12 +320,16 @@ def query_data(repo_meta, commit):
         assert event_base_sha != repo_meta["branch"]
         logger.info("We have a new base: %s", event_base_sha)
         # When using the correct event_base_sha the "commits" field will be correct
-        compare_response = fetch_api(
-            f"repos/{repo_meta['owner']}/{repo_meta['repo']}/compare/{event_base_sha}...{commit}"
-        )
+        compare_response = compare_shas(
+            repo_meta["owner"],
+            repo_meta["repo"],
+            event_base_sha,
+            commit,
+            get_comparison_object=True,
+        )  # fetch_api(f"repos/{repo_meta['owner']}/{repo_meta['repo']}/compare/{event_base_sha}...{commit}")
 
     commits = []
-    for _commit in compare_response["commits"]:
+    for _commit in compare_response.commits:
         commits.append(
             {
                 "message": _commit["commit"]["message"],

treeherder/etl/push_loader.py

@@ -103,8 +103,17 @@ def get_info(self):
         return info
 
     def process_push(self, push_data):
-        commits = push_data
-        head_commit = commits[-1]
+        revisions = []
+        head_commit = None
+        for commit in push_data:
+            revisions.append(
+                {
+                    "comment": commit.commit.message,
+                    "author": f"{commit.commit.author.name} <{commit.commit.author.email}>",
+                    "revision": commit.sha,
+                }
+            )
+            head_commit = commit
         push = {
             "revision": head_commit.sha,
             # A push can be co-authored
@@ -113,19 +122,8 @@ def process_push(self, push_data):
             "push_timestamp": head_commit.commit.committer.date.timestamp(),
             # We want the original author's email to show up in the UI
             "author": head_commit.commit.author.email,
+            "revisions": revisions,
         }
-
-        revisions = []
-        for commit in commits:
-            revisions.append(
-                {
-                    "comment": commit.commit.message,
-                    "author": f"{commit.commit.author.name} <{commit.commit.author.email}>",
-                    "revision": commit.sha,
-                }
-            )
-
-        push["revisions"] = revisions
         return push
 
 

treeherder/utils/github.py

@@ -78,10 +78,12 @@ def get_releases(owner, repo, params=None):
     return releases
 
 
-def compare_shas(owner, repo, base, head):
+def compare_shas(owner, repo, base, head, get_comparison_object=False):
     repo = pygithub_get_repo(owner, repo)
     comparison = repo.compare(base, head)
-    return [commit for commit in comparison.commits]
+    if get_comparison_object:
+        return comparison
+    return (commit for commit in comparison.commits)
 
 
 def get_all_commits(owner, repo, params=None):
@@ -119,4 +121,4 @@ def get_pull_request(owner, repo, pr_id):
 
 def get_pull_request_commits(owner, repo, pr_id):
     pr = get_pull_request(owner, repo, pr_id)
-    return [commit for commit in pr.get_commits()]
+    return (commit for commit in pr.get_commits())