Request Github token from Taskcluster's auth service

pyproject.toml

@@ -25,7 +25,7 @@ dependencies = [
     "jsonschema[format-nongpl]",
     "json-e>=2.5.0",
     "PyYAML",
-    "taskcluster>=40",
+    "taskcluster>=106",
     "taskcluster-taskgraph",
 ]
 

scriptworker.yaml.tmpl

@@ -45,9 +45,15 @@ verify_cot_signature: false
 # Chain of Trust job type, e.g. signing
 cot_job_type: scriptworker
 cot_product: firefox
-# Calls to Github API are limited to 60 an hour. Using an API token allows to raise the limit to
-# 5000 per hour. https://developer.github.com/v3/#rate-limiting
-github_oauth_token: somegithubtoken
+
+# Scriptworker first tries to obtain a repository scoped token from Taskcluster's auth service,
+# using the Github App registered under this name.
+# github_app_name: read
+#
+# `github_oauth_token` is used as a fallback if that fails (e.g. missing scopes or app not
+# configured). Without either, calls to the Github API are unauthenticated and limited to 60 an
+# hour. See https://developer.github.com/v3/#rate-limiting
+# github_oauth_token: somegithubtoken
 
 
 #-----------------------------------------------------------------------------------------------

src/scriptworker/constants.py

@@ -79,7 +79,12 @@
         "max_chain_length": 20,
         # Calls to Github API are limited to 60 an hour. Using an API token allows to raise the limit to
         # 5000 per hour. https://developer.github.com/v3/#rate-limiting
+        # Scriptworker first tries to obtain a repository scoped token from Taskcluster's auth
+        # service, falling back to this token if that fails.
         "github_oauth_token": "",
+        # The name of the Github App registered with Taskcluster's auth service, used to obtain
+        # a repository scoped token via `auth.githubRepoToken`.
+        "github_app_name": "read",
         # ed25519 settings
         "ed25519_private_key_path": "...",
         "ed25519_public_keys": immutabledict(

src/scriptworker/cot/verify.py

@@ -20,7 +20,6 @@
 from copy import deepcopy
 from urllib.parse import urlparse
 
-import aiohttp
 import dictdiffer
 import jsone
 from immutabledict import immutabledict
@@ -40,7 +39,7 @@
 from scriptworker.context import Context
 from scriptworker.ed25519 import ed25519_public_key_from_string, verify_ed25519_signature
 from scriptworker.exceptions import BaseDownloadError, CoTError, ScriptWorkerEd25519Error
-from scriptworker.github import GitHubRepository, extract_github_repo_full_name, extract_github_repo_owner_and_name, extract_github_repo_ssh_url
+from scriptworker.github import GitHubRepository, extract_github_repo_full_name, extract_github_repo_owner_and_name, extract_github_repo_ssh_url, is_github_url
 from scriptworker.log import contextual_log_handler, get_chain_of_trust_log_filename
 from scriptworker.task import (
     get_action_callback_name,
@@ -1118,7 +1117,7 @@ async def _get_additional_github_releases_jsone_context(decision_link):
     repo_owner, repo_name = extract_github_repo_owner_and_name(repo_url)
     tag_name = get_revision(task, source_env_prefix)
 
-    github_repo = GitHubRepository(repo_owner, repo_name, context.config["github_oauth_token"])
+    github_repo = GitHubRepository(context, repo_owner, repo_name)
     release_data = await github_repo.get_release(tag_name)
 
     # The release data expose by the API[1] is not the same as the original event[2]. That's why
@@ -1200,17 +1199,16 @@ async def _get_additional_github_pull_request_jsone_context(decision_link):
     repo_url = repo_url.replace("git@github.com:", "ssh://github.com/", 1)
     repo_owner, repo_name = extract_github_repo_owner_and_name(repo_url)
     pull_request_number = get_pull_request_number(task, source_env_prefix)
-    token = context.config["github_oauth_token"]
 
-    github_repo = GitHubRepository(repo_owner, repo_name, token)
-    repo_definition = github_repo.definition
+    github_repo = GitHubRepository(context, repo_owner, repo_name)
+    repo_definition = await github_repo.get_definition()
 
     # We need to query the repository where the pull request was made to extract
     # pull request data. The pull request could be created on the same repo as
     # the commit, or an upstream repo. We can compare the base and head repo URLs
     # to infer where the pull request lives.
     if repo_definition["fork"] and base_repo_url != repo_url:
-        github_repo = GitHubRepository(owner=repo_definition["parent"]["owner"]["login"], repo_name=repo_definition["parent"]["name"], token=token)
+        github_repo = GitHubRepository(context, repo_definition["parent"]["owner"]["login"], repo_definition["parent"]["name"])
 
     pull_request_data = await github_repo.get_pull_request(pull_request_number)
     # Even though pull_request_data['head']['repo']['pushed_at'] does exist,
@@ -1245,7 +1243,7 @@ async def _get_additional_github_push_jsone_context(decision_link):
     repo_owner, repo_name = extract_github_repo_owner_and_name(repo_url)
     commit_hash = get_revision(task, source_env_prefix)
 
-    github_repo = GitHubRepository(repo_owner, repo_name, context.config["github_oauth_token"])
+    github_repo = GitHubRepository(context, repo_owner, repo_name)
     commit_data = await github_repo.get_commit(commit_hash)
 
     committer = commit_data["committer"] or {}
@@ -1399,16 +1397,11 @@ def build_taskcluster_yml_url(link):
     """
     source_env_prefix = link.context.config["source_env_prefix"]
     repo_url = get_repo(link.task, source_env_prefix)
-    repo_url = repo_url.replace("git@github.com:", "ssh://github.com/", 1)
     revision = get_revision(link.task, source_env_prefix)
     repo_parts = urlparse(repo_url)
-    if repo_parts.netloc == "github.com":
-        user, repo_name = extract_github_repo_owner_and_name(repo_url)
-        url = f"https://raw.githubusercontent.com/{user}/{repo_name}/{revision}/.taskcluster.yml"
-    elif repo_parts.netloc == "hg.mozilla.org":
-        url = f"{repo_parts.scheme}://{repo_parts.netloc}{repo_parts.path}/raw-file/{revision}/.taskcluster.yml"
-    else:
+    if repo_parts.netloc != "hg.mozilla.org":
         raise CoTError("Unsupported VCS server!")
+    url = f"{repo_parts.scheme}://{repo_parts.netloc}{repo_parts.path}/raw-file/{revision}/.taskcluster.yml"
     log.debug(f"{link.name} .taskcluster.yml is at {url}")
     return url
 
@@ -1430,19 +1423,20 @@ async def get_in_tree_template(link):
 
     """
     context = link.context
-    source_url = build_taskcluster_yml_url(link)
-    repo_url = get_repo(link.task, link.context.config["source_env_prefix"])
+    source_env_prefix = context.config["source_env_prefix"]
+    repo_url = get_repo(link.task, source_env_prefix)
+    repo_url = repo_url.replace("git@github.com:", "ssh://github.com/", 1)
 
-    auth = None
-    if (
-        (repo_url.startswith(("ssh://", "git@github.com")) or any(vcs_rule.get("require_secret") for vcs_rule in context.config["trusted_vcs_rules"]))
-        and "github.com" in repo_url
-        and context.config.get("github_oauth_token")
-    ):
-        auth = aiohttp.BasicAuth(context.config["github_oauth_token"])
+    if is_github_url(repo_url):
+        revision = get_revision(link.task, source_env_prefix)
+        repo_owner, repo_name = extract_github_repo_owner_and_name(repo_url)
+        github_repo = GitHubRepository(context, repo_owner, repo_name)
+        content = await github_repo.get_file_contents(".taskcluster.yml", ref=revision)
+        return load_json_or_yaml(content, file_type="yaml")
+
+    source_url = build_taskcluster_yml_url(link)
     url_hash = hashlib.sha1(source_url.encode("ascii")).hexdigest()
-    tmpl = await load_json_or_yaml_from_url(context, source_url, os.path.join(context.config["work_dir"], "{}_taskcluster.yml".format(url_hash)), auth=auth)
-    return tmpl
+    return await load_json_or_yaml_from_url(context, source_url, os.path.join(context.config["work_dir"], "{}_taskcluster.yml".format(url_hash)))
 
 
 def _get_action_from_actions_json(all_actions, callback_name):

src/scriptworker/github.py

@@ -6,6 +6,8 @@
 
 from github3 import GitHub
 from github3.exceptions import GitHubException
+from taskcluster.aio import Auth
+from taskcluster.exceptions import TaskclusterFailure
 
 from scriptworker.exceptions import ConfigError
 from scriptworker.utils import get_parts_of_url_path, get_single_item_from_sequence, retry_async_decorator, retry_request, retry_sync
@@ -23,30 +25,96 @@
 class GitHubRepository:
     """Wrapper around GitHub API. Used to access public data."""
 
-    def __init__(self, owner, repo_name, token=""):
-        """Build the GitHub API URL which points to the definition of the repository.
+    GITHUB_PERMISSIONS = {"contents": "read", "metadata": "read", "pull_requests": "read"}
+
+    def __init__(self, context, owner, repo_name):
+        """Store the repository coordinates. The github3 repository object is built lazily.
 
         Args:
-            owner (str): the owner's GitHub username
+            context (scriptworker.context.Context): the scriptworker context
+            owner (str): the owner of the repository
             repo_name (str): the name of the repository
-            token (str): the GitHub API token
+
+        """
+        self._context = context
+        self._owner = owner
+        self._repo_name = repo_name
+        self._repository_cache = None
+        self._repository_lock = asyncio.Lock()
+
+    async def _get_repository(self):
+        """Build and cache the github3 repository object.
 
         Returns:
-            dict: a representation of the repo definition
+            github3.repos.repo.Repository: the github3 repository object
 
         """
-        github = retry_sync(GitHub, kwargs={"token": token}, sleeptime_kwargs=_GITHUB_LIBRARY_SLEEP_TIME_KWARGS)
-        self._github_repository = retry_sync(github.repository, args=(owner, repo_name), sleeptime_kwargs=_GITHUB_LIBRARY_SLEEP_TIME_KWARGS)
+        async with self._repository_lock:
+            if self._repository_cache is None:
+                token = await self._get_token(self._context, self._owner, self._repo_name)
+                github = retry_sync(GitHub, kwargs={"token": token}, sleeptime_kwargs=_GITHUB_LIBRARY_SLEEP_TIME_KWARGS)
+                self._repository_cache = retry_sync(github.repository, args=(self._owner, self._repo_name), sleeptime_kwargs=_GITHUB_LIBRARY_SLEEP_TIME_KWARGS)
+
+        return self._repository_cache
+
+    async def _get_token(self, context, owner, repo_name):
+        """Get a repository scoped GitHub token from Taskcluster's auth service.
+
+        Falls back to ``context.config["github_oauth_token"]`` if the auth service call
+        fails, e.g. because of missing scopes.
+
+        Args:
+            context (scriptworker.context.Context): the scriptworker context
+            owner (str): the owner of the repository
+            repo_name (str): the name of the repository
 
-    @property
-    def definition(self):
+        Returns:
+            str: the scoped GitHub token, or the fallback token
+
+        """
+        if not context.credentials:
+            return context.config.get("github_oauth_token", "")
+
+        try:
+            auth = Auth(options={"rootUrl": context.config["taskcluster_root_url"], "credentials": context.credentials})
+            response = await auth.githubRepoToken(
+                context.config["github_app_name"], owner, payload={"repositories": [repo_name], "permissions": self.GITHUB_PERMISSIONS}
+            )
+            return response["token"]
+        except TaskclusterFailure as e:
+            # TODO When opening a PR from a fork, we're guaranteed to hit this
+            # fallback as the task won't have auth service scopes for the repo
+            # fork. We'll need to improve this before we can stop depending on
+            # `github_oauth_token`.
+            log.warning(f"Could not obtain Github token from Taskcluster for {owner}/{repo_name}, falling back to `github_oauth_token`: {e}")
+            return context.config.get("github_oauth_token", "")
+
+    async def get_definition(self):
         """Fetch the definition of the repository, exposed by the GitHub API.
 
         Returns:
             dict: a representation of the repo definition
 
         """
-        return self._github_repository.as_dict()
+        repository = await self._get_repository()
+        return repository.as_dict()
+
+    @retry_async_decorator(retry_exceptions=GitHubException)
+    async def get_file_contents(self, path, ref=None):
+        """Fetch the decoded contents of a file in the repository.
+
+        Args:
+            path (str): the path to the file, relative to the repository root
+            ref (str, optional): the commit/branch/tag to read the file from.
+                Defaults to the repository's default branch.
+
+        Returns:
+            str: the decoded contents of the file
+
+        """
+        repository = await self._get_repository()
+        contents = repository.file_contents(path, ref=ref)
+        return contents.decoded.decode("utf-8")
 
     @retry_async_decorator(retry_exceptions=GitHubException)
     async def get_commit(self, commit_hash):
@@ -59,7 +127,8 @@ async def get_commit(self, commit_hash):
             dict: a representation of the commit
 
         """
-        return self._github_repository.commit(commit_hash).as_dict()
+        repository = await self._get_repository()
+        return repository.commit(commit_hash).as_dict()
 
     @retry_async_decorator(retry_exceptions=GitHubException)
     async def get_pull_request(self, pull_request_number):
@@ -72,7 +141,8 @@ async def get_pull_request(self, pull_request_number):
             dict: a representation of the pull request
 
         """
-        return self._github_repository.pull_request(pull_request_number).as_dict()
+        repository = await self._get_repository()
+        return repository.pull_request(pull_request_number).as_dict()
 
     @retry_async_decorator(retry_exceptions=GitHubException)
     async def get_release(self, tag_name):
@@ -85,7 +155,8 @@ async def get_release(self, tag_name):
             dict: a representation of the tag
 
         """
-        return self._github_repository.release_from_tag(tag_name).as_dict()
+        repository = await self._get_repository()
+        return repository.release_from_tag(tag_name).as_dict()
 
     @retry_async_decorator(retry_exceptions=GitHubException)
     async def get_tag_hash(self, tag_name):
@@ -98,8 +169,9 @@ async def get_tag_hash(self, tag_name):
             str: the commit hash linked by the tag
 
         """
+        repository = await self._get_repository()
         tag_object = get_single_item_from_sequence(
-            sequence=self._github_repository.tags(),
+            sequence=repository.tags(),
             condition=lambda tag: tag.name == tag_name,
             no_item_error_message='No tag "{}" exist'.format(tag_name),
             too_many_item_error_message='Too many tags "{}" found'.format(tag_name),
@@ -128,7 +200,8 @@ async def has_commit_landed_on_repository(self, context, revision):
         if not _is_git_full_hash(revision):
             revision = await self.get_tag_hash(tag_name=revision)
 
-        html_text = await _fetch_github_branch_commits_data(context, self._github_repository.html_url, revision)
+        repository = await self._get_repository()
+        html_text = await _fetch_github_branch_commits_data(context, repository.html_url, revision)
 
         # https://github.com/{repo_owner}/{repo_name}/branch_commits/{revision} just returns some \n
         # when the commit hasn't landed on the origin repo. Otherwise, some HTML data is returned - it

src/scriptworker/task.py

@@ -550,7 +550,7 @@ async def is_pull_request(context, task):
         if not revision and can_skip:
             continue
 
-        github_repository = GitHubRepository(repo_owner, repo_name, context.config["github_oauth_token"])
+        github_repository = GitHubRepository(context, repo_owner, repo_name)
         conditions.append(not await github_repository.has_commit_landed_on_repository(context, revision))
 
     return any(conditions)