diff --git a/.gitallowed b/.gitallowed new file mode 100644 index 000000000000..2a8633eb280f --- /dev/null +++ b/.gitallowed @@ -0,0 +1,6 @@ +# Part of the Carbon Language project, under the Apache License v2.0 with LLVM +# Exceptions. See /LICENSE for license information. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +# Firebase API key +AIzaSyDyAAcQeReDNXVUd_ZdkSzYgqIDJe-kPK8 diff --git a/src/scripts/.gitignore b/.gitignore similarity index 92% rename from src/scripts/.gitignore rename to .gitignore index 3cadb3b12a28..3d6a90f79d9b 100644 --- a/src/scripts/.gitignore +++ b/.gitignore @@ -2,4 +2,4 @@ # Exceptions. See /LICENSE for license information. # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -__pycache__ +**/__pycache__/ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 5f703fc8d05b..a2e164f729f4 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -33,7 +33,7 @@ repos: Part of the Carbon Language project, under the Apache License v2.0 with LLVM Exceptions. See /LICENSE for license information. SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception - exclude: '^src/jekyll/(Gemfile.lock|theme/.*)$' + exclude: '^website/(firebase/.firebaserc|jekyll/(Gemfile.lock|theme/.*))$' - id: check-google-doc-style - id: markdown-toc - repo: https://github.com/codespell-project/codespell @@ -41,13 +41,13 @@ repos: hooks: - id: codespell args: ['-I', '.codespell_ignore'] - exclude: '^src/jekyll/Gemfile.lock$' + exclude: '^website/jekyll/Gemfile.lock$' - repo: local hooks: - id: proposal-list name: Update list of proposals description: Updates the list of proposals in proposals/README.md - entry: src/scripts/update_proposal_list.py + entry: proposals/scripts/update_proposal_list.py language: python files: ^proposals/(README|p.*)\.md$ pass_filenames: false @@ -60,7 +60,7 @@ repos: rev: v2.1.2 hooks: - id: prettier - exclude: ^src/jekyll/theme/ + exclude: ^website/jekyll/theme/ # Run linters last, as formatters and other checks may fix issues. - repo: https://gitlab.com/PyCQA/flake8 rev: 3.8.4 diff --git a/README.md b/README.md index 9154b9a60bc4..6f71c7187558 100644 --- a/README.md +++ b/README.md @@ -142,7 +142,5 @@ very high. Carbon's main repositories are: - **carbon-lang** - Carbon language specification and documentation. -- **carbon-project-tools** - Tools for contributors to the Carbon language - project. - **carbon-toolchain** - Carbon language toolchain and reference implementation. diff --git a/docs/project/contribution_tools.md b/docs/project/contribution_tools.md index 108c90cc27be..196cd7bfa666 100644 --- a/docs/project/contribution_tools.md +++ b/docs/project/contribution_tools.md @@ -99,17 +99,17 @@ When modifying or adding pre-commit hooks, please run #### new_proposal.py -[new_proposal.py](/src/scripts/new_proposal.py) is a helper for generating the -PR and proposal file for a new proposal. It's documented in +[new_proposal.py](/proposals/scripts/new_proposal.py) is a helper for generating +the PR and proposal file for a new proposal. It's documented in [the proposal template](/proposals/template.md). **NOTE**: This requires installing [the gh CLI](#gh). #### pr_comments.py -[pr_comments.py](https://github.com/carbon-language/carbon-project-tools/blob/trunk/github/pr_comments.py) -is a helper for scanning comments in GitHub. It's particularly intended to help -find threads which need to be resolved. +[pr_comments.py](/github/pr_comments.py) is a helper for scanning comments in +GitHub. It's particularly intended to help find threads which need to be +resolved. Options can be seen with `-h`. A couple key options to be aware of are: diff --git a/docs/project/groups.md b/docs/project/groups.md index 6645dedb1c81..a71b9a18bf07 100644 --- a/docs/project/groups.md +++ b/docs/project/groups.md @@ -24,7 +24,7 @@ We use a mix of: - [GitHub organization](https://github.com/orgs/carbon-language/people) - [GitHub team: Contributors with label access](https://github.com/orgs/carbon-language/teams/contributors-with-label-access): Mirrors the GitHub organization for write access. - [Manually updated](https://github.com/carbon-language/carbon-project-tools/blob/trunk/github/update_label_access.py). + [Manually updated](/github/update_label_access.py). - [Discourse Forums account](https://forums.carbon-lang.dev) - [Discord Chat access](https://discord.com/app) - [Google group](https://groups.google.com/g/carbon-lang-contributors): Grants diff --git a/src/README.md b/github/README.md similarity index 53% rename from src/README.md rename to github/README.md index 257f79e3fe7d..063f5e835320 100644 --- a/src/README.md +++ b/github/README.md @@ -1,4 +1,4 @@ -# Doc Tooling +# GitHub -This directory contains tooling related to managing Carbon's documentation. +Scripts for use with GitHub. -If you're looking for language tooling, see the `carbon-toolchains` repository. +See individual scripts for more details. + +Please use `pytest` for testing: + +``` +$ pip install pytest +``` diff --git a/github/github_helpers.py b/github/github_helpers.py new file mode 100644 index 000000000000..5f7104b0cd30 --- /dev/null +++ b/github/github_helpers.py @@ -0,0 +1,102 @@ +"""GitHub GraphQL helpers. + +https://developer.github.com/v4/explorer/ is very useful for building queries. +""" + +__copyright__ = """ +Part of the Carbon Language project, under the Apache License v2.0 with LLVM +Exceptions. See /LICENSE for license information. +SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +""" + +import os + +# https://pypi.org/project/gql/ +import gql +import gql.transport.requests + +_ENV_TOKEN = "GITHUB_ACCESS_TOKEN" + +# Query elements for pagination. +PAGINATION = """pageInfo { + hasNextPage + endCursor +} +totalCount""" + + +def add_access_token_arg(parser, permissions): + """Adds a flag to set the access token.""" + access_token = os.environ.get(_ENV_TOKEN, default=None) + parser.add_argument( + "--access-token", + metavar="ACCESS_TOKEN", + default=access_token, + required=not access_token, + help="The access token for use with GitHub. May also be specified in " + "the environment as %s. The access token should have permissions: %s" + % (_ENV_TOKEN, permissions), + ) + + +class Client(object): + """A GitHub GraphQL client.""" + + def __init__(self, parsed_args): + """Connects to GitHub.""" + transport = gql.transport.requests.RequestsHTTPTransport( + url="https://api.github.com/graphql", + headers={"Authorization": "bearer %s" % parsed_args.access_token}, + ) + self._client = gql.Client( + transport=transport, fetch_schema_from_transport=True + ) + + def execute(self, query): + """Runs a query.""" + return self._client.execute(gql.gql(query)) + + def execute_and_paginate(self, query, path, first_page=None): + """Runs a query with pagination. + + Arguments: + query: The GraphQL query template, which must have both 'cursor' and + 'pagination' fields to fill in. The cursor should be part of the + location query (with 'first'), and the pagination should be at the + same level as nodes. + path: A list of strings indicating the path to the nodes in the + result. + first_page: An optional object for the first page of results, which + will otherwise automatically be collected. This exists for callers + to optimize by collecting other data with the first page. + """ + format = {"cursor": "", "pagination": PAGINATION} + count = 0 + exp_count = None + while True: + if first_page: + result = first_page + first_page = None + else: + result = self.execute(query % format) + # Follow the path to the nodes being paginated. + node_parent = result + for entry in path: + node_parent = node_parent[entry] + # Store the total count of responses. + if not exp_count: + exp_count = node_parent["totalCount"] + # Yield each node individually. + for node in node_parent["nodes"]: + yield node + count += 1 + # Check for pagination, verifying the total count on exit. + page_info = node_parent["pageInfo"] + if not page_info["hasNextPage"]: + assert exp_count == count, "exp %d != actual %d at path %s" % ( + exp_count, + count, + path, + ) + return + format["cursor"] = ' after: "%s"' % page_info["endCursor"] diff --git a/github/github_helpers_test.py b/github/github_helpers_test.py new file mode 100644 index 000000000000..890f27534405 --- /dev/null +++ b/github/github_helpers_test.py @@ -0,0 +1,181 @@ +"""Tests for github_helpers.py.""" + +__copyright__ = """ +Part of the Carbon Language project, under the Apache License v2.0 with LLVM +Exceptions. See /LICENSE for license information. +SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +""" + +import unittest +from unittest import mock + +import github_helpers + + +_TEST_QUERY = """ +query { + top(login: "foo") { + child(first: 100%(cursor)s) { + nodes { + login + } + %(pagination)s + } + } +} +""" + +_TEST_QUERY_PATH = ("top", "child") + +_EXP_QUERY_FIRST_PAGE = """ +query { + top(login: "foo") { + child(first: 100) { + nodes { + login + } + pageInfo { + hasNextPage + endCursor +} +totalCount + } + } +} +""" + +_EXP_QUERY_SECOND_PAGE = """ +query { + top(login: "foo") { + child(first: 100 after: "CURSOR") { + nodes { + login + } + pageInfo { + hasNextPage + endCursor +} +totalCount + } + } +} +""" + + +class TestGithubHelpers(unittest.TestCase): + def setUp(self): + patcher = mock.patch.object( + github_helpers.Client, "__init__", lambda self, parsed_args: None + ) + self.addCleanup(patcher.stop) + patcher.start() + self.client = github_helpers.Client(None) + + @staticmethod + def mock_result(nodes, total_count=None, has_next_page=False): + if total_count is None: + total_count = len(nodes) + end_cursor = None + if has_next_page: + end_cursor = "CURSOR" + return { + "top": { + "child": { + "nodes": nodes, + "pageInfo": { + "hasNextPage": has_next_page, + "endCursor": end_cursor, + }, + "totalCount": total_count, + } + } + } + + def test_execute_and_paginate_empty(self): + self.client.execute = mock.MagicMock(return_value=self.mock_result([])) + self.assertEqual( + list( + self.client.execute_and_paginate(_TEST_QUERY, _TEST_QUERY_PATH) + ), + [], + ) + self.client.execute.assert_called_once_with(_EXP_QUERY_FIRST_PAGE) + + def test_execute_and_paginate_one_page(self): + self.client.execute = mock.MagicMock( + return_value=self.mock_result(["foo", "bar", "baz"]) + ) + self.assertEqual( + list( + self.client.execute_and_paginate(_TEST_QUERY, _TEST_QUERY_PATH) + ), + ["foo", "bar", "baz"], + ) + self.client.execute.assert_called_once_with(_EXP_QUERY_FIRST_PAGE) + + def test_execute_and_paginate_one_page_count_mismatch(self): + self.client.execute = mock.MagicMock( + return_value=self.mock_result(["foo"], total_count=2), + ) + self.assertRaises( + AssertionError, + list, + self.client.execute_and_paginate(_TEST_QUERY, _TEST_QUERY_PATH), + ) + self.client.execute.assert_called_once_with(_EXP_QUERY_FIRST_PAGE) + + def test_execute_and_paginate_two_page(self): + def paging(query): + if query == _EXP_QUERY_FIRST_PAGE: + return self.mock_result( + ["foo", "bar"], total_count=3, has_next_page=True + ) + elif query == _EXP_QUERY_SECOND_PAGE: + return self.mock_result(["baz"], total_count="unused") + else: + raise ValueError("Bad query: %s" % query) + + self.client.execute = mock.MagicMock(side_effect=paging) + self.assertEqual( + list( + self.client.execute_and_paginate(_TEST_QUERY, _TEST_QUERY_PATH) + ), + ["foo", "bar", "baz"], + ) + self.assertEqual(self.client.execute.call_count, 2) + + def test_execute_and_paginate_first_page_done(self): + self.client.execute = mock.MagicMock() + self.assertEqual( + list( + self.client.execute_and_paginate( + _TEST_QUERY, + _TEST_QUERY_PATH, + first_page=self.mock_result(["foo"]), + ) + ), + ["foo"], + ) + self.assertEqual(self.client.execute.call_count, 0) + + def test_execute_and_paginate_first_page_continue(self): + self.client.execute = mock.MagicMock( + return_value=self.mock_result(["bar"], total_count="unused") + ) + self.assertEqual( + list( + self.client.execute_and_paginate( + _TEST_QUERY, + _TEST_QUERY_PATH, + first_page=self.mock_result( + ["foo"], total_count=2, has_next_page=True + ), + ) + ), + ["foo", "bar"], + ) + self.client.execute.assert_called_once_with(_EXP_QUERY_SECOND_PAGE) + + +if __name__ == "__main__": + unittest.main() diff --git a/github/pr_comments.py b/github/pr_comments.py new file mode 100755 index 000000000000..e3a3bc76e5d3 --- /dev/null +++ b/github/pr_comments.py @@ -0,0 +1,444 @@ +#!/usr/bin/env python3 + +"""Figure out comments on a GitHub PR.""" + +__copyright__ = """ +Part of the Carbon Language project, under the Apache License v2.0 with LLVM +Exceptions. See /LICENSE for license information. +SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +""" + +import argparse +import datetime +import hashlib +import textwrap + +import github_helpers + +# The main query, into which other queries are composed. +_QUERY = """ +{ + repository(owner: "carbon-language", name: "%(repo)s") { + pullRequest(number: %(pr_num)d) { + author { + login + } + createdAt + title + + %(comments)s + %(reviews)s + %(review_threads)s + } + } +} +""" + +# Queries for comments on the PR. These are direct, non-review comments on the +# PR. +_QUERY_COMMENTS = """ + comments(first: 100%(cursor)s) { + nodes { + author { + login + } + body + createdAt + url + } + %(pagination)s + } +""" + +# Queries for reviews on the PR, which have a non-empty body if a review has +# a summary comment. +_QUERY_REVIEWS = """ + reviews(first: 100%(cursor)s) { + nodes { + author { + login + } + body + createdAt + url + } + %(pagination)s + } +""" + +# Queries for review threads on the PR. +_QUERY_REVIEW_THREADS = """ + reviewThreads(first: 100%(cursor)s) { + nodes { + comments(first: 100) { + nodes { + author { + login + } + body + createdAt + originalPosition + originalCommit { + abbreviatedOid + } + path + } + } + isResolved + resolvedBy { + createdAt + login + } + } + %(pagination)s + } +""" + + +class _Comment(object): + """A comment, either on a review thread or top-level on the PR.""" + + def __init__(self, author, timestamp, body): + self.author = author + self.timestamp = datetime.datetime.strptime( + timestamp, "%Y-%m-%dT%H:%M:%SZ" + ) + self.body = body + + @staticmethod + def from_raw_comment(raw_comment): + """Creates the comment from a raw comment dict.""" + return _Comment( + raw_comment["author"]["login"], + raw_comment["createdAt"], + raw_comment["body"], + ) + + @staticmethod + def _rewrap(content): + """Rewraps a comment to fit in 80 columns with an indent.""" + lines = [] + for line in content.split("\n"): + lines.extend( + [ + x + for x in textwrap.wrap( + line, + width=80, + initial_indent=" " * 4, + subsequent_indent=" " * 4, + ) + ] + ) + return "\n".join(lines) + + def format(self, long): + """Formats the comment.""" + if long: + return "%s%s at %s:\n%s" % ( + " " * 2, + self.author, + self.timestamp.strftime("%Y-%m-%d %H:%M"), + self._rewrap(self.body), + ) + else: + # Compact newlines down into pilcrows, leaving a space after. + body = self.body.replace("\r", "").replace("\n", "¶ ") + while "¶ ¶" in body: + body = body.replace("¶ ¶", "¶¶") + line = "%s%s: %s" % (" " * 2, self.author, body) + return line if len(line) <= 80 else line[:77] + "..." + + +class _PRComment(_Comment): + """A comment on the top-level PR.""" + + def __init__(self, raw_comment): + super().__init__( + raw_comment["author"]["login"], + raw_comment["createdAt"], + raw_comment["body"], + ) + self.url = raw_comment["url"] + + def __lt__(self, other): + return self.timestamp < other.timestamp + + def format(self, long): + return "%s\n%s" % (self.url, super().format(long)) + + +class _Thread(object): + """A review thread on a line of code.""" + + def __init__(self, parsed_args, thread): + self.is_resolved = thread["isResolved"] + + comments = thread["comments"]["nodes"] + first_comment = comments[0] + self.line = first_comment["originalPosition"] + self.path = first_comment["path"] + + # Link to the comment in the commit; GitHub features work better there + # than in the conversation view. The diff_url allows viewing changes + # since the comment, although the comment won't be visible there. + template = ( + "https://github.com/carbon-language/%(repo)s/pull/%(pr_num)s/" + "files/%(oid)s%(head)s#diff-%(path_md5)s%(line_side)s%(line)s" + ) + # GitHub uses an md5 of the file's path for the link. + path_md5 = hashlib.md5() + path_md5.update(bytearray(self.path, "utf-8")) + format_dict = { + "head": "", + "line_side": "R", + "line": self.line, + "oid": first_comment["originalCommit"]["abbreviatedOid"], + "path_md5": path_md5.hexdigest(), + "pr_num": parsed_args.pr_num, + "repo": parsed_args.repo, + } + self.url = template % format_dict + format_dict["head"] = "..HEAD" + format_dict["line_side"] = "L" + self.diff_url = template % format_dict + + self.comments = [ + _Comment.from_raw_comment(comment) + for comment in thread["comments"]["nodes"] + ] + if self.is_resolved: + self.comments.append( + _Comment( + thread["resolvedBy"]["login"], + thread["resolvedBy"]["createdAt"], + "", + ) + ) + + def __lt__(self, other): + """Sort threads by line then timestamp.""" + if self.line != other.line: + return self.line < other.line + return self.comments[0].timestamp < other.comments[0].timestamp + + def format(self, long): + """Formats the review thread with comments.""" + lines = [] + lines.append( + "%s\n - line %d; %s" + % ( + self.url, + self.line, + ("resolved" if self.is_resolved else "unresolved"), + ) + ) + if self.diff_url: + lines.append(" - diff: %s" % self.diff_url) + for comment in self.comments: + lines.append(comment.format(long)) + return "\n".join(lines) + + def has_comment_from(self, comments_from): + """Returns true if comments has a comment from comments_from.""" + for comment in self.comments: + if comment.author == comments_from: + return True + return False + + +def _parse_args(args=None): + """Parses command-line arguments and flags.""" + parser = argparse.ArgumentParser(description="Lists comments on a PR.") + parser.add_argument( + "pr_num", + metavar="PR#", + type=int, + help="The pull request to fetch comments from.", + ) + github_helpers.add_access_token_arg(parser, "repo") + parser.add_argument( + "--comments-after", + metavar="LOGIN", + help="Only print threads where the final comment is not from the given " + "user. For example, use when looking for threads that you still need " + "to respond to.", + ) + parser.add_argument( + "--comments-from", + metavar="LOGIN", + help="Only print threads with comments from the given user. For " + "example, use when looking for threads that you've commented on.", + ) + parser.add_argument( + "--include-resolved", + action="store_true", + help="Whether to include resolved review threads. By default, only " + "unresolved threads will be shown.", + ) + parser.add_argument( + "--repo", + choices=["carbon-lang", "carbon-toolchain"], + default="carbon-lang", + help="The Carbon repo to query. Defaults to %(default)s.", + ) + parser.add_argument( + "--long", + action="store_true", + help="Prints long output, with the full comment.", + ) + return parser.parse_args(args=args) + + +def _query(parsed_args, field_name=None): + """Returns a query for the passed field_name, or all by default.""" + print(".", end="", flush=True) + format = { + "pr_num": parsed_args.pr_num, + "repo": parsed_args.repo, + "comments": "", + "review_threads": "", + "reviews": "", + } + if field_name: + # Use a cursor for pagination of the field. + if field_name == "comments": + format["comments"] = _QUERY_COMMENTS + elif field_name == "reviewThreads": + format["review_threads"] = _QUERY_REVIEW_THREADS + elif field_name == "reviews": + format["reviews"] = _QUERY_REVIEWS + else: + raise ValueError("Unexpected field_name: %s" % field_name) + else: + # Fetch the first page of all fields. + subformat = {"cursor": "", "pagination": github_helpers.PAGINATION} + format["comments"] = _QUERY_COMMENTS % subformat + format["review_threads"] = _QUERY_REVIEW_THREADS % subformat + format["reviews"] = _QUERY_REVIEWS % subformat + return _QUERY % format + + +def _accumulate_pr_comment(parsed_args, comments, raw_comment): + """Collects top-level comments and reviews.""" + # Elide reviews that have no top-level comment body. + if raw_comment["body"]: + comments.append(_PRComment(raw_comment)) + + +def _accumulate_thread(parsed_args, threads_by_path, raw_thread): + """Adds threads to threads_by_path for later sorting.""" + thread = _Thread(parsed_args, raw_thread) + + # Optionally skip resolved threads. + if not parsed_args.include_resolved and thread.is_resolved: + return + + # Optionally skip threads where the given user isn't the last commenter. + if ( + parsed_args.comments_after + and thread.comments[-1].author == parsed_args.comments_after + ): + return + + # Optionally skip threads where the given user hasn't commented. + if parsed_args.comments_from and not thread.has_comment_from( + parsed_args.comments_from + ): + return + + if thread.path not in threads_by_path: + threads_by_path[thread.path] = [] + threads_by_path[thread.path].append(thread) + + +def _paginate( + field_name, accumulator, parsed_args, client, main_result, output +): + """Paginates through the given field_name, accumulating results.""" + query = _query(parsed_args, field_name=field_name) + path = ("repository", "pullRequest", field_name) + for node in client.execute_and_paginate( + query, path, first_page=main_result + ): + accumulator(parsed_args, output, node) + + +def _fetch_comments(parsed_args): + """Fetches comments and review threads from GitHub.""" + # Each _query call will print a '.' for progress. + print( + "Loading https://github.com/carbon-language/%s/pull/%d ..." + % (parsed_args.repo, parsed_args.pr_num), + end="", + flush=True, + ) + + client = github_helpers.Client(parsed_args) + + # Get the initial set of review threads, and print the PR summary. + main_result = client.execute(_query(parsed_args)) + pull_request = main_result["repository"]["pullRequest"] + + # Paginate comments, reviews, and review threads. + comments = [] + _paginate( + "comments", + _accumulate_pr_comment, + parsed_args, + client, + main_result, + comments, + ) + # Combine reviews into comments for interleaving. + _paginate( + "reviews", + _accumulate_pr_comment, + parsed_args, + client, + main_result, + comments, + ) + threads_by_path = {} + _paginate( + "reviewThreads", + _accumulate_thread, + parsed_args, + client, + main_result, + threads_by_path, + ) + + # Now that loading is done (no more progress indicators), print the header. + print() + pr_desc = _Comment( + pull_request["author"]["login"], + pull_request["createdAt"], + pull_request["title"], + ) + print(pr_desc.format(parsed_args.long)) + return comments, threads_by_path + + +def main(): + parsed_args = _parse_args() + comments, threads_by_path = _fetch_comments(parsed_args) + + for comment in sorted(comments): + print() + print(comment.format(parsed_args.long)) + + for path, threads in sorted(threads_by_path.items()): + # Print a header for each path. + print() + print("=" * 80) + print(path) + print("=" * 80) + + for thread in sorted(threads): + print() + print(thread.format(parsed_args.long)) + + +if __name__ == "__main__": + main() diff --git a/github/pr_comments_test.py b/github/pr_comments_test.py new file mode 100644 index 000000000000..fa068479bf83 --- /dev/null +++ b/github/pr_comments_test.py @@ -0,0 +1,278 @@ +"""Tests for pr_comments.py.""" + +__copyright__ = """ +Part of the Carbon Language project, under the Apache License v2.0 with LLVM +Exceptions. See /LICENSE for license information. +SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +""" + +import os +import unittest +from unittest import mock + +import pr_comments + + +class TestPRComments(unittest.TestCase): + def test_format_comment_short(self): + created_at = "2001-02-03T04:05:06Z" + self.assertEqual( + pr_comments._Comment("author", created_at, "brief").format(False), + " author: brief", + ) + self.assertEqual( + pr_comments._Comment("author", created_at, "brief\nwrap").format( + False + ), + " author: brief¶ wrap", + ) + self.assertEqual( + pr_comments._Comment( + "author", created_at, "brief\n\n\nwrap" + ).format(False), + " author: brief¶¶¶ wrap", + ) + self.assertEqual( + pr_comments._Comment( + "author", + created_at, + "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed " + "do eiusmo", + ).format(False), + " author: Lorem ipsum dolor sit amet, consectetur adipiscing " + "elit, sed do eiusmo", + ) + self.assertEqual( + pr_comments._Comment( + "author", + created_at, + "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed " + "do eiusmod", + ).format(False), + " author: Lorem ipsum dolor sit amet, consectetur adipiscing " + "elit, sed do eiu...", + ) + + def test_format_comment_long(self): + created_at = "2001-02-03T04:05:06Z" + self.assertEqual( + pr_comments._Comment("author", created_at, "brief").format(True), + " author at 2001-02-03 04:05:\n brief", + ) + self.assertEqual( + pr_comments._Comment("author", created_at, "brief\nwrap").format( + True + ), + " author at 2001-02-03 04:05:\n brief\n wrap", + ) + + body = ( + "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed " + "do eiusmod tempor incididunt ut labore et dolore magna " + "aliqua.\n" + "Ut enim ad minim veniam," + ) + self.assertEqual( + pr_comments._Comment("author", created_at, body).format(True), + " author at 2001-02-03 04:05:\n" + " Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed " + "do eiusmod\n" + " tempor incididunt ut labore et dolore magna aliqua.\n" + " Ut enim ad minim veniam,", + ) + + @staticmethod + def fake_thread(**kwargs): + with mock.patch.dict(os.environ, {}): + parsed_args = pr_comments._parse_args(["83"]) + return pr_comments._Thread( + parsed_args, TestPRComments.fake_thread_dict(**kwargs) + ) + + @staticmethod + def fake_thread_dict( + is_resolved=False, + path="foo.md", + line=3, + created_at="2001-02-03T04:05:06Z", + ): + thread_dict = { + "isResolved": is_resolved, + "comments": { + "nodes": [ + { + "author": {"login": "author"}, + "body": "comment", + "createdAt": created_at, + "originalCommit": {"abbreviatedOid": "abcdef"}, + "originalPosition": line, + "path": path, + "url": "http://xyz", + }, + { + "author": {"login": "other"}, + "body": "reply", + "createdAt": "2001-02-03T04:15:16Z", + }, + ], + }, + } + if is_resolved: + thread_dict["resolvedBy"] = { + "login": "resolver", + "createdAt": "2001-02-03T04:25:26Z", + } + return thread_dict + + def test_thread_format(self): + self.assertEqual( + self.fake_thread().format(False), + "https://github.com/carbon-language/carbon-lang/pull/83/" + "files/abcdef#diff-d8ca3b3d314d8209367af0eea2373b6fR3\n" + " - line 3; unresolved\n" + " - diff: https://github.com/carbon-language/carbon-lang/pull/83/" + "files/abcdef..HEAD#diff-d8ca3b3d314d8209367af0eea2373b6fL3\n" + " author: comment\n" + " other: reply", + ) + self.assertEqual( + self.fake_thread().format(True), + "https://github.com/carbon-language/carbon-lang/pull/83/files/" + "abcdef#diff-d8ca3b3d314d8209367af0eea2373b6fR3\n" + " - line 3; unresolved\n" + " - diff: https://github.com/carbon-language/carbon-lang/pull/83/" + "files/abcdef..HEAD#diff-d8ca3b3d314d8209367af0eea2373b6fL3\n" + " author at 2001-02-03 04:05:\n" + " comment\n" + " other at 2001-02-03 04:15:\n" + " reply", + ) + + self.assertEqual( + self.fake_thread(is_resolved=True).format(False), + "https://github.com/carbon-language/carbon-lang/pull/83/" + "files/abcdef#diff-d8ca3b3d314d8209367af0eea2373b6fR3\n" + " - line 3; resolved\n" + " - diff: https://github.com/carbon-language/carbon-lang/pull/83/" + "files/abcdef..HEAD#diff-d8ca3b3d314d8209367af0eea2373b6fL3\n" + " author: comment\n" + " other: reply\n" + " resolver: ", + ) + self.assertEqual( + self.fake_thread(is_resolved=True).format(True), + "https://github.com/carbon-language/carbon-lang/pull/83/" + "files/abcdef#diff-d8ca3b3d314d8209367af0eea2373b6fR3\n" + " - line 3; resolved\n" + " - diff: https://github.com/carbon-language/carbon-lang/pull/83/" + "files/abcdef..HEAD#diff-d8ca3b3d314d8209367af0eea2373b6fL3\n" + " author at 2001-02-03 04:05:\n" + " comment\n" + " other at 2001-02-03 04:15:\n" + " reply\n" + " resolver at 2001-02-03 04:25:\n" + " ", + ) + + def test_thread_lt(self): + thread1 = self.fake_thread(line=2) + thread2 = self.fake_thread() + thread3 = self.fake_thread(created_at="2002-02-03T04:05:06Z") + + self.assertTrue(thread1 < thread2) + self.assertFalse(thread2 < thread1) + + self.assertFalse(thread2 < thread2) + + self.assertTrue(thread2 < thread3) + self.assertFalse(thread3 < thread2) + + def test_accumulate_thread(self): + with mock.patch.dict(os.environ, {}): + parsed_args = pr_comments._parse_args(["83"]) + threads_by_path = {} + review_threads = [ + self.fake_thread_dict(line=2), + self.fake_thread_dict(line=4), + self.fake_thread_dict(path="other.md"), + self.fake_thread_dict(), + ] + for thread in review_threads: + pr_comments._accumulate_thread( + parsed_args, + threads_by_path, + thread, + ) + self.assertEqual(sorted(threads_by_path.keys()), ["foo.md", "other.md"]) + threads = sorted(threads_by_path["foo.md"]) + self.assertEqual(len(threads), 3) + self.assertEqual(threads[0].line, 2) + self.assertEqual(threads[1].line, 3) + self.assertEqual(threads[2].line, 4) + self.assertEqual(len(threads_by_path["other.md"]), 1) + + @staticmethod + def fake_pr_comment(**kwargs): + return pr_comments._PRComment( + TestPRComments.fake_pr_comment_dict(**kwargs) + ) + + @staticmethod + def fake_pr_comment_dict( + body="comment", + created_at="2001-02-03T04:05:06Z", + ): + pr_comment_dict = { + "author": {"login": "author"}, + "body": body, + "createdAt": created_at, + "url": "http://xyz", + } + return pr_comment_dict + + def test_pr_comment_format(self): + self.assertEqual( + self.fake_pr_comment().format(False), + "http://xyz\n author: comment", + ) + self.assertEqual( + self.fake_pr_comment().format(True), + "http://xyz\n author at 2001-02-03 04:05:\n comment", + ) + + def test_pr_comment_lt(self): + pr_comment1 = self.fake_pr_comment() + pr_comment2 = self.fake_pr_comment(created_at="2002-02-03T04:05:06Z") + + self.assertTrue(pr_comment1 < pr_comment2) + self.assertFalse(pr_comment2 < pr_comment1) + + self.assertFalse(pr_comment2 < pr_comment2) + + def test_accumulate_pr_comment(self): + with mock.patch.dict(os.environ, {}): + parsed_args = pr_comments._parse_args(["83"]) + raw_comments = [ + self.fake_pr_comment_dict(body="x"), + self.fake_pr_comment_dict(body=""), + self.fake_pr_comment_dict( + body="y", created_at="2000-02-03T04:05:06Z" + ), + self.fake_pr_comment_dict( + body="z", created_at="2002-02-03T04:05:06Z" + ), + ] + comments = [] + for raw_comment in raw_comments: + pr_comments._accumulate_pr_comment( + parsed_args, comments, raw_comment + ) + comments.sort() + self.assertEqual(len(comments), 3) + self.assertEqual(comments[0].body, "y") + self.assertEqual(comments[1].body, "x") + self.assertEqual(comments[2].body, "z") + + +if __name__ == "__main__": + unittest.main() diff --git a/github/update_label_access.py b/github/update_label_access.py new file mode 100755 index 000000000000..2abc0899c79d --- /dev/null +++ b/github/update_label_access.py @@ -0,0 +1,144 @@ +#!/usr/bin/env python3 + +"""Updates the contributors-with-label-access team. + +This team exists because we need a team to manage triage access to repos; +GitHub doesn't allow the org to be set to triage access, only read/write. It +will be updated to include all members of the carbon-language organization. +""" + +__copyright__ = """ +Part of the Carbon Language project, under the Apache License v2.0 with LLVM +Exceptions. See /LICENSE for license information. +SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +""" + +import argparse + +# https://github.com/PyGithub/PyGithub +# GraphQL is preferred, but falling back to pygithub for unsupported mutations. +import github + +import github_helpers + +# The organization to mirror members from. +_ORG = "carbon-language" + +# The team to mirror to. +_TEAM = "contributors-with-label-access" + +# Accounts in the org to skip mirroring. +_IGNORE_ACCOUNTS = ("CarbonLangInfra", "google-admin", "googlebot") + +# Queries organization members. +_ORG_MEMBER_QUERY = """ +query { + organization(login: "%s") { + membersWithRole(first: 100%%(cursor)s) { + nodes { + login + } + %%(pagination)s + } + } +} +""" + +# The path for nodes in _ORG_MEMBER_QUERY. +_ORG_MEMBER_PATH = ("organization", "membersWithRole") + +# Queries team members. +_TEAM_MEMBER_QUERY = """ +query { + organization(login: "%s") { + team(slug: "%s") { + members(first: 100%%(cursor)s) { + nodes { + login + } + %%(pagination)s + } + } + } +} +""" + +# The path for nodes in _TEAM_MEMBER_QUERY. +_TEAM_MEMBER_PATH = ("organization", "team", "members") + + +def _parse_args(args=None): + """Parses command-line arguments and flags.""" + parser = argparse.ArgumentParser(description=__doc__) + github_helpers.add_access_token_arg(parser, "admin:org, repo") + return parser.parse_args(args=args) + + +def _load_org_members(client): + """Loads org members.""" + print("Loading %s..." % _ORG) + org_members = set() + ignored = set() + for node in client.execute_and_paginate( + _ORG_MEMBER_QUERY % _ORG, _ORG_MEMBER_PATH + ): + login = node["login"] + if login not in _IGNORE_ACCOUNTS: + org_members.add(login) + else: + ignored.add(login) + print( + "%s has %d non-ignored members, and %d ignored." + % (_ORG, len(org_members), len(ignored)) + ) + unignored = set(_IGNORE_ACCOUNTS) - ignored + assert not unignored, "Missing ignored accounts: %s" % unignored + return org_members + + +def _load_team_members(client): + """Load team members.""" + print("Loading %s..." % _TEAM) + team_members = set() + for node in client.execute_and_paginate( + _TEAM_MEMBER_QUERY % (_ORG, _TEAM), _TEAM_MEMBER_PATH + ): + team_members.add(node["login"]) + print("%s has %d members." % (_ORG, len(team_members))) + return team_members + + +def _update_team(gh, org_members, team_members): + """Updates the team if needed. + + This switches to pygithub because GraphQL lacks equivalent mutation support. + """ + gh_team = gh.get_organization(_ORG).get_team_by_slug(_TEAM) + add_members = org_members - team_members + if add_members: + print("Adding members: %s" % ", ".join(add_members)) + for member in add_members: + gh_team.add_membership(gh.get_user(member)) + + remove_members = team_members - org_members + if remove_members: + print("Removing members: %s" % ", ".join(remove_members)) + for member in remove_members: + gh_team.remove_membership(gh.get_user(member)) + + +def main(): + parsed_args = _parse_args() + print("Connecting...") + client = github_helpers.Client(parsed_args) + + org_members = _load_org_members(client) + team_members = _load_team_members(client) + if org_members != team_members: + gh = github.Github(parsed_args.access_token) + _update_team(gh, org_members, team_members) + print("Done!") + + +if __name__ == "__main__": + main() diff --git a/github/update_label_access_test.py b/github/update_label_access_test.py new file mode 100644 index 000000000000..71489f551306 --- /dev/null +++ b/github/update_label_access_test.py @@ -0,0 +1,100 @@ +"""Tests for update_label_access.py.""" + +__copyright__ = """ +Part of the Carbon Language project, under the Apache License v2.0 with LLVM +Exceptions. See /LICENSE for license information. +SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +""" + +import github +import unittest +from unittest import mock + +import github_helpers +import update_label_access + + +class TestUpdateLabelAccess(unittest.TestCase): + def setUp(self): + self.client = mock.create_autospec(github_helpers.Client, instance=True) + self.gh = mock.create_autospec(github.Github, instance=True) + self.gh_org = mock.create_autospec(github.Organization, instance=True) + self.gh_team = mock.create_autospec(github.Team, instance=True) + + self.gh.get_organization = mock.MagicMock(return_value=self.gh_org) + self.gh_org.get_team_by_slug = mock.MagicMock(return_value=self.gh_team) + + def _mock_nodes(self, logins): + self.client.execute_and_paginate.return_value = [ + {"login": login} for login in logins + ] + + def test_load_org_members_empty(self): + self._mock_nodes([]) + self.assertRaises( + AssertionError, update_label_access._load_org_members, self.client + ) + + def test_load_org_members_missing_ignored(self): + self._mock_nodes(["foo", "bar"]) + self.assertRaises( + AssertionError, update_label_access._load_org_members, self.client + ) + + def test_load_org_members_ignored_only(self): + self._mock_nodes(update_label_access._IGNORE_ACCOUNTS) + self.assertEqual( + update_label_access._load_org_members(self.client), set() + ) + + def test_load_org_members_found(self): + self._mock_nodes( + ["foo", "bar"] + list(update_label_access._IGNORE_ACCOUNTS) + ) + self.assertEqual( + update_label_access._load_org_members(self.client), + set(["foo", "bar"]), + ) + + def test_load_team_members_empty(self): + self._mock_nodes([]) + self.assertEqual( + update_label_access._load_team_members(self.client), set() + ) + + def test_load_team_members_found(self): + self._mock_nodes(["foo", "bar"]) + self.assertEqual( + update_label_access._load_team_members(self.client), + set(["foo", "bar"]), + ) + + def test_update_team_empty(self): + update_label_access._update_team(self.gh, set(), set()) + + def test_update_team_equal(self): + update_label_access._update_team( + self.gh, set(["foo", "bar"]), set(["foo", "bar"]) + ) + + def test_update_team_add(self): + self.gh.get_user = mock.MagicMock(return_value="bar-user") + self.gh_team.add_membership = mock.MagicMock() + update_label_access._update_team( + self.gh, set(["foo", "bar"]), set(["foo"]) + ) + self.gh.get_user.assert_called_once_with("bar") + self.gh_team.add_membership.assert_called_once_with("bar-user") + + def test_update_team_remove(self): + self.gh.get_user = mock.MagicMock(return_value="bar-user") + self.gh_team.remove_membership = mock.MagicMock() + update_label_access._update_team( + self.gh, set(["foo"]), set(["foo", "bar"]) + ) + self.gh.get_user.assert_called_once_with("bar") + self.gh_team.remove_membership.assert_called_once_with("bar-user") + + +if __name__ == "__main__": + unittest.main() diff --git a/src/scripts/README.md b/proposals/scripts/README.md similarity index 80% rename from src/scripts/README.md rename to proposals/scripts/README.md index ed4c698a2039..27f035b96c8b 100644 --- a/src/scripts/README.md +++ b/proposals/scripts/README.md @@ -6,7 +6,7 @@ Exceptions. See /LICENSE for license information. SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception --> -Self-contained scripts used as part of maintaining Carbon infrastructure. +Scripts for use with proposals. See individual scripts for more details. diff --git a/src/scripts/new_proposal.py b/proposals/scripts/new_proposal.py similarity index 100% rename from src/scripts/new_proposal.py rename to proposals/scripts/new_proposal.py diff --git a/src/scripts/new_proposal_test.py b/proposals/scripts/new_proposal_test.py similarity index 100% rename from src/scripts/new_proposal_test.py rename to proposals/scripts/new_proposal_test.py diff --git a/src/scripts/update_proposal_list.py b/proposals/scripts/update_proposal_list.py similarity index 100% rename from src/scripts/update_proposal_list.py rename to proposals/scripts/update_proposal_list.py diff --git a/proposals/template.md b/proposals/template.md index c6e4d2826829..192eb17840fc 100644 --- a/proposals/template.md +++ b/proposals/template.md @@ -23,7 +23,7 @@ SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception ## TODO: Initial proposal setup -> TIP: Run `../src/scripts/new_proposal.py "TITLE"` to do new proposal setup. +> TIP: Run `./scripts/new_proposal.py "TITLE"` to do new proposal setup. 1. Copy this template to `new.md`, and create a commit. 2. Create a GitHub pull request, to get a pull request number.