diff --git a/github_backup/github_backup.py b/github_backup/github_backup.py index 64c08c3..3126f47 100644 --- a/github_backup/github_backup.py +++ b/github_backup/github_backup.py @@ -42,7 +42,9 @@ DISCUSSION_PAGE_SIZE, DISCUSSION_REPLIES_QUERY, ISSUE_CROSS_REFERENCE_COUNT_QUERY, + PULL_COMMENTS_QUERY, PULL_CROSS_REFERENCE_COUNT_QUERY, + PULL_PAGE_SIZE, ) FILE_URI_PREFIX = "file://" @@ -2367,7 +2369,7 @@ def backup_repositories(args, output_directory, repositories): lfs_clone=args.lfs_clone, no_prune=args.no_prune, ) - if args.include_issues or args.include_everything: + if repository["has_issues"] and (args.include_issues or args.include_everything): incremental_resource_work_attempted = True issue_cwd = os.path.join(repo_cwd, "issues") args.since = read_resource_last_update( @@ -2376,7 +2378,7 @@ def backup_repositories(args, output_directory, repositories): backup_issues(args, repo_cwd, repository, repos_template) write_resource_last_update(args, issue_cwd, repository) - if args.include_pulls or args.include_everything: + if repository.get("has_pull_requests", True) and (args.include_pulls or args.include_everything): incremental_resource_work_attempted = True pulls_cwd = os.path.join(repo_cwd, "pulls") args.since = read_resource_last_update( @@ -2385,7 +2387,7 @@ def backup_repositories(args, output_directory, repositories): backup_pulls(args, repo_cwd, repository, repos_template) write_resource_last_update(args, pulls_cwd, repository) - if args.include_discussions or args.include_everything: + if repository["has_discussions"] and (args.include_discussions or args.include_everything): backup_discussions(args, repo_cwd, repository) if args.include_milestones or args.include_everything: @@ -3144,7 +3146,6 @@ def pull_is_due_for_repository_checkpoint(pull): # regular comments need to be fetched via issue API. # For backwards compatibility with versions <= 0.41.0 # keep name "comment_data" for review comments - comments_regular_template = _issue_template + "/{0}/comments" comments_template = _pulls_template + "/{0}/comments" commits_template = _pulls_template + "/{0}/commits" reviews_template = _pulls_template + "/{0}/reviews" @@ -3175,8 +3176,49 @@ def pull_is_due_for_repository_checkpoint(pull): if ( args.include_pull_comments or args.include_everything ) and should_fetch_non_review_data: - template = comments_regular_template.format(number) - pulls[number]["comment_regular_data"] = retrieve_data(args, template) + after = None + comments_page = 1 + comments = [] + comment_total_count = 0 + owner, name = _repository_owner_name(repository) + while True: + data = retrieve_graphql_data( + args, + PULL_COMMENTS_QUERY, + { + "owner": owner, + "name": name, + "number": number, + "after": after, + "pageSize": PULL_PAGE_SIZE, + }, + log_context="pull request {0}#{1} comments page {2}".format( + repository["full_name"], number, comments_page + ), + ) + repository_data = data.get("repository") or {} + pr_data = repository_data.get("pullRequest") + if pr_data is None: + raise Exception( + "Pull request #{0} not found in {1}".format( + number, repository["full_name"] + ) + ) + + comments_connection = pr_data.get("comments") or {} + comment_total_count = comments_connection.get( + "totalCount", comment_total_count + ) + comments.extend(_connection_nodes(comments_connection)) + + page_info = comments_connection.get("pageInfo") or {} + if not page_info.get("hasNextPage"): + break + + after = page_info.get("endCursor") + comments_page += 1 + pulls[number]["comment_regular_data"] = comments + template = comments_template.format(number) pulls[number]["comment_data"] = retrieve_data(args, template) if include_pull_reviews: @@ -3230,7 +3272,13 @@ def backup_milestones(args, repo_cwd, repository, repos_template): query_args = {"state": "all"} - _milestones = retrieve_data(args, template, query_args=query_args) + try: + _milestones = retrieve_data(args, template, query_args=query_args) + except HTTPError as e: + if e.code == 404: + logger.info("Milestones are not available for this repository, skipping") + return + raise milestones = {} for milestone in _milestones: @@ -3303,7 +3351,13 @@ def backup_labels(args, repo_cwd, repository, repos_template): label_cwd = os.path.join(repo_cwd, "labels") output_file = "{0}/labels.json".format(label_cwd) template = "{0}/{1}/labels".format(repos_template, repository["full_name"]) - _backup_data(args, "labels", template, output_file, label_cwd) + try: + _backup_data(args, "labels", template, output_file, label_cwd) + except HTTPError as e: + if e.code == 404: + logger.info("Labels are not available for this repository, skipping") + return + raise def backup_hooks(args, repo_cwd, repository, repos_template): @@ -3348,7 +3402,13 @@ def backup_releases(args, repo_cwd, repository, repos_template, include_assets=F query_args = {} release_template = "{0}/{1}/releases".format(repos_template, repository_fullname) - releases = retrieve_data(args, release_template, query_args=query_args) + try: + releases = retrieve_data(args, release_template, query_args=query_args) + except HTTPError as e: + if e.code == 404: + logger.info("Releases are not available for this repository, skipping") + return + raise if args.skip_prerelease: releases = [r for r in releases if not r["prerelease"] and not r["draft"]] diff --git a/github_backup/graphql_queries.py b/github_backup/graphql_queries.py index a80d131..ed2ac4d 100644 --- a/github_backup/graphql_queries.py +++ b/github_backup/graphql_queries.py @@ -1,6 +1,7 @@ """GraphQL query templates used by github-backup.""" DISCUSSION_PAGE_SIZE = 100 +PULL_PAGE_SIZE = 100 CROSS_REFERENCE_PAGE_SIZE = 100 @@ -62,6 +63,72 @@ } """ +PULL_COMMENTS_QUERY = """ +query($owner: String!, $name: String!, $number: Int!, $after: String, $pageSize: Int!) { + repository(owner: $owner, name: $name) { + pullRequest(number: $number) { + comments(first: $pageSize, after: $after) { + totalCount + nodes { + ...IssueCommentFields + } + pageInfo { + hasNextPage + endCursor + } + } + } + } +} + +fragment ActorFields on Actor { + avatarUrl + login + resourcePath + url +} + +fragment ReactionGroupFields on ReactionGroup { + content + reactors { + totalCount + } +} + +fragment IssueCommentFields on IssueComment { + author { + ...ActorFields + } + authorAssociation + body + bodyHTML + bodyText + createdAt + createdViaEmail + databaseId + editor { + ...ActorFields + } + id + includesCreatedEdit + isMinimized + isPinned + lastEditedAt + minimizedReason + pinnedAt + pinnedBy { + ...ActorFields + } + publishedAt + reactionGroups { + ...ReactionGroupFields + } + resourcePath + updatedAt + url +} +""" + DISCUSSION_LIST_QUERY = """ query($owner: String!, $name: String!, $after: String, $pageSize: Int!) { repository(owner: $owner, name: $name) { diff --git a/tests/test_all_starred.py b/tests/test_all_starred.py index 9776926..914e179 100644 --- a/tests/test_all_starred.py +++ b/tests/test_all_starred.py @@ -30,6 +30,9 @@ def test_all_starred_clones_without_repositories_flag(self, mock_get_url, mock_f "private": False, "fork": False, "has_wiki": False, + "has_issues": True, + "has_pull_requests": True, + "has_discussions": True, "is_starred": True, # This flag is set for starred repos } @@ -56,6 +59,9 @@ def test_starred_repo_not_cloned_without_all_starred_flag(self, mock_get_url, mo "private": False, "fork": False, "has_wiki": False, + "has_issues": True, + "has_pull_requests": True, + "has_discussions": True, "is_starred": True, } @@ -80,6 +86,9 @@ def test_non_starred_repo_not_cloned_with_only_all_starred(self, mock_get_url, m "private": False, "fork": False, "has_wiki": False, + "has_issues": True, + "has_pull_requests": True, + "has_discussions": True, # No is_starred flag } @@ -103,6 +112,9 @@ def test_repositories_flag_still_works(self, mock_get_url, mock_fetch, create_ar "private": False, "fork": False, "has_wiki": False, + "has_issues": True, + "has_pull_requests": True, + "has_discussions": True, } with patch('github_backup.github_backup.mkdir_p'): diff --git a/tests/test_incremental_per_repository.py b/tests/test_incremental_per_repository.py index f1fd67a..7ad5ddc 100644 --- a/tests/test_incremental_per_repository.py +++ b/tests/test_incremental_per_repository.py @@ -15,6 +15,9 @@ def _repo(name, updated_at, pushed_at=None): "private": False, "fork": False, "has_wiki": False, + "has_issues": True, + "has_pull_requests": True, + "has_discussions": True, "updated_at": updated_at, "pushed_at": pushed_at, } diff --git a/tests/test_skip_assets_on.py b/tests/test_skip_assets_on.py index 519750e..d3fd743 100644 --- a/tests/test_skip_assets_on.py +++ b/tests/test_skip_assets_on.py @@ -22,6 +22,9 @@ def _create_mock_repository(self, name="test-repo", owner="testuser"): "private": False, "fork": False, "has_wiki": False, + "has_issues": True, + "has_pull_requests": True, + "has_discussions": True, } def _create_mock_release(self, tag="v1.0.0"):