From a504c6aed40e34331ee8b7c1fcad0154de99fe2f Mon Sep 17 00:00:00 2001 From: Igor Bernstein Date: Tue, 17 Nov 2026 18:53:10 +0000 Subject: [PATCH 1/7] ci(bigtable): extract StaleResourceCleaner into nightly Kokoro job --- .kokoro/build.sh | 15 + .kokoro/nightly/bigtable-cleanup.cfg | 37 + java-bigtable/google-cloud-bigtable/pom.xml | 20 +- .../env/StaleResourceCleaner.java | 425 +++++++++ .../env/StaleResourceCleanerTest.java | 894 ++++++++++++++++++ .../test_helpers/env/TestEnvRule.java | 279 +----- 6 files changed, 1382 insertions(+), 288 deletions(-) create mode 100644 .kokoro/nightly/bigtable-cleanup.cfg create mode 100644 java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/test_helpers/env/StaleResourceCleaner.java create mode 100644 java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/test_helpers/env/StaleResourceCleanerTest.java diff --git a/.kokoro/build.sh b/.kokoro/build.sh index ecdd0a3ea514..24e31496f9cd 100755 --- a/.kokoro/build.sh +++ b/.kokoro/build.sh @@ -169,6 +169,21 @@ case ${JOB_TYPE} in printf "Finished integration tests for modules:\n%s\n" "${BUILD_SUBDIR}" fi ;; + bigtable-cleanup) + echo "Compiling and building all modules for ${BUILD_SUBDIR}" + install_modules "${BUILD_SUBDIR}" + echo "Running in subdir: ${BUILD_SUBDIR}" + pushd "${BUILD_SUBDIR}" + echo "INTEGRATION_TEST_ARGS: ${INTEGRATION_TEST_ARGS}" + mvn test-compile exec:java -pl google-cloud-bigtable -Pquick-build \ + ${INTEGRATION_TEST_ARGS} \ + -B -ntp \ + -DtrimStackTrace=false \ + -Dorg.slf4j.simpleLogger.showDateTime=true \ + -Dorg.slf4j.simpleLogger.dateTimeFormat=HH:mm:ss:SSS + RETURN_CODE=$? + popd + ;; graalvm-presubmit) generate_graalvm_presubmit_modules_list if [[ "$(release_please_snapshot_pull_request)" == "true" ]]; then diff --git a/.kokoro/nightly/bigtable-cleanup.cfg b/.kokoro/nightly/bigtable-cleanup.cfg new file mode 100644 index 000000000000..9060f5ddd921 --- /dev/null +++ b/.kokoro/nightly/bigtable-cleanup.cfg @@ -0,0 +1,37 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Configure the docker image for kokoro-trampoline. +env_vars: { + key: "TRAMPOLINE_IMAGE" + value: "gcr.io/cloud-devrel-kokoro-resources/java11" +} + +env_vars: { + key: "JOB_TYPE" + value: "bigtable-cleanup" +} + +env_vars: { + key: "GOOGLE_CLOUD_PROJECT" + value: "gcloud-devel" +} + +env_vars: { + key: "GOOGLE_APPLICATION_CREDENTIALS" + value: "secret_manager/java-it-service-account" +} + +env_vars: { + key: "SECRET_MANAGER_KEYS" + value: "java-it-service-account" +} + +env_vars: { + key: "BUILD_SUBDIR" + value: "java-bigtable" +} + +env_vars: { + key: "INTEGRATION_TEST_ARGS" + value: "-Dbigtable.project=gcloud-devel -Dbigtable.instances=google-cloud-bigtable,google-cloud-bigtable-session" +} diff --git a/java-bigtable/google-cloud-bigtable/pom.xml b/java-bigtable/google-cloud-bigtable/pom.xml index b22256648258..b5018f1a0730 100644 --- a/java-bigtable/google-cloud-bigtable/pom.xml +++ b/java-bigtable/google-cloud-bigtable/pom.xml @@ -485,7 +485,6 @@ enable-verbose-grpc-logs - true src/test/resources/logging-verbose.properties @@ -527,8 +526,6 @@ emulator - ${bigtable.enable-grpc-logs} - ${project.build.directory}/test-grpc-logs/emulator-it com.google.cloud.bigtable.**.it.*IT @@ -564,8 +561,6 @@ ${bigtable.cfe-data-endpoint} ${bigtable.cfe-admin-endpoint} ${bigtable.cfe-jwt-audience} - ${bigtable.enable-grpc-logs} - ${project.build.directory}/test-grpc-logs/prod-it com.google.cloud.bigtable.**.it.*IT @@ -608,8 +603,6 @@ ${bigtable.directpath-data-endpoint} ${bigtable.directpath-admin-endpoint} ${bigtable.directpath-jwt-audience} - ${bigtable.enable-grpc-logs} - ${project.build.directory}/test-grpc-logs/directpath-it REQUIRE_DIRECT_PATH @@ -650,8 +643,6 @@ ${bigtable.cfe-data-endpoint} ${bigtable.cfe-admin-endpoint} ${bigtable.cfe-jwt-audience} - ${bigtable.enable-grpc-logs} - ${project.build.directory}/test-grpc-logs/cfe-it REQUIRE_CFE @@ -689,8 +680,6 @@ ${bigtable.directpath-data-endpoint} ${bigtable.directpath-admin-endpoint} ${bigtable.directpath-jwt-audience} - ${bigtable.enable-grpc-logs} - ${project.build.directory}/test-grpc-logs/directpath-ipv4only-it REQUIRE_DIRECT_PATH_IPV4 @@ -818,6 +807,15 @@ + + org.codehaus.mojo + exec-maven-plugin + + com.google.cloud.bigtable.test_helpers.env.StaleResourceCleaner + test + + + org.codehaus.mojo flatten-maven-plugin diff --git a/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/test_helpers/env/StaleResourceCleaner.java b/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/test_helpers/env/StaleResourceCleaner.java new file mode 100644 index 000000000000..a8192b7b8661 --- /dev/null +++ b/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/test_helpers/env/StaleResourceCleaner.java @@ -0,0 +1,425 @@ +/* + * Copyright 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.cloud.bigtable.test_helpers.env; + +import com.google.api.core.ApiFuture; +import com.google.api.gax.rpc.NotFoundException; +import com.google.cloud.bigtable.admin.v2.BigtableInstanceAdminClient; +import com.google.cloud.bigtable.admin.v2.BigtableInstanceAdminSettings; +import com.google.cloud.bigtable.admin.v2.BigtableTableAdminClient; +import com.google.cloud.bigtable.admin.v2.BigtableTableAdminSettings; +import com.google.cloud.bigtable.admin.v2.models.AppProfile; +import com.google.cloud.bigtable.admin.v2.models.Cluster; +import com.google.cloud.bigtable.admin.v2.models.Instance; +import com.google.cloud.bigtable.admin.v2.models.LogicalView; +import com.google.cloud.bigtable.admin.v2.models.MaterializedView; +import com.google.cloud.bigtable.admin.v2.models.Table; +import com.google.cloud.bigtable.admin.v2.models.UpdateAuthorizedViewRequest; +import com.google.cloud.bigtable.admin.v2.models.UpdateLogicalViewRequest; +import com.google.cloud.bigtable.admin.v2.models.UpdateMaterializedViewRequest; +import com.google.cloud.bigtable.admin.v2.models.UpdateTableRequest; +import com.google.common.base.Splitter; +import com.google.common.base.Strings; +import java.io.IOException; +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.ExecutionException; +import java.util.logging.Level; +import java.util.logging.Logger; +import javax.annotation.Nullable; + +/** + * Standalone utility to clean up stale test resources older than 24 hours across shared Bigtable + * instances and dynamically created test instances in a project. + */ +public class StaleResourceCleaner { + + private static final Logger LOGGER = Logger.getLogger(StaleResourceCleaner.class.getName()); + + static final String PROJECT_PROPERTY_NAME = "bigtable.project"; + static final String INSTANCES_PROPERTY_NAME = "bigtable.instances"; + static final String INSTANCE_PROPERTY_NAME = "bigtable.instance"; + static final String ADMIN_ENDPOINT_PROPERTY_NAME = "bigtable.admin-endpoint"; + static final String CFE_ADMIN_ENDPOINT_PROPERTY_NAME = "bigtable.cfe-admin-endpoint"; + + @FunctionalInterface + interface TableAdminFactory { + BigtableTableAdminClient createForInstance(String instanceId) throws IOException; + } + + private final BigtableInstanceAdminClient instanceAdmin; + private final TableAdminFactory tableAdminFactory; + private final List sharedInstanceIds; + + StaleResourceCleaner( + BigtableInstanceAdminClient instanceAdmin, + TableAdminFactory tableAdminFactory, + List sharedInstanceIds) { + this.instanceAdmin = instanceAdmin; + this.tableAdminFactory = tableAdminFactory; + this.sharedInstanceIds = sharedInstanceIds; + } + + public static void main(String[] args) + throws IOException, ExecutionException, InterruptedException { + String projectId = System.getProperty(PROJECT_PROPERTY_NAME); + if (Strings.isNullOrEmpty(projectId)) { + throw new IllegalArgumentException( + "Missing required system property: " + PROJECT_PROPERTY_NAME); + } + + List sharedInstanceIds = + parseInstanceIds( + System.getProperty(INSTANCES_PROPERTY_NAME), + System.getProperty(INSTANCE_PROPERTY_NAME)); + + String adminEndpoint = + resolveAdminEndpoint( + System.getProperty(ADMIN_ENDPOINT_PROPERTY_NAME), + System.getProperty(CFE_ADMIN_ENDPOINT_PROPERTY_NAME)); + + BigtableInstanceAdminSettings.Builder instanceAdminSettings = + BigtableInstanceAdminSettings.newBuilder().setProjectId(projectId); + if (!Strings.isNullOrEmpty(adminEndpoint)) { + instanceAdminSettings.stubSettings().setEndpoint(adminEndpoint); + } + + TableAdminFactory tableAdminFactory = + instanceId -> { + BigtableTableAdminSettings.Builder tableAdminSettings = + BigtableTableAdminSettings.newBuilder() + .setProjectId(projectId) + .setInstanceId(instanceId); + if (!Strings.isNullOrEmpty(adminEndpoint)) { + tableAdminSettings.stubSettings().setEndpoint(adminEndpoint); + } + return BigtableTableAdminClient.create(tableAdminSettings.build()); + }; + + try (BigtableInstanceAdminClient instanceAdmin = + BigtableInstanceAdminClient.create(instanceAdminSettings.build())) { + StaleResourceCleaner cleaner = + new StaleResourceCleaner(instanceAdmin, tableAdminFactory, sharedInstanceIds); + cleaner.cleanUp(); + } + } + + static List parseInstanceIds( + @Nullable String instancesProp, @Nullable String instanceProp) { + String raw = !Strings.isNullOrEmpty(instancesProp) ? instancesProp : instanceProp; + if (Strings.isNullOrEmpty(raw)) { + throw new IllegalArgumentException( + "Missing required system property: " + + INSTANCES_PROPERTY_NAME + + " or " + + INSTANCE_PROPERTY_NAME); + } + List instanceIds = Splitter.on(',').trimResults().omitEmptyStrings().splitToList(raw); + if (instanceIds.isEmpty()) { + throw new IllegalArgumentException( + "No instance IDs found in system property: " + + INSTANCES_PROPERTY_NAME + + " or " + + INSTANCE_PROPERTY_NAME); + } + return instanceIds; + } + + static String resolveAdminEndpoint( + @Nullable String adminEndpoint, @Nullable String cfeAdminEndpoint) { + if (!Strings.isNullOrEmpty(adminEndpoint)) { + return adminEndpoint; + } + if (!Strings.isNullOrEmpty(cfeAdminEndpoint)) { + return cfeAdminEndpoint; + } + return ""; + } + + static boolean isStale(String resourceId, String stalePrefix) { + return resourceId.startsWith(PrefixGenerator.PREFIX) && resourceId.compareTo(stalePrefix) < 0; + } + + public void cleanUp() throws IOException, ExecutionException, InterruptedException { + String stalePrefix = PrefixGenerator.newTimePrefix(Instant.now().minus(1, ChronoUnit.DAYS)); + cleanUp(stalePrefix); + } + + void cleanUp(String stalePrefix) throws IOException, ExecutionException, InterruptedException { + LOGGER.info( + "Starting stale resource cleanup with stalePrefix=" + + stalePrefix + + " for shared instances=" + + sharedInstanceIds); + for (String instanceId : sharedInstanceIds) { + cleanUpSharedInstance(instanceId, stalePrefix); + } + cleanUpStaleInstances(stalePrefix); + } + + void cleanUpSharedInstance(String instanceId, String stalePrefix) + throws IOException, ExecutionException, InterruptedException { + LOGGER.info("Cleaning up shared instance: " + instanceId); + try (BigtableTableAdminClient tableAdmin = tableAdminFactory.createForInstance(instanceId)) { + cleanUpStaleMaterializedViews(instanceId, stalePrefix); + cleanUpStaleLogicalViews(instanceId, stalePrefix); + cleanUpTablesAndAuthorizedViews(tableAdmin, stalePrefix); + cleanUpStaleAppProfiles(instanceId, stalePrefix); + cleanUpClustersAndBackups(instanceId, tableAdmin, stalePrefix); + } + } + + private void cleanUpStaleMaterializedViews(String instanceId, String stalePrefix) { + for (MaterializedView view : instanceAdmin.listMaterializedViews(instanceId)) { + if (!isStale(view.getId(), stalePrefix)) { + continue; + } + LOGGER.info( + "Deleting stale materialized view: " + view.getId() + " in instance " + instanceId); + unprotectAndDeleteMaterializedView(instanceId, view); + } + } + + private void unprotectAndDeleteMaterializedView(String instanceId, MaterializedView view) { + try { + if (view.isDeletionProtected()) { + instanceAdmin.updateMaterializedView( + UpdateMaterializedViewRequest.of(instanceId, view.getId()) + .setDeletionProtection(false)); + } + instanceAdmin.deleteMaterializedView(instanceId, view.getId()); + } catch (NotFoundException ignored) { + // nothing to clean up, the materialized view was already deleted + } + } + + private void cleanUpStaleLogicalViews(String instanceId, String stalePrefix) { + for (LogicalView view : instanceAdmin.listLogicalViews(instanceId)) { + if (!isStale(view.getId(), stalePrefix)) { + continue; + } + LOGGER.info("Deleting stale logical view: " + view.getId() + " in instance " + instanceId); + unprotectAndDeleteLogicalView(instanceId, view); + } + } + + private void unprotectAndDeleteLogicalView(String instanceId, LogicalView view) { + try { + if (view.isDeletionProtected()) { + instanceAdmin.updateLogicalView( + UpdateLogicalViewRequest.of(instanceId, view.getId()).setDeletionProtection(false)); + } + instanceAdmin.deleteLogicalView(instanceId, view.getId()); + } catch (NotFoundException ignored) { + // nothing to clean up, the logical view was already deleted + } + } + + private void cleanUpTablesAndAuthorizedViews( + BigtableTableAdminClient tableAdmin, String stalePrefix) { + LOGGER.info("Start cleaning up stale tables with stalePrefix=" + stalePrefix); + for (String tableId : tableAdmin.listTables()) { + if (!tableId.startsWith(PrefixGenerator.PREFIX)) { + cleanUpStaleAuthorizedViewsOnPersistentTable(tableAdmin, tableId, stalePrefix); + continue; + } + if (isStale(tableId, stalePrefix)) { + LOGGER.info("Preparing stale table for delete: " + tableId); + try { + prepTableForDelete(tableAdmin, tableId); + LOGGER.info("Deleting stale table: " + tableId); + tableAdmin.deleteTable(tableId); + } catch (NotFoundException e) { + LOGGER.log(Level.WARNING, "Deleting stale table failed: " + tableId, e); + } + } + } + } + + private void prepTableForDelete(BigtableTableAdminClient tableAdmin, String tableId) { + Table table = tableAdmin.getTable(tableId); + if (table.isDeletionProtected() || table.getChangeStreamRetention() != null) { + tableAdmin.updateTable( + UpdateTableRequest.of(tableId) + .setDeletionProtection(false) + .disableChangeStreamRetention()); + } + + for (String viewId : tableAdmin.listAuthorizedViews(tableId)) { + try { + tableAdmin.updateAuthorizedView( + UpdateAuthorizedViewRequest.of(tableId, viewId).setDeletionProtection(false)); + } catch (NotFoundException ignored) { + // nothing to clean up, the view was already deleted + } + } + } + + private void cleanUpStaleAuthorizedViewsOnPersistentTable( + BigtableTableAdminClient tableAdmin, String tableId, String stalePrefix) { + for (String viewId : tableAdmin.listAuthorizedViews(tableId)) { + if (!isStale(viewId, stalePrefix)) { + continue; + } + LOGGER.info("Deleting stale authorized view: " + viewId + " on persistent table: " + tableId); + try { + tableAdmin.updateAuthorizedView( + UpdateAuthorizedViewRequest.of(tableId, viewId).setDeletionProtection(false)); + tableAdmin.deleteAuthorizedView(tableId, viewId); + } catch (NotFoundException ignored) { + // nothing to clean up, the view was already deleted + } + } + } + + private void cleanUpStaleAppProfiles(String instanceId, String stalePrefix) { + for (AppProfile appProfile : instanceAdmin.listAppProfiles(instanceId)) { + if (!isStale(appProfile.getId(), stalePrefix)) { + continue; + } + try { + instanceAdmin.deleteAppProfile(instanceId, appProfile.getId(), true); + } catch (NotFoundException ignored) { + // nothing to clean up, the app profile was already deleted + } + } + } + + private void cleanUpClustersAndBackups( + String instanceId, BigtableTableAdminClient tableAdmin, String stalePrefix) + throws ExecutionException, InterruptedException { + for (Cluster cluster : instanceAdmin.listClusters(instanceId)) { + if (!cluster.getId().startsWith(PrefixGenerator.PREFIX)) { + try { + deleteStaleBackups(tableAdmin, cluster.getId(), stalePrefix); + } catch (NotFoundException ignored) { + // nothing to clean up, the cluster or backup was already deleted + } + continue; + } + if (!isStale(cluster.getId(), stalePrefix)) { + continue; + } + + try { + deleteAllBackups(tableAdmin, cluster.getId()); + } catch (NotFoundException ignored) { + // nothing to clean up, the backup was already deleted + } + try { + instanceAdmin.deleteCluster(instanceId, cluster.getId()); + } catch (NotFoundException ignored) { + // nothing to clean up, the cluster was already deleted + } + } + } + + private void cleanUpStaleInstances(String stalePrefix) + throws IOException, ExecutionException, InterruptedException { + for (Instance instance : instanceAdmin.listInstances()) { + if (!isStale(instance.getId(), stalePrefix)) { + continue; + } + prepInstanceForDelete(instance.getId()); + try { + deleteInstance(instance.getId()); + } catch (NotFoundException ignored) { + // nothing to clean up, the instance was already deleted + } + } + } + + private void prepInstanceForDelete(String instanceId) { + for (MaterializedView materializedView : instanceAdmin.listMaterializedViews(instanceId)) { + unprotectAndDeleteMaterializedView(instanceId, materializedView); + } + for (LogicalView logicalView : instanceAdmin.listLogicalViews(instanceId)) { + unprotectAndDeleteLogicalView(instanceId, logicalView); + } + } + + private void deleteInstance(String instanceId) + throws IOException, ExecutionException, InterruptedException { + try (BigtableTableAdminClient tableAdmin = tableAdminFactory.createForInstance(instanceId)) { + List clusters = instanceAdmin.listClusters(instanceId); + + boolean isFirstCluster = true; + + for (Cluster cluster : clusters) { + deleteAllBackups(tableAdmin, cluster.getId()); + // Skip the first cluster so that it can be deleted by deleteInstance (instances can't exist + // without clusters) + if (!isFirstCluster) { + try { + instanceAdmin.deleteCluster(instanceId, cluster.getId()); + } catch (NotFoundException ignored) { + // nothing to clean up, the cluster was already deleted + } + } + isFirstCluster = false; + } + } + + try { + instanceAdmin.deleteInstance(instanceId); + } catch (NotFoundException ignored) { + // nothing to clean up, the instance was already deleted + } + } + + private void deleteAllBackups(BigtableTableAdminClient tableAdmin, String clusterId) + throws ExecutionException, InterruptedException { + List> futures = new ArrayList<>(); + + for (String backupId : tableAdmin.listBackups(clusterId)) { + ApiFuture f = tableAdmin.deleteBackupAsync(clusterId, backupId); + futures.add(f); + } + + awaitBackupDeletions(futures); + } + + private void deleteStaleBackups( + BigtableTableAdminClient tableAdmin, String clusterId, String stalePrefix) + throws ExecutionException, InterruptedException { + List> futures = new ArrayList<>(); + + for (String backupId : tableAdmin.listBackups(clusterId)) { + if (isStale(backupId, stalePrefix)) { + ApiFuture f = tableAdmin.deleteBackupAsync(clusterId, backupId); + futures.add(f); + } + } + + awaitBackupDeletions(futures); + } + + private void awaitBackupDeletions(List> futures) + throws ExecutionException, InterruptedException { + for (ApiFuture future : futures) { + try { + future.get(); + } catch (ExecutionException e) { + if (e.getCause() instanceof NotFoundException) { + continue; + } + throw e; + } + } + } +} diff --git a/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/test_helpers/env/StaleResourceCleanerTest.java b/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/test_helpers/env/StaleResourceCleanerTest.java new file mode 100644 index 000000000000..e3d6d97ba4fb --- /dev/null +++ b/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/test_helpers/env/StaleResourceCleanerTest.java @@ -0,0 +1,894 @@ +/* + * Copyright 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.cloud.bigtable.test_helpers.env; + +import static com.google.common.truth.Truth.assertThat; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.inOrder; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; +import static org.mockito.Mockito.withSettings; + +import com.google.api.core.ApiFutures; +import com.google.api.gax.core.BackgroundResource; +import com.google.api.gax.grpc.GrpcStatusCode; +import com.google.api.gax.longrunning.OperationFuture; +import com.google.api.gax.longrunning.OperationFutures; +import com.google.api.gax.longrunning.OperationSnapshot; +import com.google.api.gax.rpc.OperationCallable; +import com.google.api.gax.rpc.UnaryCallable; +import com.google.api.gax.rpc.testing.FakeOperationSnapshot; +import com.google.bigtable.admin.v2.AppProfile; +import com.google.bigtable.admin.v2.AuthorizedView; +import com.google.bigtable.admin.v2.Backup; +import com.google.bigtable.admin.v2.ChangeStreamConfig; +import com.google.bigtable.admin.v2.Cluster; +import com.google.bigtable.admin.v2.DeleteAppProfileRequest; +import com.google.bigtable.admin.v2.DeleteAuthorizedViewRequest; +import com.google.bigtable.admin.v2.DeleteBackupRequest; +import com.google.bigtable.admin.v2.DeleteClusterRequest; +import com.google.bigtable.admin.v2.DeleteInstanceRequest; +import com.google.bigtable.admin.v2.DeleteLogicalViewRequest; +import com.google.bigtable.admin.v2.DeleteMaterializedViewRequest; +import com.google.bigtable.admin.v2.DeleteTableRequest; +import com.google.bigtable.admin.v2.GetTableRequest; +import com.google.bigtable.admin.v2.Instance; +import com.google.bigtable.admin.v2.ListAppProfilesRequest; +import com.google.bigtable.admin.v2.ListAuthorizedViewsRequest; +import com.google.bigtable.admin.v2.ListBackupsRequest; +import com.google.bigtable.admin.v2.ListClustersRequest; +import com.google.bigtable.admin.v2.ListClustersResponse; +import com.google.bigtable.admin.v2.ListInstancesRequest; +import com.google.bigtable.admin.v2.ListInstancesResponse; +import com.google.bigtable.admin.v2.ListLogicalViewsRequest; +import com.google.bigtable.admin.v2.ListMaterializedViewsRequest; +import com.google.bigtable.admin.v2.ListTablesRequest; +import com.google.bigtable.admin.v2.LogicalView; +import com.google.bigtable.admin.v2.MaterializedView; +import com.google.bigtable.admin.v2.Table; +import com.google.bigtable.admin.v2.Table.View; +import com.google.bigtable.admin.v2.UpdateAuthorizedViewMetadata; +import com.google.bigtable.admin.v2.UpdateAuthorizedViewRequest; +import com.google.bigtable.admin.v2.UpdateLogicalViewMetadata; +import com.google.bigtable.admin.v2.UpdateLogicalViewRequest; +import com.google.bigtable.admin.v2.UpdateMaterializedViewMetadata; +import com.google.bigtable.admin.v2.UpdateMaterializedViewRequest; +import com.google.bigtable.admin.v2.UpdateTableMetadata; +import com.google.bigtable.admin.v2.UpdateTableRequest; +import com.google.cloud.bigtable.admin.v2.BaseBigtableInstanceAdminClient.ListAppProfilesPage; +import com.google.cloud.bigtable.admin.v2.BaseBigtableInstanceAdminClient.ListAppProfilesPagedResponse; +import com.google.cloud.bigtable.admin.v2.BaseBigtableInstanceAdminClient.ListLogicalViewsPage; +import com.google.cloud.bigtable.admin.v2.BaseBigtableInstanceAdminClient.ListLogicalViewsPagedResponse; +import com.google.cloud.bigtable.admin.v2.BaseBigtableInstanceAdminClient.ListMaterializedViewsPage; +import com.google.cloud.bigtable.admin.v2.BaseBigtableInstanceAdminClient.ListMaterializedViewsPagedResponse; +import com.google.cloud.bigtable.admin.v2.BaseBigtableTableAdminClient.ListAuthorizedViewsPage; +import com.google.cloud.bigtable.admin.v2.BaseBigtableTableAdminClient.ListAuthorizedViewsPagedResponse; +import com.google.cloud.bigtable.admin.v2.BaseBigtableTableAdminClient.ListBackupsPage; +import com.google.cloud.bigtable.admin.v2.BaseBigtableTableAdminClient.ListBackupsPagedResponse; +import com.google.cloud.bigtable.admin.v2.BaseBigtableTableAdminClient.ListTablesPage; +import com.google.cloud.bigtable.admin.v2.BaseBigtableTableAdminClient.ListTablesPagedResponse; +import com.google.cloud.bigtable.admin.v2.BigtableInstanceAdminClient; +import com.google.cloud.bigtable.admin.v2.BigtableTableAdminClient; +import com.google.cloud.bigtable.admin.v2.internal.NameUtil; +import com.google.cloud.bigtable.admin.v2.stub.BigtableInstanceAdminStub; +import com.google.cloud.bigtable.admin.v2.stub.EnhancedBigtableTableAdminStub; +import com.google.cloud.bigtable.admin.v2.stub.GrpcBigtableTableAdminStub; +import com.google.common.collect.ImmutableList; +import com.google.protobuf.Duration; +import com.google.protobuf.Empty; +import com.google.protobuf.FieldMask; +import io.grpc.Status.Code; +import java.lang.reflect.Field; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.InOrder; + +class StaleResourceCleanerTest { + + private static final String PROJECT_ID = "test-project"; + private static final String SHARED_INSTANCE_ID = "google-cloud-bigtable"; + private static final String STALE_PREFIX = "temp-0850000000"; + private static final String STALE_ID_PREFIX = "temp-0800000001-"; + private static final String FRESH_ID_PREFIX = "temp-0899999999-"; + + private BigtableInstanceAdminStub mockInstanceStub; + private BigtableInstanceAdminClient instanceAdmin; + private Map tableStubsByInstance; + + private UnaryCallable + mockListMaterializedViewsCallable; + private OperationCallable< + UpdateMaterializedViewRequest, MaterializedView, UpdateMaterializedViewMetadata> + mockUpdateMaterializedViewCallable; + private UnaryCallable mockDeleteMaterializedViewCallable; + + private UnaryCallable + mockListLogicalViewsCallable; + private OperationCallable + mockUpdateLogicalViewCallable; + private UnaryCallable mockDeleteLogicalViewCallable; + + private UnaryCallable + mockListAppProfilesCallable; + private UnaryCallable mockDeleteAppProfileCallable; + + private UnaryCallable mockListClustersCallable; + private UnaryCallable mockDeleteClusterCallable; + + private UnaryCallable mockListInstancesCallable; + private UnaryCallable mockDeleteInstanceCallable; + + @SuppressWarnings("unchecked") + @BeforeEach + void setUp() { + mockInstanceStub = mock(BigtableInstanceAdminStub.class, withSettings().withoutAnnotations()); + instanceAdmin = BigtableInstanceAdminClient.create(PROJECT_ID, mockInstanceStub); + tableStubsByInstance = new HashMap<>(); + + mockListMaterializedViewsCallable = + mock(UnaryCallable.class, withSettings().withoutAnnotations()); + mockUpdateMaterializedViewCallable = + mock(OperationCallable.class, withSettings().withoutAnnotations()); + mockDeleteMaterializedViewCallable = + mock(UnaryCallable.class, withSettings().withoutAnnotations()); + + mockListLogicalViewsCallable = mock(UnaryCallable.class, withSettings().withoutAnnotations()); + mockUpdateLogicalViewCallable = + mock(OperationCallable.class, withSettings().withoutAnnotations()); + mockDeleteLogicalViewCallable = mock(UnaryCallable.class, withSettings().withoutAnnotations()); + + mockListAppProfilesCallable = mock(UnaryCallable.class, withSettings().withoutAnnotations()); + mockDeleteAppProfileCallable = mock(UnaryCallable.class, withSettings().withoutAnnotations()); + + mockListClustersCallable = mock(UnaryCallable.class, withSettings().withoutAnnotations()); + mockDeleteClusterCallable = mock(UnaryCallable.class, withSettings().withoutAnnotations()); + + mockListInstancesCallable = mock(UnaryCallable.class, withSettings().withoutAnnotations()); + mockDeleteInstanceCallable = mock(UnaryCallable.class, withSettings().withoutAnnotations()); + + when(mockInstanceStub.listMaterializedViewsPagedCallable()) + .thenReturn(mockListMaterializedViewsCallable); + when(mockInstanceStub.updateMaterializedViewOperationCallable()) + .thenReturn(mockUpdateMaterializedViewCallable); + when(mockInstanceStub.deleteMaterializedViewCallable()) + .thenReturn(mockDeleteMaterializedViewCallable); + + when(mockInstanceStub.listLogicalViewsPagedCallable()).thenReturn(mockListLogicalViewsCallable); + when(mockInstanceStub.updateLogicalViewOperationCallable()) + .thenReturn(mockUpdateLogicalViewCallable); + when(mockInstanceStub.deleteLogicalViewCallable()).thenReturn(mockDeleteLogicalViewCallable); + + when(mockInstanceStub.listAppProfilesPagedCallable()).thenReturn(mockListAppProfilesCallable); + when(mockInstanceStub.deleteAppProfileCallable()).thenReturn(mockDeleteAppProfileCallable); + + when(mockInstanceStub.listClustersCallable()).thenReturn(mockListClustersCallable); + when(mockInstanceStub.deleteClusterCallable()).thenReturn(mockDeleteClusterCallable); + + when(mockInstanceStub.listInstancesCallable()).thenReturn(mockListInstancesCallable); + when(mockInstanceStub.deleteInstanceCallable()).thenReturn(mockDeleteInstanceCallable); + + when(mockDeleteMaterializedViewCallable.futureCall(any())) + .thenReturn(ApiFutures.immediateFuture(Empty.getDefaultInstance())); + when(mockDeleteLogicalViewCallable.futureCall(any())) + .thenReturn(ApiFutures.immediateFuture(Empty.getDefaultInstance())); + when(mockDeleteAppProfileCallable.futureCall(any())) + .thenReturn(ApiFutures.immediateFuture(Empty.getDefaultInstance())); + when(mockDeleteClusterCallable.futureCall(any())) + .thenReturn(ApiFutures.immediateFuture(Empty.getDefaultInstance())); + when(mockDeleteInstanceCallable.futureCall(any())) + .thenReturn(ApiFutures.immediateFuture(Empty.getDefaultInstance())); + } + + @Test + void testParseInstanceIds() { + assertThat( + StaleResourceCleaner.parseInstanceIds( + "google-cloud-bigtable, google-cloud-bigtable-session", "ignored-fallback")) + .containsExactly("google-cloud-bigtable", "google-cloud-bigtable-session") + .inOrder(); + + assertThat(StaleResourceCleaner.parseInstanceIds(null, "google-cloud-bigtable")) + .containsExactly("google-cloud-bigtable"); + + assertThat(StaleResourceCleaner.parseInstanceIds("", "google-cloud-bigtable")) + .containsExactly("google-cloud-bigtable"); + + assertThrows( + IllegalArgumentException.class, () -> StaleResourceCleaner.parseInstanceIds(null, null)); + assertThrows( + IllegalArgumentException.class, () -> StaleResourceCleaner.parseInstanceIds(" , ", "")); + } + + @Test + void testResolveAdminEndpoint() { + assertThat( + StaleResourceCleaner.resolveAdminEndpoint( + "test-bigtableadmin.sandbox.googleapis.com:443", + "staging-bigtableadmin.sandbox.googleapis.com:443")) + .isEqualTo("test-bigtableadmin.sandbox.googleapis.com:443"); + + assertThat( + StaleResourceCleaner.resolveAdminEndpoint( + "", "staging-bigtableadmin.sandbox.googleapis.com:443")) + .isEqualTo("staging-bigtableadmin.sandbox.googleapis.com:443"); + + assertThat(StaleResourceCleaner.resolveAdminEndpoint(null, null)).isEmpty(); + } + + @Test + @SuppressWarnings("unchecked") + void testCleanUpSharedInstance_viewsTablesAndAuthorizedViews() throws Exception { + EnhancedBigtableTableAdminStub tableStub = createTableStubForInstance(SHARED_INSTANCE_ID); + + String staleProtectedMv = STALE_ID_PREFIX + "mv-protected"; + String staleUnprotectedMv = STALE_ID_PREFIX + "mv-unprotected"; + String freshMv = FRESH_ID_PREFIX + "mv-fresh"; + String permMv = "permanent-mv"; + + mockMaterializedViews( + SHARED_INSTANCE_ID, + ImmutableList.of( + buildMaterializedView(SHARED_INSTANCE_ID, staleProtectedMv, true), + buildMaterializedView(SHARED_INSTANCE_ID, staleUnprotectedMv, false), + buildMaterializedView(SHARED_INSTANCE_ID, freshMv, true), + buildMaterializedView(SHARED_INSTANCE_ID, permMv, false))); + + mockOperationFuture( + mockUpdateMaterializedViewCallable, + buildMaterializedView(SHARED_INSTANCE_ID, staleProtectedMv, false)); + + String staleProtectedLv = STALE_ID_PREFIX + "lv-protected"; + String staleUnprotectedLv = STALE_ID_PREFIX + "lv-unprotected"; + String freshLv = FRESH_ID_PREFIX + "lv-fresh"; + + mockLogicalViews( + SHARED_INSTANCE_ID, + ImmutableList.of( + buildLogicalView(SHARED_INSTANCE_ID, staleProtectedLv, true), + buildLogicalView(SHARED_INSTANCE_ID, staleUnprotectedLv, false), + buildLogicalView(SHARED_INSTANCE_ID, freshLv, true))); + + mockOperationFuture( + mockUpdateLogicalViewCallable, + buildLogicalView(SHARED_INSTANCE_ID, staleProtectedLv, false)); + + String staleTable = STALE_ID_PREFIX + "table"; + String freshTable = FRESH_ID_PREFIX + "table"; + String persistentTable = "integration-tests"; + + mockListTables( + tableStub, SHARED_INSTANCE_ID, ImmutableList.of(staleTable, freshTable, persistentTable)); + + UnaryCallable mockGetTableCallable = + mock(UnaryCallable.class, withSettings().withoutAnnotations()); + OperationCallable mockUpdateTableCallable = + mock(OperationCallable.class, withSettings().withoutAnnotations()); + UnaryCallable mockDeleteTableCallable = + mock(UnaryCallable.class, withSettings().withoutAnnotations()); + UnaryCallable + mockListAuthorizedViewsCallable = + mock(UnaryCallable.class, withSettings().withoutAnnotations()); + OperationCallable + mockUpdateAuthorizedViewCallable = + mock(OperationCallable.class, withSettings().withoutAnnotations()); + UnaryCallable mockDeleteAuthorizedViewCallable = + mock(UnaryCallable.class, withSettings().withoutAnnotations()); + + when(tableStub.getTableCallable()).thenReturn(mockGetTableCallable); + when(tableStub.updateTableOperationCallable()).thenReturn(mockUpdateTableCallable); + when(tableStub.deleteTableCallable()).thenReturn(mockDeleteTableCallable); + when(tableStub.listAuthorizedViewsPagedCallable()).thenReturn(mockListAuthorizedViewsCallable); + when(tableStub.updateAuthorizedViewOperationCallable()) + .thenReturn(mockUpdateAuthorizedViewCallable); + when(tableStub.deleteAuthorizedViewCallable()).thenReturn(mockDeleteAuthorizedViewCallable); + + Table staleTableProto = + Table.newBuilder() + .setName(NameUtil.formatTableName(PROJECT_ID, SHARED_INSTANCE_ID, staleTable)) + .setDeletionProtection(true) + .setChangeStreamConfig( + ChangeStreamConfig.newBuilder() + .setRetentionPeriod(Duration.newBuilder().setSeconds(86400))) + .build(); + when(mockGetTableCallable.futureCall( + GetTableRequest.newBuilder() + .setName(NameUtil.formatTableName(PROJECT_ID, SHARED_INSTANCE_ID, staleTable)) + .setView(View.SCHEMA_VIEW) + .build())) + .thenReturn(ApiFutures.immediateFuture(staleTableProto)); + + mockOperationFuture( + mockUpdateTableCallable, + Table.newBuilder() + .setName(NameUtil.formatTableName(PROJECT_ID, SHARED_INSTANCE_ID, staleTable)) + .setDeletionProtection(false) + .build()); + when(mockDeleteTableCallable.futureCall(any())) + .thenReturn(ApiFutures.immediateFuture(Empty.getDefaultInstance())); + + String viewOnStaleTable = "av-on-stale-table"; + mockAuthorizedViews( + mockListAuthorizedViewsCallable, + SHARED_INSTANCE_ID, + staleTable, + ImmutableList.of(viewOnStaleTable)); + + String staleAvOnPersistentTable = STALE_ID_PREFIX + "av-stale"; + String freshAvOnPersistentTable = FRESH_ID_PREFIX + "av-fresh"; + String permAvOnPersistentTable = "perm-av"; + mockAuthorizedViews( + mockListAuthorizedViewsCallable, + SHARED_INSTANCE_ID, + persistentTable, + ImmutableList.of( + staleAvOnPersistentTable, freshAvOnPersistentTable, permAvOnPersistentTable)); + + mockOperationFuture( + mockUpdateAuthorizedViewCallable, + AuthorizedView.newBuilder() + .setName( + NameUtil.formatAuthorizedViewName( + PROJECT_ID, SHARED_INSTANCE_ID, persistentTable, staleAvOnPersistentTable)) + .setSubsetView(AuthorizedView.SubsetView.getDefaultInstance()) + .setDeletionProtection(false) + .build()); + when(mockDeleteAuthorizedViewCallable.futureCall(any())) + .thenReturn(ApiFutures.immediateFuture(Empty.getDefaultInstance())); + + mockAppProfiles(SHARED_INSTANCE_ID, ImmutableList.of()); + mockClusters(SHARED_INSTANCE_ID, ImmutableList.of()); + mockInstances(ImmutableList.of()); + + StaleResourceCleaner cleaner = createCleaner(ImmutableList.of(SHARED_INSTANCE_ID)); + cleaner.cleanUp(STALE_PREFIX); + + // MaterializedViews: stale ones deleted before table deletion; only protected one updated + InOrder inOrder = + inOrder( + mockUpdateMaterializedViewCallable, + mockDeleteMaterializedViewCallable, + mockUpdateLogicalViewCallable, + mockDeleteLogicalViewCallable, + mockDeleteTableCallable); + inOrder + .verify(mockUpdateMaterializedViewCallable) + .futureCall( + UpdateMaterializedViewRequest.newBuilder() + .setMaterializedView( + MaterializedView.newBuilder() + .setName( + NameUtil.formatMaterializedViewName( + PROJECT_ID, SHARED_INSTANCE_ID, staleProtectedMv)) + .setDeletionProtection(false)) + .setUpdateMask(FieldMask.newBuilder().addPaths("deletion_protection")) + .build()); + inOrder + .verify(mockDeleteMaterializedViewCallable) + .futureCall( + DeleteMaterializedViewRequest.newBuilder() + .setName( + NameUtil.formatMaterializedViewName( + PROJECT_ID, SHARED_INSTANCE_ID, staleProtectedMv)) + .build()); + inOrder + .verify(mockDeleteMaterializedViewCallable) + .futureCall( + DeleteMaterializedViewRequest.newBuilder() + .setName( + NameUtil.formatMaterializedViewName( + PROJECT_ID, SHARED_INSTANCE_ID, staleUnprotectedMv)) + .build()); + inOrder + .verify(mockUpdateLogicalViewCallable) + .futureCall( + UpdateLogicalViewRequest.newBuilder() + .setLogicalView( + LogicalView.newBuilder() + .setName( + NameUtil.formatLogicalViewName( + PROJECT_ID, SHARED_INSTANCE_ID, staleProtectedLv)) + .setDeletionProtection(false)) + .setUpdateMask(FieldMask.newBuilder().addPaths("deletion_protection")) + .build()); + inOrder + .verify(mockDeleteLogicalViewCallable) + .futureCall( + DeleteLogicalViewRequest.newBuilder() + .setName( + NameUtil.formatLogicalViewName( + PROJECT_ID, SHARED_INSTANCE_ID, staleProtectedLv)) + .build()); + inOrder + .verify(mockDeleteLogicalViewCallable) + .futureCall( + DeleteLogicalViewRequest.newBuilder() + .setName( + NameUtil.formatLogicalViewName( + PROJECT_ID, SHARED_INSTANCE_ID, staleUnprotectedLv)) + .build()); + inOrder + .verify(mockDeleteTableCallable) + .futureCall( + DeleteTableRequest.newBuilder() + .setName(NameUtil.formatTableName(PROJECT_ID, SHARED_INSTANCE_ID, staleTable)) + .build()); + + verify(mockDeleteMaterializedViewCallable, never()) + .futureCall( + DeleteMaterializedViewRequest.newBuilder() + .setName( + NameUtil.formatMaterializedViewName(PROJECT_ID, SHARED_INSTANCE_ID, freshMv)) + .build()); + verify(mockDeleteMaterializedViewCallable, never()) + .futureCall( + DeleteMaterializedViewRequest.newBuilder() + .setName( + NameUtil.formatMaterializedViewName(PROJECT_ID, SHARED_INSTANCE_ID, permMv)) + .build()); + verify(mockDeleteLogicalViewCallable, never()) + .futureCall( + DeleteLogicalViewRequest.newBuilder() + .setName(NameUtil.formatLogicalViewName(PROJECT_ID, SHARED_INSTANCE_ID, freshLv)) + .build()); + + // Persistent table is NOT deleted, but its stale AuthorizedView is unprotected and deleted + verify(mockDeleteTableCallable, never()) + .futureCall( + DeleteTableRequest.newBuilder() + .setName(NameUtil.formatTableName(PROJECT_ID, SHARED_INSTANCE_ID, persistentTable)) + .build()); + verify(mockDeleteTableCallable, never()) + .futureCall( + DeleteTableRequest.newBuilder() + .setName(NameUtil.formatTableName(PROJECT_ID, SHARED_INSTANCE_ID, freshTable)) + .build()); + + verify(mockDeleteAuthorizedViewCallable) + .futureCall( + DeleteAuthorizedViewRequest.newBuilder() + .setName( + NameUtil.formatAuthorizedViewName( + PROJECT_ID, SHARED_INSTANCE_ID, persistentTable, staleAvOnPersistentTable)) + .build()); + verify(mockDeleteAuthorizedViewCallable, never()) + .futureCall( + DeleteAuthorizedViewRequest.newBuilder() + .setName( + NameUtil.formatAuthorizedViewName( + PROJECT_ID, SHARED_INSTANCE_ID, persistentTable, freshAvOnPersistentTable)) + .build()); + verify(mockDeleteAuthorizedViewCallable, never()) + .futureCall( + DeleteAuthorizedViewRequest.newBuilder() + .setName( + NameUtil.formatAuthorizedViewName( + PROJECT_ID, SHARED_INSTANCE_ID, persistentTable, permAvOnPersistentTable)) + .build()); + } + + @Test + @SuppressWarnings("unchecked") + void testCleanUpSharedInstance_appProfilesClustersAndBackups() throws Exception { + EnhancedBigtableTableAdminStub tableStub = createTableStubForInstance(SHARED_INSTANCE_ID); + + mockMaterializedViews(SHARED_INSTANCE_ID, ImmutableList.of()); + mockLogicalViews(SHARED_INSTANCE_ID, ImmutableList.of()); + mockListTables(tableStub, SHARED_INSTANCE_ID, ImmutableList.of()); + + String staleAp = STALE_ID_PREFIX + "ap"; + String freshAp = FRESH_ID_PREFIX + "ap"; + String defaultAp = "default"; + mockAppProfiles(SHARED_INSTANCE_ID, ImmutableList.of(staleAp, freshAp, defaultAp)); + + String primaryCluster = "google-cloud-bigtable-c0"; + String staleCluster = STALE_ID_PREFIX + "cluster"; + String freshCluster = FRESH_ID_PREFIX + "cluster"; + mockClusters(SHARED_INSTANCE_ID, ImmutableList.of(primaryCluster, staleCluster, freshCluster)); + + UnaryCallable mockListBackupsCallable = + mock(UnaryCallable.class, withSettings().withoutAnnotations()); + UnaryCallable mockDeleteBackupCallable = + mock(UnaryCallable.class, withSettings().withoutAnnotations()); + when(tableStub.listBackupsPagedCallable()).thenReturn(mockListBackupsCallable); + when(tableStub.deleteBackupCallable()).thenReturn(mockDeleteBackupCallable); + when(mockDeleteBackupCallable.futureCall(any())) + .thenReturn(ApiFutures.immediateFuture(Empty.getDefaultInstance())); + + String staleBackupInPrimary = STALE_ID_PREFIX + "backup"; + String freshBackupInPrimary = FRESH_ID_PREFIX + "backup"; + String permBackupInPrimary = "perm-backup"; + mockBackups( + mockListBackupsCallable, + SHARED_INSTANCE_ID, + primaryCluster, + ImmutableList.of(staleBackupInPrimary, freshBackupInPrimary, permBackupInPrimary)); + + String backupInStaleCluster = "any-backup-id"; + mockBackups( + mockListBackupsCallable, + SHARED_INSTANCE_ID, + staleCluster, + ImmutableList.of(backupInStaleCluster)); + + mockInstances(ImmutableList.of()); + + StaleResourceCleaner cleaner = createCleaner(ImmutableList.of(SHARED_INSTANCE_ID)); + cleaner.cleanUp(STALE_PREFIX); + + // AppProfiles: only staleAp deleted with ignoreWarnings=true + verify(mockDeleteAppProfileCallable) + .futureCall( + DeleteAppProfileRequest.newBuilder() + .setName(NameUtil.formatAppProfileName(PROJECT_ID, SHARED_INSTANCE_ID, staleAp)) + .setIgnoreWarnings(true) + .build()); + verify(mockDeleteAppProfileCallable, never()) + .futureCall( + DeleteAppProfileRequest.newBuilder() + .setName(NameUtil.formatAppProfileName(PROJECT_ID, SHARED_INSTANCE_ID, freshAp)) + .setIgnoreWarnings(true) + .build()); + + // Primary cluster: not deleted, only stale backup in it is deleted + verify(mockDeleteClusterCallable, never()) + .futureCall( + DeleteClusterRequest.newBuilder() + .setName(NameUtil.formatClusterName(PROJECT_ID, SHARED_INSTANCE_ID, primaryCluster)) + .build()); + verify(mockDeleteBackupCallable) + .futureCall( + DeleteBackupRequest.newBuilder() + .setName( + NameUtil.formatBackupName( + PROJECT_ID, SHARED_INSTANCE_ID, primaryCluster, staleBackupInPrimary)) + .build()); + verify(mockDeleteBackupCallable, never()) + .futureCall( + DeleteBackupRequest.newBuilder() + .setName( + NameUtil.formatBackupName( + PROJECT_ID, SHARED_INSTANCE_ID, primaryCluster, freshBackupInPrimary)) + .build()); + verify(mockDeleteBackupCallable, never()) + .futureCall( + DeleteBackupRequest.newBuilder() + .setName( + NameUtil.formatBackupName( + PROJECT_ID, SHARED_INSTANCE_ID, primaryCluster, permBackupInPrimary)) + .build()); + + // Stale cluster: all backups deleted, cluster deleted + verify(mockDeleteBackupCallable) + .futureCall( + DeleteBackupRequest.newBuilder() + .setName( + NameUtil.formatBackupName( + PROJECT_ID, SHARED_INSTANCE_ID, staleCluster, backupInStaleCluster)) + .build()); + verify(mockDeleteClusterCallable) + .futureCall( + DeleteClusterRequest.newBuilder() + .setName(NameUtil.formatClusterName(PROJECT_ID, SHARED_INSTANCE_ID, staleCluster)) + .build()); + } + + @Test + @SuppressWarnings("unchecked") + void testCleanUpStaleInstances() throws Exception { + String staleInstance = STALE_ID_PREFIX + "inst"; + String freshInstance = FRESH_ID_PREFIX + "inst"; + + mockInstances(ImmutableList.of(SHARED_INSTANCE_ID, staleInstance, freshInstance)); + + String mvOnStaleInst = "mv-1"; + String lvOnStaleInst = "lv-1"; + mockMaterializedViews( + staleInstance, ImmutableList.of(buildMaterializedView(staleInstance, mvOnStaleInst, true))); + mockLogicalViews( + staleInstance, ImmutableList.of(buildLogicalView(staleInstance, lvOnStaleInst, true))); + mockOperationFuture( + mockUpdateMaterializedViewCallable, + buildMaterializedView(staleInstance, mvOnStaleInst, false)); + mockOperationFuture( + mockUpdateLogicalViewCallable, buildLogicalView(staleInstance, lvOnStaleInst, false)); + + String cluster0 = "c0"; + String cluster1 = "c1"; + mockClusters(staleInstance, ImmutableList.of(cluster0, cluster1)); + + EnhancedBigtableTableAdminStub staleInstTableStub = createTableStubForInstance(staleInstance); + UnaryCallable mockListBackupsCallable = + mock(UnaryCallable.class, withSettings().withoutAnnotations()); + UnaryCallable mockDeleteBackupCallable = + mock(UnaryCallable.class, withSettings().withoutAnnotations()); + when(staleInstTableStub.listBackupsPagedCallable()).thenReturn(mockListBackupsCallable); + when(staleInstTableStub.deleteBackupCallable()).thenReturn(mockDeleteBackupCallable); + when(mockDeleteBackupCallable.futureCall(any())) + .thenReturn(ApiFutures.immediateFuture(Empty.getDefaultInstance())); + + mockBackups(mockListBackupsCallable, staleInstance, cluster0, ImmutableList.of("b0")); + mockBackups(mockListBackupsCallable, staleInstance, cluster1, ImmutableList.of("b1")); + + StaleResourceCleaner cleaner = createCleaner(ImmutableList.of()); + cleaner.cleanUp(STALE_PREFIX); + + verify(mockDeleteMaterializedViewCallable) + .futureCall( + DeleteMaterializedViewRequest.newBuilder() + .setName( + NameUtil.formatMaterializedViewName(PROJECT_ID, staleInstance, mvOnStaleInst)) + .build()); + verify(mockDeleteLogicalViewCallable) + .futureCall( + DeleteLogicalViewRequest.newBuilder() + .setName(NameUtil.formatLogicalViewName(PROJECT_ID, staleInstance, lvOnStaleInst)) + .build()); + + // Backups in both clusters deleted, only secondary cluster1 deleted before deleteInstance + verify(mockDeleteBackupCallable) + .futureCall( + DeleteBackupRequest.newBuilder() + .setName(NameUtil.formatBackupName(PROJECT_ID, staleInstance, cluster0, "b0")) + .build()); + verify(mockDeleteBackupCallable) + .futureCall( + DeleteBackupRequest.newBuilder() + .setName(NameUtil.formatBackupName(PROJECT_ID, staleInstance, cluster1, "b1")) + .build()); + verify(mockDeleteClusterCallable, never()) + .futureCall( + DeleteClusterRequest.newBuilder() + .setName(NameUtil.formatClusterName(PROJECT_ID, staleInstance, cluster0)) + .build()); + verify(mockDeleteClusterCallable) + .futureCall( + DeleteClusterRequest.newBuilder() + .setName(NameUtil.formatClusterName(PROJECT_ID, staleInstance, cluster1)) + .build()); + verify(mockDeleteInstanceCallable) + .futureCall( + DeleteInstanceRequest.newBuilder() + .setName(NameUtil.formatInstanceName(PROJECT_ID, staleInstance)) + .build()); + verify(mockDeleteInstanceCallable, never()) + .futureCall( + DeleteInstanceRequest.newBuilder() + .setName(NameUtil.formatInstanceName(PROJECT_ID, SHARED_INSTANCE_ID)) + .build()); + verify(mockDeleteInstanceCallable, never()) + .futureCall( + DeleteInstanceRequest.newBuilder() + .setName(NameUtil.formatInstanceName(PROJECT_ID, freshInstance)) + .build()); + } + + private StaleResourceCleaner createCleaner(List sharedInstanceIds) { + return new StaleResourceCleaner( + instanceAdmin, + instanceId -> + BigtableTableAdminClient.create( + PROJECT_ID, instanceId, tableStubsByInstance.get(instanceId)), + sharedInstanceIds); + } + + private EnhancedBigtableTableAdminStub createTableStubForInstance(String instanceId) + throws Exception { + EnhancedBigtableTableAdminStub stub = + mock(EnhancedBigtableTableAdminStub.class, withSettings().withoutAnnotations()); + Field backgroundResourcesField = + GrpcBigtableTableAdminStub.class.getDeclaredField("backgroundResources"); + backgroundResourcesField.setAccessible(true); + backgroundResourcesField.set(stub, mock(BackgroundResource.class)); + tableStubsByInstance.put(instanceId, stub); + return stub; + } + + private MaterializedView buildMaterializedView( + String instanceId, String viewId, boolean deletionProtected) { + return MaterializedView.newBuilder() + .setName(NameUtil.formatMaterializedViewName(PROJECT_ID, instanceId, viewId)) + .setDeletionProtection(deletionProtected) + .setQuery("SELECT 1 FROM table") + .build(); + } + + private LogicalView buildLogicalView( + String instanceId, String viewId, boolean deletionProtected) { + return LogicalView.newBuilder() + .setName(NameUtil.formatLogicalViewName(PROJECT_ID, instanceId, viewId)) + .setDeletionProtection(deletionProtected) + .setQuery("SELECT 1 FROM table") + .build(); + } + + private void mockMaterializedViews(String instanceId, List views) { + ListMaterializedViewsPage page = mock(ListMaterializedViewsPage.class); + when(page.getValues()).thenReturn(views); + when(page.hasNextPage()).thenReturn(false); + ListMaterializedViewsPagedResponse response = mock(ListMaterializedViewsPagedResponse.class); + when(response.getPage()).thenReturn(page); + when(mockListMaterializedViewsCallable.futureCall( + ListMaterializedViewsRequest.newBuilder() + .setParent(NameUtil.formatInstanceName(PROJECT_ID, instanceId)) + .build())) + .thenReturn(ApiFutures.immediateFuture(response)); + } + + private void mockLogicalViews(String instanceId, List views) { + ListLogicalViewsPage page = mock(ListLogicalViewsPage.class); + when(page.getValues()).thenReturn(views); + when(page.hasNextPage()).thenReturn(false); + ListLogicalViewsPagedResponse response = mock(ListLogicalViewsPagedResponse.class); + when(response.getPage()).thenReturn(page); + when(mockListLogicalViewsCallable.futureCall( + ListLogicalViewsRequest.newBuilder() + .setParent(NameUtil.formatInstanceName(PROJECT_ID, instanceId)) + .build())) + .thenReturn(ApiFutures.immediateFuture(response)); + } + + @SuppressWarnings("unchecked") + private void mockListTables( + EnhancedBigtableTableAdminStub tableStub, String instanceId, List tableIds) { + UnaryCallable mockListTablesCallable = + mock(UnaryCallable.class, withSettings().withoutAnnotations()); + when(tableStub.listTablesPagedCallable()).thenReturn(mockListTablesCallable); + + List tables = new ArrayList<>(); + for (String tableId : tableIds) { + tables.add( + Table.newBuilder() + .setName(NameUtil.formatTableName(PROJECT_ID, instanceId, tableId)) + .build()); + } + ListTablesPage page = mock(ListTablesPage.class); + when(page.getValues()).thenReturn(tables); + when(page.hasNextPage()).thenReturn(false); + ListTablesPagedResponse response = mock(ListTablesPagedResponse.class); + when(response.getPage()).thenReturn(page); + when(mockListTablesCallable.futureCall( + ListTablesRequest.newBuilder() + .setParent(NameUtil.formatInstanceName(PROJECT_ID, instanceId)) + .build())) + .thenReturn(ApiFutures.immediateFuture(response)); + } + + private void mockAuthorizedViews( + UnaryCallable callable, + String instanceId, + String tableId, + List viewIds) { + List views = new ArrayList<>(); + for (String viewId : viewIds) { + views.add( + AuthorizedView.newBuilder() + .setName(NameUtil.formatAuthorizedViewName(PROJECT_ID, instanceId, tableId, viewId)) + .build()); + } + ListAuthorizedViewsPage page = mock(ListAuthorizedViewsPage.class); + when(page.getValues()).thenReturn(views); + when(page.hasNextPage()).thenReturn(false); + ListAuthorizedViewsPagedResponse response = mock(ListAuthorizedViewsPagedResponse.class); + when(response.getPage()).thenReturn(page); + when(callable.futureCall( + ListAuthorizedViewsRequest.newBuilder() + .setParent(NameUtil.formatTableName(PROJECT_ID, instanceId, tableId)) + .build())) + .thenReturn(ApiFutures.immediateFuture(response)); + } + + private void mockAppProfiles(String instanceId, List appProfileIds) { + List profiles = new ArrayList<>(); + for (String id : appProfileIds) { + profiles.add( + AppProfile.newBuilder() + .setName(NameUtil.formatAppProfileName(PROJECT_ID, instanceId, id)) + .setMultiClusterRoutingUseAny( + AppProfile.MultiClusterRoutingUseAny.getDefaultInstance()) + .build()); + } + ListAppProfilesPage page = mock(ListAppProfilesPage.class); + when(page.getValues()).thenReturn(profiles); + when(page.hasNextPage()).thenReturn(false); + ListAppProfilesPagedResponse response = mock(ListAppProfilesPagedResponse.class); + when(response.getPage()).thenReturn(page); + when(mockListAppProfilesCallable.futureCall( + ListAppProfilesRequest.newBuilder() + .setParent(NameUtil.formatInstanceName(PROJECT_ID, instanceId)) + .build())) + .thenReturn(ApiFutures.immediateFuture(response)); + } + + private void mockClusters(String instanceId, List clusterIds) { + List clusters = new ArrayList<>(); + for (String clusterId : clusterIds) { + clusters.add( + Cluster.newBuilder() + .setName(NameUtil.formatClusterName(PROJECT_ID, instanceId, clusterId)) + .setLocation(NameUtil.formatLocationName(PROJECT_ID, "us-central1-b")) + .build()); + } + when(mockListClustersCallable.futureCall( + ListClustersRequest.newBuilder() + .setParent(NameUtil.formatInstanceName(PROJECT_ID, instanceId)) + .build())) + .thenReturn( + ApiFutures.immediateFuture( + ListClustersResponse.newBuilder().addAllClusters(clusters).build())); + } + + private void mockBackups( + UnaryCallable callable, + String instanceId, + String clusterId, + List backupIds) { + List backups = new ArrayList<>(); + for (String backupId : backupIds) { + backups.add( + Backup.newBuilder() + .setName(NameUtil.formatBackupName(PROJECT_ID, instanceId, clusterId, backupId)) + .build()); + } + ListBackupsPage page = mock(ListBackupsPage.class); + when(page.getValues()).thenReturn(backups); + when(page.hasNextPage()).thenReturn(false); + ListBackupsPagedResponse response = mock(ListBackupsPagedResponse.class); + when(response.getPage()).thenReturn(page); + when(callable.futureCall( + ListBackupsRequest.newBuilder() + .setParent(NameUtil.formatClusterName(PROJECT_ID, instanceId, clusterId)) + .build())) + .thenReturn(ApiFutures.immediateFuture(response)); + } + + private void mockInstances(List instanceIds) { + List instances = new ArrayList<>(); + for (String instanceId : instanceIds) { + instances.add( + Instance.newBuilder() + .setName(NameUtil.formatInstanceName(PROJECT_ID, instanceId)) + .setDisplayName(instanceId) + .build()); + } + when(mockListInstancesCallable.futureCall( + ListInstancesRequest.newBuilder() + .setParent(NameUtil.formatProjectName(PROJECT_ID)) + .build())) + .thenReturn( + ApiFutures.immediateFuture( + ListInstancesResponse.newBuilder().addAllInstances(instances).build())); + } + + private void mockOperationFuture( + OperationCallable callable, RespT response) { + OperationSnapshot snapshot = + FakeOperationSnapshot.newBuilder() + .setDone(true) + .setErrorCode(GrpcStatusCode.of(Code.OK)) + .setName("fake-op") + .setResponse(response) + .build(); + OperationFuture future = OperationFutures.immediateOperationFuture(snapshot); + when(callable.futureCall(any())).thenReturn(future); + } +} diff --git a/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/test_helpers/env/TestEnvRule.java b/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/test_helpers/env/TestEnvRule.java index dcf0938a5c8b..47d047708597 100644 --- a/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/test_helpers/env/TestEnvRule.java +++ b/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/test_helpers/env/TestEnvRule.java @@ -17,29 +17,6 @@ import static com.google.common.truth.TruthJUnit.assume; -import com.google.api.core.ApiFuture; -import com.google.api.gax.rpc.NotFoundException; -import com.google.cloud.bigtable.admin.v2.BigtableTableAdminClient; -import com.google.cloud.bigtable.admin.v2.BigtableTableAdminSettings; -import com.google.cloud.bigtable.admin.v2.models.AppProfile; -import com.google.cloud.bigtable.admin.v2.models.Cluster; -import com.google.cloud.bigtable.admin.v2.models.Instance; -import com.google.cloud.bigtable.admin.v2.models.LogicalView; -import com.google.cloud.bigtable.admin.v2.models.MaterializedView; -import com.google.cloud.bigtable.admin.v2.models.Table; -import com.google.cloud.bigtable.admin.v2.models.UpdateAuthorizedViewRequest; -import com.google.cloud.bigtable.admin.v2.models.UpdateLogicalViewRequest; -import com.google.cloud.bigtable.admin.v2.models.UpdateMaterializedViewRequest; -import com.google.cloud.bigtable.admin.v2.models.UpdateTableRequest; -import com.google.common.collect.ImmutableSet; -import java.io.IOException; -import java.time.Instant; -import java.time.temporal.ChronoUnit; -import java.util.ArrayList; -import java.util.List; -import java.util.Set; -import java.util.concurrent.ExecutionException; -import java.util.logging.Handler; import java.util.logging.Level; import java.util.logging.Logger; import org.junit.rules.TestRule; @@ -54,17 +31,11 @@ * *
    *
  • {@code emulator}: uses the cbtemulator component that can be installed by gcloud - *
  • {@code prod}: uses a pre-existing production table. The target table is defined using - * system properties listed in {@link CloudEnv} and application default credentials - *
  • {@code direct_path}: uses a pre-existing table in the direct path test environment. The - * target table is defined using system properties listed in {@link CloudEnv} and application - * default credentials + *
  • {@code cloud}: uses a pre-existing cloud instance and table. The target table is defined + * using system properties listed in {@link CloudEnv} and application default credentials *
* *

By default, {@code emulator} will be used - * - *

Also, when the system property {@code bigtable.grpc-log-dir} is set, it will enable fine - * grained gRPC logging to the configured path. */ public class TestEnvRule implements TestRule { @@ -75,10 +46,6 @@ public class TestEnvRule implements TestRule { private AbstractTestEnv testEnv; - private Handler grpcLogHandler; - private static final Set GRPC_LOGGER_NAMES = - ImmutableSet.of("io.grpc", "io.grpc.netty.shaded"); - @Override public Statement apply(final Statement base, final Description description) { return new Statement() { @@ -121,254 +88,12 @@ protected void before(Description description) throws Throwable { } private void after() { - try { - cleanUpStale(); - } catch (Exception e) { - LOGGER.log(Level.WARNING, "Failed to cleanup environment", e); - } - try { testEnv.stop(); } catch (Exception e) { LOGGER.log(Level.WARNING, "Failed to stop the environment", e); } testEnv = null; - teardownLogging(); - } - - private void teardownLogging() { - if (grpcLogHandler == null) { - return; - } - - for (String grpcLoggerName : GRPC_LOGGER_NAMES) { - Logger.getLogger(grpcLoggerName).removeHandler(grpcLogHandler); - } - grpcLogHandler.flush(); - grpcLogHandler = null; - } - - void cleanUpStale() throws ExecutionException, InterruptedException, IOException { - // Sortable resource prefix - time, process identifier, serial counterck - String stalePrefix = PrefixGenerator.newTimePrefix(Instant.now().minus(1, ChronoUnit.DAYS)); - - cleanupStaleTables(stalePrefix); - if (env().isInstanceAdminSupported()) { - cleanUpStaleAppProfile(stalePrefix); - cleanUpStaleClusters(stalePrefix); - cleanUpStaleInstances(stalePrefix); - } - } - - /** - * Clean up AppProfile that were dynamically created in the default instance that have been - * orphaned. - */ - private void cleanupStaleTables(String stalePrefix) { - LOGGER.info("Start cleaning up stale tables with stalePrefix=" + stalePrefix); - for (String tableId : env().getTableAdminClient().listTables()) { - if (!tableId.startsWith(PrefixGenerator.PREFIX)) { - LOGGER.info("Skip cleaning up table: " + tableId); - continue; - } - if (stalePrefix.compareTo(tableId) > 0) { - LOGGER.info("Preparing stale table for delete: " + tableId); - prepTableForDelete(tableId); - try { - LOGGER.info("Deleting stable table: " + tableId); - env().getTableAdminClient().deleteTable(tableId); - } catch (NotFoundException e) { - LOGGER.log(Level.WARNING, "Deleting stale table failed: " + tableId, e); - } - } - } - } - - private void prepTableForDelete(String tableId) { - if (!(env() instanceof EmulatorEnv)) { - // unprotect table - Table table = env().getTableAdminClient().getTable(tableId); - if (table.isDeletionProtected() || table.getChangeStreamRetention() != null) { - env() - .getTableAdminClient() - .updateTable( - UpdateTableRequest.of(tableId) - .setDeletionProtection(false) - .disableChangeStreamRetention()); - } - - // Unprotected views - for (String viewId : env().getTableAdminClient().listAuthorizedViews(tableId)) { - try { - env() - .getTableAdminClient() - .updateAuthorizedView( - UpdateAuthorizedViewRequest.of(tableId, viewId).setDeletionProtection(false)); - } catch (NotFoundException ignored) { - // nothing to clean up, the view was already deleted - } - } - } - } - - /** - * Clean up AppProfile that were dynamically created in the default instance that have been - * orphaned. - */ - private void cleanUpStaleAppProfile(String stalePrefix) { - for (AppProfile appProfile : - env().getInstanceAdminClient().listAppProfiles(env().getInstanceId())) { - if (!appProfile.getId().startsWith(PrefixGenerator.PREFIX)) { - continue; - } - boolean isNewerThanStale = appProfile.getId().compareTo(stalePrefix) > 0; - if (isNewerThanStale) { - continue; - } - try { - env() - .getInstanceAdminClient() - .deleteAppProfile(env().getInstanceId(), appProfile.getId(), true); - } catch (NotFoundException ignored) { - // nothing to clean up, the app profile was already deleted - } - } - } - - /** - * Clean up clusters that were dynamically created in the default instance that have been - * orphaned. - */ - private void cleanUpStaleClusters(String stalePrefix) - throws ExecutionException, InterruptedException { - for (Cluster cluster : env().getInstanceAdminClient().listClusters(env().getInstanceId())) { - if (!cluster.getId().startsWith(PrefixGenerator.PREFIX)) { - continue; - } - boolean isNewerThanStale = cluster.getId().compareTo(stalePrefix) > 0; - if (isNewerThanStale) { - continue; - } - - try { - deleteBackups(env().getTableAdminClient(), cluster.getId()); - } catch (NotFoundException ignored) { - // nothing to clean up, the backup was already deleted - } - try { - env().getInstanceAdminClient().deleteCluster(env().getInstanceId(), cluster.getId()); - } catch (NotFoundException ignored) { - // nothing to clean up, the cluster was already deleted - } - } - } - - private void prepInstanceForDelete(String instanceId) { - if (env() instanceof EmulatorEnv) { - return; - } - // Unprotected MaterializedViews. - for (MaterializedView materializedView : - env().getInstanceAdminClient().listMaterializedViews(instanceId)) { - try { - env() - .getInstanceAdminClient() - .updateMaterializedView( - UpdateMaterializedViewRequest.of(instanceId, materializedView.getId()) - .setDeletionProtection(false)); - } catch (NotFoundException ignored) { - // nothing to clean up, the materialized view was already deleted - } - } - // Unprotected LogicalViews. - for (LogicalView logicalView : env().getInstanceAdminClient().listLogicalViews(instanceId)) { - try { - env() - .getInstanceAdminClient() - .updateLogicalView( - UpdateLogicalViewRequest.of(instanceId, logicalView.getId()) - .setDeletionProtection(false)); - } catch (NotFoundException ignored) { - // nothing to clean up, the logical view was already deleted - } - } - } - - /** Clean up dynamically created (non-default) instances that have been orphaned. */ - private void cleanUpStaleInstances(String stalePrefix) - throws IOException, ExecutionException, InterruptedException { - for (Instance instance : env().getInstanceAdminClient().listInstances()) { - if (!instance.getId().startsWith(PrefixGenerator.PREFIX)) { - continue; - } - boolean isNewerThanStale = instance.getId().compareTo(stalePrefix) > 0; - if (isNewerThanStale) { - continue; - } - prepInstanceForDelete(instance.getId()); - try { - deleteInstance(instance.getId()); - } catch (NotFoundException ignored) { - // nothing to clean up, the instance was already deleted - } - } - } - - /** Delete an instance with all of its resources. */ - private void deleteInstance(String instanceId) - throws IOException, ExecutionException, InterruptedException { - BigtableTableAdminSettings settings = - env().getTableAdminSettings().toBuilder().setInstanceId(instanceId).build(); - - // Delete all child resources (backups & clusters) that wont be automatically deleted - try (BigtableTableAdminClient tableAdmin = BigtableTableAdminClient.create(settings)) { - List clusters = env().getInstanceAdminClient().listClusters(instanceId); - - boolean isFirstCluster = true; - - for (Cluster cluster : clusters) { - deleteBackups(tableAdmin, cluster.getId()); - // Skip the first cluster so that it can be delete by deleteInstance (instances can't exist - // without clusters) - if (!isFirstCluster) { - try { - env().getInstanceAdminClient().deleteCluster(instanceId, cluster.getId()); - } catch (NotFoundException ignored) { - // nothing to clean up, the cluster was already deleted - } - } - isFirstCluster = false; - } - } - - // Delete everything else - try { - env().getInstanceAdminClient().deleteInstance(instanceId); - } catch (NotFoundException ignored) { - // nothing to clean up, the instance was already deleted - } - } - - private void deleteBackups(BigtableTableAdminClient tableAdmin, String clusterId) - throws ExecutionException, InterruptedException { - List> futures = new ArrayList<>(); - - for (String backupId : tableAdmin.listBackups(clusterId)) { - ApiFuture f = tableAdmin.deleteBackupAsync(clusterId, backupId); - futures.add(f); - } - - for (ApiFuture future : futures) { - try { - future.get(); - } catch (ExecutionException e) { - // Ignore not found - if (e.getCause() instanceof NotFoundException) { - continue; - } - throw e; - } - } } public AbstractTestEnv env() { From 7e706f7d987a8656a22dbb31cf90de9139b8ec55 Mon Sep 17 00:00:00 2001 From: Igor Bernstein Date: Tue, 17 Nov 2026 19:42:47 +0000 Subject: [PATCH 2/7] fix(bigtable): handle concurrent NotFoundException during stale resource cleanup --- .../env/StaleResourceCleaner.java | 35 +++-- .../env/StaleResourceCleanerTest.java | 120 ++++++++++++++++++ 2 files changed, 142 insertions(+), 13 deletions(-) diff --git a/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/test_helpers/env/StaleResourceCleaner.java b/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/test_helpers/env/StaleResourceCleaner.java index a8192b7b8661..fcf678733154 100644 --- a/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/test_helpers/env/StaleResourceCleaner.java +++ b/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/test_helpers/env/StaleResourceCleaner.java @@ -272,18 +272,23 @@ private void prepTableForDelete(BigtableTableAdminClient tableAdmin, String tabl private void cleanUpStaleAuthorizedViewsOnPersistentTable( BigtableTableAdminClient tableAdmin, String tableId, String stalePrefix) { - for (String viewId : tableAdmin.listAuthorizedViews(tableId)) { - if (!isStale(viewId, stalePrefix)) { - continue; - } - LOGGER.info("Deleting stale authorized view: " + viewId + " on persistent table: " + tableId); - try { - tableAdmin.updateAuthorizedView( - UpdateAuthorizedViewRequest.of(tableId, viewId).setDeletionProtection(false)); - tableAdmin.deleteAuthorizedView(tableId, viewId); - } catch (NotFoundException ignored) { - // nothing to clean up, the view was already deleted + try { + for (String viewId : tableAdmin.listAuthorizedViews(tableId)) { + if (!isStale(viewId, stalePrefix)) { + continue; + } + LOGGER.info( + "Deleting stale authorized view: " + viewId + " on persistent table: " + tableId); + try { + tableAdmin.updateAuthorizedView( + UpdateAuthorizedViewRequest.of(tableId, viewId).setDeletionProtection(false)); + tableAdmin.deleteAuthorizedView(tableId, viewId); + } catch (NotFoundException ignored) { + // nothing to clean up, the view was already deleted + } } + } catch (NotFoundException ignored) { + // nothing to clean up, the table was already deleted } } @@ -335,8 +340,8 @@ private void cleanUpStaleInstances(String stalePrefix) if (!isStale(instance.getId(), stalePrefix)) { continue; } - prepInstanceForDelete(instance.getId()); try { + prepInstanceForDelete(instance.getId()); deleteInstance(instance.getId()); } catch (NotFoundException ignored) { // nothing to clean up, the instance was already deleted @@ -361,7 +366,11 @@ private void deleteInstance(String instanceId) boolean isFirstCluster = true; for (Cluster cluster : clusters) { - deleteAllBackups(tableAdmin, cluster.getId()); + try { + deleteAllBackups(tableAdmin, cluster.getId()); + } catch (NotFoundException ignored) { + // nothing to clean up, the cluster or backup was already deleted + } // Skip the first cluster so that it can be deleted by deleteInstance (instances can't exist // without clusters) if (!isFirstCluster) { diff --git a/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/test_helpers/env/StaleResourceCleanerTest.java b/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/test_helpers/env/StaleResourceCleanerTest.java index e3d6d97ba4fb..0893e936a261 100644 --- a/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/test_helpers/env/StaleResourceCleanerTest.java +++ b/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/test_helpers/env/StaleResourceCleanerTest.java @@ -31,6 +31,7 @@ import com.google.api.gax.longrunning.OperationFuture; import com.google.api.gax.longrunning.OperationFutures; import com.google.api.gax.longrunning.OperationSnapshot; +import com.google.api.gax.rpc.NotFoundException; import com.google.api.gax.rpc.OperationCallable; import com.google.api.gax.rpc.UnaryCallable; import com.google.api.gax.rpc.testing.FakeOperationSnapshot; @@ -682,6 +683,125 @@ void testCleanUpStaleInstances() throws Exception { .build()); } + @Test + @SuppressWarnings("unchecked") + void testConcurrentNotFoundExceptionsAreIgnored() throws Exception { + EnhancedBigtableTableAdminStub sharedTableStub = createTableStubForInstance(SHARED_INSTANCE_ID); + mockMaterializedViews(SHARED_INSTANCE_ID, ImmutableList.of()); + mockLogicalViews(SHARED_INSTANCE_ID, ImmutableList.of()); + mockAppProfiles(SHARED_INSTANCE_ID, ImmutableList.of()); + mockClusters(SHARED_INSTANCE_ID, ImmutableList.of()); + + String concurrentlyDeletedTable = "deleted-persistent-table"; + String survivingPersistentTable = "integration-tests"; + String staleAv = STALE_ID_PREFIX + "av"; + mockListTables( + sharedTableStub, + SHARED_INSTANCE_ID, + ImmutableList.of(concurrentlyDeletedTable, survivingPersistentTable)); + + UnaryCallable + mockListAuthorizedViewsCallable = + mock(UnaryCallable.class, withSettings().withoutAnnotations()); + OperationCallable + mockUpdateAuthorizedViewCallable = + mock(OperationCallable.class, withSettings().withoutAnnotations()); + UnaryCallable mockDeleteAuthorizedViewCallable = + mock(UnaryCallable.class, withSettings().withoutAnnotations()); + when(sharedTableStub.listAuthorizedViewsPagedCallable()) + .thenReturn(mockListAuthorizedViewsCallable); + when(sharedTableStub.updateAuthorizedViewOperationCallable()) + .thenReturn(mockUpdateAuthorizedViewCallable); + when(sharedTableStub.deleteAuthorizedViewCallable()) + .thenReturn(mockDeleteAuthorizedViewCallable); + + NotFoundException notFound = + new NotFoundException(null, GrpcStatusCode.of(Code.NOT_FOUND), false); + when(mockListAuthorizedViewsCallable.futureCall( + ListAuthorizedViewsRequest.newBuilder() + .setParent( + NameUtil.formatTableName( + PROJECT_ID, SHARED_INSTANCE_ID, concurrentlyDeletedTable)) + .build())) + .thenReturn(ApiFutures.immediateFailedFuture(notFound)); + mockAuthorizedViews( + mockListAuthorizedViewsCallable, + SHARED_INSTANCE_ID, + survivingPersistentTable, + ImmutableList.of(staleAv)); + mockOperationFuture( + mockUpdateAuthorizedViewCallable, + AuthorizedView.newBuilder() + .setName( + NameUtil.formatAuthorizedViewName( + PROJECT_ID, SHARED_INSTANCE_ID, survivingPersistentTable, staleAv)) + .setSubsetView(AuthorizedView.SubsetView.getDefaultInstance()) + .setDeletionProtection(false) + .build()); + when(mockDeleteAuthorizedViewCallable.futureCall(any())) + .thenReturn(ApiFutures.immediateFuture(Empty.getDefaultInstance())); + + // Stale instances: first one throws NotFoundException in prepInstanceForDelete, second one has + // cluster0 throw NotFoundException in deleteAllBackups but continues to cluster1 and deletes + String concurrentlyDeletedInst = STALE_ID_PREFIX + "deleted-inst"; + String remainingStaleInst = STALE_ID_PREFIX + "remaining-inst"; + mockInstances(ImmutableList.of(concurrentlyDeletedInst, remainingStaleInst)); + + when(mockListMaterializedViewsCallable.futureCall( + ListMaterializedViewsRequest.newBuilder() + .setParent(NameUtil.formatInstanceName(PROJECT_ID, concurrentlyDeletedInst)) + .build())) + .thenReturn(ApiFutures.immediateFailedFuture(notFound)); + + mockMaterializedViews(remainingStaleInst, ImmutableList.of()); + mockLogicalViews(remainingStaleInst, ImmutableList.of()); + mockClusters(remainingStaleInst, ImmutableList.of("c0", "c1")); + + EnhancedBigtableTableAdminStub remainingInstTableStub = + createTableStubForInstance(remainingStaleInst); + UnaryCallable mockListBackupsCallable = + mock(UnaryCallable.class, withSettings().withoutAnnotations()); + UnaryCallable mockDeleteBackupCallable = + mock(UnaryCallable.class, withSettings().withoutAnnotations()); + when(remainingInstTableStub.listBackupsPagedCallable()).thenReturn(mockListBackupsCallable); + when(remainingInstTableStub.deleteBackupCallable()).thenReturn(mockDeleteBackupCallable); + when(mockDeleteBackupCallable.futureCall(any())) + .thenReturn(ApiFutures.immediateFuture(Empty.getDefaultInstance())); + + when(mockListBackupsCallable.futureCall( + ListBackupsRequest.newBuilder() + .setParent(NameUtil.formatClusterName(PROJECT_ID, remainingStaleInst, "c0")) + .build())) + .thenReturn(ApiFutures.immediateFailedFuture(notFound)); + mockBackups(mockListBackupsCallable, remainingStaleInst, "c1", ImmutableList.of("b1")); + + StaleResourceCleaner cleaner = createCleaner(ImmutableList.of(SHARED_INSTANCE_ID)); + cleaner.cleanUp(STALE_PREFIX); + + verify(mockDeleteAuthorizedViewCallable) + .futureCall( + DeleteAuthorizedViewRequest.newBuilder() + .setName( + NameUtil.formatAuthorizedViewName( + PROJECT_ID, SHARED_INSTANCE_ID, survivingPersistentTable, staleAv)) + .build()); + verify(mockDeleteBackupCallable) + .futureCall( + DeleteBackupRequest.newBuilder() + .setName(NameUtil.formatBackupName(PROJECT_ID, remainingStaleInst, "c1", "b1")) + .build()); + verify(mockDeleteClusterCallable) + .futureCall( + DeleteClusterRequest.newBuilder() + .setName(NameUtil.formatClusterName(PROJECT_ID, remainingStaleInst, "c1")) + .build()); + verify(mockDeleteInstanceCallable) + .futureCall( + DeleteInstanceRequest.newBuilder() + .setName(NameUtil.formatInstanceName(PROJECT_ID, remainingStaleInst)) + .build()); + } + private StaleResourceCleaner createCleaner(List sharedInstanceIds) { return new StaleResourceCleaner( instanceAdmin, From cadaa9ec644c61e9044071f999e8c0654c76a931 Mon Sep 17 00:00:00 2001 From: Igor Bernstein Date: Tue, 17 Nov 2026 20:16:39 +0000 Subject: [PATCH 3/7] ci(bigtable): use dedicated bigtable-cleanup.sh script and CloudEnv --- .kokoro/bigtable-cleanup.sh | 48 ++++++ .kokoro/build.sh | 15 -- .kokoro/nightly/bigtable-cleanup.cfg | 14 +- java-bigtable/google-cloud-bigtable/pom.xml | 13 +- .../env/StaleResourceCleaner.java | 148 ++++-------------- .../env/StaleResourceCleanerTest.java | 75 ++++----- 6 files changed, 119 insertions(+), 194 deletions(-) create mode 100755 .kokoro/bigtable-cleanup.sh diff --git a/.kokoro/bigtable-cleanup.sh b/.kokoro/bigtable-cleanup.sh new file mode 100755 index 000000000000..62e883035723 --- /dev/null +++ b/.kokoro/bigtable-cleanup.sh @@ -0,0 +1,48 @@ +#!/bin/bash +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -eo pipefail + +## Get the directory of the build script +scriptDir=$(realpath $(dirname "${BASH_SOURCE[0]}")) +## cd to the parent directory, i.e. the root of the git repo +cd ${scriptDir}/.. + +# include common functions +source ${scriptDir}/common.sh + +# Use GCP Maven Mirror +mkdir -p ${HOME}/.m2 +cp settings.xml ${HOME}/.m2 + +setup_application_credentials + +echo "Compiling and building all modules for java-bigtable" +install_modules "java-bigtable" + +pushd java-bigtable +RETURN_CODE=0 +for instance in google-cloud-bigtable google-cloud-bigtable-session; do + echo "Cleaning up stale resources in instance: ${instance}" + mvn -B -ntp -pl google-cloud-bigtable \ + test-compile exec:java@cleanup-stale-resources \ + -Dbigtable.project="${GOOGLE_CLOUD_PROJECT:-gcloud-devel}" \ + -Dbigtable.instance="${instance}" \ + -Dbigtable.table=integration-tests || RETURN_CODE=$? +done +popd + +echo "exiting with ${RETURN_CODE}" +exit ${RETURN_CODE} diff --git a/.kokoro/build.sh b/.kokoro/build.sh index 24e31496f9cd..ecdd0a3ea514 100755 --- a/.kokoro/build.sh +++ b/.kokoro/build.sh @@ -169,21 +169,6 @@ case ${JOB_TYPE} in printf "Finished integration tests for modules:\n%s\n" "${BUILD_SUBDIR}" fi ;; - bigtable-cleanup) - echo "Compiling and building all modules for ${BUILD_SUBDIR}" - install_modules "${BUILD_SUBDIR}" - echo "Running in subdir: ${BUILD_SUBDIR}" - pushd "${BUILD_SUBDIR}" - echo "INTEGRATION_TEST_ARGS: ${INTEGRATION_TEST_ARGS}" - mvn test-compile exec:java -pl google-cloud-bigtable -Pquick-build \ - ${INTEGRATION_TEST_ARGS} \ - -B -ntp \ - -DtrimStackTrace=false \ - -Dorg.slf4j.simpleLogger.showDateTime=true \ - -Dorg.slf4j.simpleLogger.dateTimeFormat=HH:mm:ss:SSS - RETURN_CODE=$? - popd - ;; graalvm-presubmit) generate_graalvm_presubmit_modules_list if [[ "$(release_please_snapshot_pull_request)" == "true" ]]; then diff --git a/.kokoro/nightly/bigtable-cleanup.cfg b/.kokoro/nightly/bigtable-cleanup.cfg index 9060f5ddd921..3811fdbfac82 100644 --- a/.kokoro/nightly/bigtable-cleanup.cfg +++ b/.kokoro/nightly/bigtable-cleanup.cfg @@ -7,8 +7,8 @@ env_vars: { } env_vars: { - key: "JOB_TYPE" - value: "bigtable-cleanup" + key: "TRAMPOLINE_BUILD_FILE" + value: "github/google-cloud-java/.kokoro/bigtable-cleanup.sh" } env_vars: { @@ -25,13 +25,3 @@ env_vars: { key: "SECRET_MANAGER_KEYS" value: "java-it-service-account" } - -env_vars: { - key: "BUILD_SUBDIR" - value: "java-bigtable" -} - -env_vars: { - key: "INTEGRATION_TEST_ARGS" - value: "-Dbigtable.project=gcloud-devel -Dbigtable.instances=google-cloud-bigtable,google-cloud-bigtable-session" -} diff --git a/java-bigtable/google-cloud-bigtable/pom.xml b/java-bigtable/google-cloud-bigtable/pom.xml index b5018f1a0730..9a330ef5e178 100644 --- a/java-bigtable/google-cloud-bigtable/pom.xml +++ b/java-bigtable/google-cloud-bigtable/pom.xml @@ -810,10 +810,15 @@ org.codehaus.mojo exec-maven-plugin - - com.google.cloud.bigtable.test_helpers.env.StaleResourceCleaner - test - + + + cleanup-stale-resources + + com.google.cloud.bigtable.test_helpers.env.StaleResourceCleaner + test + + + diff --git a/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/test_helpers/env/StaleResourceCleaner.java b/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/test_helpers/env/StaleResourceCleaner.java index fcf678733154..0ed24e89582e 100644 --- a/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/test_helpers/env/StaleResourceCleaner.java +++ b/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/test_helpers/env/StaleResourceCleaner.java @@ -18,9 +18,7 @@ import com.google.api.core.ApiFuture; import com.google.api.gax.rpc.NotFoundException; import com.google.cloud.bigtable.admin.v2.BigtableInstanceAdminClient; -import com.google.cloud.bigtable.admin.v2.BigtableInstanceAdminSettings; import com.google.cloud.bigtable.admin.v2.BigtableTableAdminClient; -import com.google.cloud.bigtable.admin.v2.BigtableTableAdminSettings; import com.google.cloud.bigtable.admin.v2.models.AppProfile; import com.google.cloud.bigtable.admin.v2.models.Cluster; import com.google.cloud.bigtable.admin.v2.models.Instance; @@ -31,8 +29,6 @@ import com.google.cloud.bigtable.admin.v2.models.UpdateLogicalViewRequest; import com.google.cloud.bigtable.admin.v2.models.UpdateMaterializedViewRequest; import com.google.cloud.bigtable.admin.v2.models.UpdateTableRequest; -import com.google.common.base.Splitter; -import com.google.common.base.Strings; import java.io.IOException; import java.time.Instant; import java.time.temporal.ChronoUnit; @@ -41,114 +37,29 @@ import java.util.concurrent.ExecutionException; import java.util.logging.Level; import java.util.logging.Logger; -import javax.annotation.Nullable; /** - * Standalone utility to clean up stale test resources older than 24 hours across shared Bigtable - * instances and dynamically created test instances in a project. + * Standalone utility to clean up stale test resources older than 24 hours in the configured {@link + * CloudEnv} shared Bigtable instance as well as dynamically created test instances in the project. */ public class StaleResourceCleaner { private static final Logger LOGGER = Logger.getLogger(StaleResourceCleaner.class.getName()); - static final String PROJECT_PROPERTY_NAME = "bigtable.project"; - static final String INSTANCES_PROPERTY_NAME = "bigtable.instances"; - static final String INSTANCE_PROPERTY_NAME = "bigtable.instance"; - static final String ADMIN_ENDPOINT_PROPERTY_NAME = "bigtable.admin-endpoint"; - static final String CFE_ADMIN_ENDPOINT_PROPERTY_NAME = "bigtable.cfe-admin-endpoint"; + private final AbstractTestEnv testEnv; - @FunctionalInterface - interface TableAdminFactory { - BigtableTableAdminClient createForInstance(String instanceId) throws IOException; + StaleResourceCleaner(AbstractTestEnv testEnv) { + this.testEnv = testEnv; } - private final BigtableInstanceAdminClient instanceAdmin; - private final TableAdminFactory tableAdminFactory; - private final List sharedInstanceIds; - - StaleResourceCleaner( - BigtableInstanceAdminClient instanceAdmin, - TableAdminFactory tableAdminFactory, - List sharedInstanceIds) { - this.instanceAdmin = instanceAdmin; - this.tableAdminFactory = tableAdminFactory; - this.sharedInstanceIds = sharedInstanceIds; - } - - public static void main(String[] args) - throws IOException, ExecutionException, InterruptedException { - String projectId = System.getProperty(PROJECT_PROPERTY_NAME); - if (Strings.isNullOrEmpty(projectId)) { - throw new IllegalArgumentException( - "Missing required system property: " + PROJECT_PROPERTY_NAME); - } - - List sharedInstanceIds = - parseInstanceIds( - System.getProperty(INSTANCES_PROPERTY_NAME), - System.getProperty(INSTANCE_PROPERTY_NAME)); - - String adminEndpoint = - resolveAdminEndpoint( - System.getProperty(ADMIN_ENDPOINT_PROPERTY_NAME), - System.getProperty(CFE_ADMIN_ENDPOINT_PROPERTY_NAME)); - - BigtableInstanceAdminSettings.Builder instanceAdminSettings = - BigtableInstanceAdminSettings.newBuilder().setProjectId(projectId); - if (!Strings.isNullOrEmpty(adminEndpoint)) { - instanceAdminSettings.stubSettings().setEndpoint(adminEndpoint); - } - - TableAdminFactory tableAdminFactory = - instanceId -> { - BigtableTableAdminSettings.Builder tableAdminSettings = - BigtableTableAdminSettings.newBuilder() - .setProjectId(projectId) - .setInstanceId(instanceId); - if (!Strings.isNullOrEmpty(adminEndpoint)) { - tableAdminSettings.stubSettings().setEndpoint(adminEndpoint); - } - return BigtableTableAdminClient.create(tableAdminSettings.build()); - }; - - try (BigtableInstanceAdminClient instanceAdmin = - BigtableInstanceAdminClient.create(instanceAdminSettings.build())) { - StaleResourceCleaner cleaner = - new StaleResourceCleaner(instanceAdmin, tableAdminFactory, sharedInstanceIds); - cleaner.cleanUp(); - } - } - - static List parseInstanceIds( - @Nullable String instancesProp, @Nullable String instanceProp) { - String raw = !Strings.isNullOrEmpty(instancesProp) ? instancesProp : instanceProp; - if (Strings.isNullOrEmpty(raw)) { - throw new IllegalArgumentException( - "Missing required system property: " - + INSTANCES_PROPERTY_NAME - + " or " - + INSTANCE_PROPERTY_NAME); - } - List instanceIds = Splitter.on(',').trimResults().omitEmptyStrings().splitToList(raw); - if (instanceIds.isEmpty()) { - throw new IllegalArgumentException( - "No instance IDs found in system property: " - + INSTANCES_PROPERTY_NAME - + " or " - + INSTANCE_PROPERTY_NAME); - } - return instanceIds; - } - - static String resolveAdminEndpoint( - @Nullable String adminEndpoint, @Nullable String cfeAdminEndpoint) { - if (!Strings.isNullOrEmpty(adminEndpoint)) { - return adminEndpoint; - } - if (!Strings.isNullOrEmpty(cfeAdminEndpoint)) { - return cfeAdminEndpoint; + public static void main(String[] args) throws Exception { + CloudEnv cloudEnv = CloudEnv.fromSystemProperties(); + cloudEnv.start(); + try { + new StaleResourceCleaner(cloudEnv).cleanUp(); + } finally { + cloudEnv.stop(); } - return ""; } static boolean isStale(String resourceId, String stalePrefix) { @@ -161,30 +72,29 @@ public void cleanUp() throws IOException, ExecutionException, InterruptedExcepti } void cleanUp(String stalePrefix) throws IOException, ExecutionException, InterruptedException { + String instanceId = testEnv.getInstanceId(); LOGGER.info( "Starting stale resource cleanup with stalePrefix=" + stalePrefix - + " for shared instances=" - + sharedInstanceIds); - for (String instanceId : sharedInstanceIds) { - cleanUpSharedInstance(instanceId, stalePrefix); - } + + " for shared instance=" + + instanceId); + cleanUpSharedInstance(instanceId, stalePrefix); cleanUpStaleInstances(stalePrefix); } void cleanUpSharedInstance(String instanceId, String stalePrefix) - throws IOException, ExecutionException, InterruptedException { + throws ExecutionException, InterruptedException { LOGGER.info("Cleaning up shared instance: " + instanceId); - try (BigtableTableAdminClient tableAdmin = tableAdminFactory.createForInstance(instanceId)) { - cleanUpStaleMaterializedViews(instanceId, stalePrefix); - cleanUpStaleLogicalViews(instanceId, stalePrefix); - cleanUpTablesAndAuthorizedViews(tableAdmin, stalePrefix); - cleanUpStaleAppProfiles(instanceId, stalePrefix); - cleanUpClustersAndBackups(instanceId, tableAdmin, stalePrefix); - } + BigtableTableAdminClient tableAdmin = testEnv.getTableAdminClient(); + cleanUpStaleMaterializedViews(instanceId, stalePrefix); + cleanUpStaleLogicalViews(instanceId, stalePrefix); + cleanUpTablesAndAuthorizedViews(tableAdmin, stalePrefix); + cleanUpStaleAppProfiles(instanceId, stalePrefix); + cleanUpClustersAndBackups(instanceId, tableAdmin, stalePrefix); } private void cleanUpStaleMaterializedViews(String instanceId, String stalePrefix) { + BigtableInstanceAdminClient instanceAdmin = testEnv.getInstanceAdminClient(); for (MaterializedView view : instanceAdmin.listMaterializedViews(instanceId)) { if (!isStale(view.getId(), stalePrefix)) { continue; @@ -196,6 +106,7 @@ private void cleanUpStaleMaterializedViews(String instanceId, String stalePrefix } private void unprotectAndDeleteMaterializedView(String instanceId, MaterializedView view) { + BigtableInstanceAdminClient instanceAdmin = testEnv.getInstanceAdminClient(); try { if (view.isDeletionProtected()) { instanceAdmin.updateMaterializedView( @@ -209,6 +120,7 @@ private void unprotectAndDeleteMaterializedView(String instanceId, MaterializedV } private void cleanUpStaleLogicalViews(String instanceId, String stalePrefix) { + BigtableInstanceAdminClient instanceAdmin = testEnv.getInstanceAdminClient(); for (LogicalView view : instanceAdmin.listLogicalViews(instanceId)) { if (!isStale(view.getId(), stalePrefix)) { continue; @@ -219,6 +131,7 @@ private void cleanUpStaleLogicalViews(String instanceId, String stalePrefix) { } private void unprotectAndDeleteLogicalView(String instanceId, LogicalView view) { + BigtableInstanceAdminClient instanceAdmin = testEnv.getInstanceAdminClient(); try { if (view.isDeletionProtected()) { instanceAdmin.updateLogicalView( @@ -293,6 +206,7 @@ private void cleanUpStaleAuthorizedViewsOnPersistentTable( } private void cleanUpStaleAppProfiles(String instanceId, String stalePrefix) { + BigtableInstanceAdminClient instanceAdmin = testEnv.getInstanceAdminClient(); for (AppProfile appProfile : instanceAdmin.listAppProfiles(instanceId)) { if (!isStale(appProfile.getId(), stalePrefix)) { continue; @@ -308,6 +222,7 @@ private void cleanUpStaleAppProfiles(String instanceId, String stalePrefix) { private void cleanUpClustersAndBackups( String instanceId, BigtableTableAdminClient tableAdmin, String stalePrefix) throws ExecutionException, InterruptedException { + BigtableInstanceAdminClient instanceAdmin = testEnv.getInstanceAdminClient(); for (Cluster cluster : instanceAdmin.listClusters(instanceId)) { if (!cluster.getId().startsWith(PrefixGenerator.PREFIX)) { try { @@ -336,6 +251,7 @@ private void cleanUpClustersAndBackups( private void cleanUpStaleInstances(String stalePrefix) throws IOException, ExecutionException, InterruptedException { + BigtableInstanceAdminClient instanceAdmin = testEnv.getInstanceAdminClient(); for (Instance instance : instanceAdmin.listInstances()) { if (!isStale(instance.getId(), stalePrefix)) { continue; @@ -350,6 +266,7 @@ private void cleanUpStaleInstances(String stalePrefix) } private void prepInstanceForDelete(String instanceId) { + BigtableInstanceAdminClient instanceAdmin = testEnv.getInstanceAdminClient(); for (MaterializedView materializedView : instanceAdmin.listMaterializedViews(instanceId)) { unprotectAndDeleteMaterializedView(instanceId, materializedView); } @@ -360,7 +277,8 @@ private void prepInstanceForDelete(String instanceId) { private void deleteInstance(String instanceId) throws IOException, ExecutionException, InterruptedException { - try (BigtableTableAdminClient tableAdmin = tableAdminFactory.createForInstance(instanceId)) { + BigtableInstanceAdminClient instanceAdmin = testEnv.getInstanceAdminClient(); + try (BigtableTableAdminClient tableAdmin = testEnv.getTableAdminClientForInstance(instanceId)) { List clusters = instanceAdmin.listClusters(instanceId); boolean isFirstCluster = true; diff --git a/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/test_helpers/env/StaleResourceCleanerTest.java b/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/test_helpers/env/StaleResourceCleanerTest.java index 0893e936a261..e03d3671ea89 100644 --- a/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/test_helpers/env/StaleResourceCleanerTest.java +++ b/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/test_helpers/env/StaleResourceCleanerTest.java @@ -15,9 +15,8 @@ */ package com.google.cloud.bigtable.test_helpers.env; -import static com.google.common.truth.Truth.assertThat; -import static org.junit.jupiter.api.Assertions.assertThrows; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.inOrder; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; @@ -200,42 +199,6 @@ void setUp() { .thenReturn(ApiFutures.immediateFuture(Empty.getDefaultInstance())); } - @Test - void testParseInstanceIds() { - assertThat( - StaleResourceCleaner.parseInstanceIds( - "google-cloud-bigtable, google-cloud-bigtable-session", "ignored-fallback")) - .containsExactly("google-cloud-bigtable", "google-cloud-bigtable-session") - .inOrder(); - - assertThat(StaleResourceCleaner.parseInstanceIds(null, "google-cloud-bigtable")) - .containsExactly("google-cloud-bigtable"); - - assertThat(StaleResourceCleaner.parseInstanceIds("", "google-cloud-bigtable")) - .containsExactly("google-cloud-bigtable"); - - assertThrows( - IllegalArgumentException.class, () -> StaleResourceCleaner.parseInstanceIds(null, null)); - assertThrows( - IllegalArgumentException.class, () -> StaleResourceCleaner.parseInstanceIds(" , ", "")); - } - - @Test - void testResolveAdminEndpoint() { - assertThat( - StaleResourceCleaner.resolveAdminEndpoint( - "test-bigtableadmin.sandbox.googleapis.com:443", - "staging-bigtableadmin.sandbox.googleapis.com:443")) - .isEqualTo("test-bigtableadmin.sandbox.googleapis.com:443"); - - assertThat( - StaleResourceCleaner.resolveAdminEndpoint( - "", "staging-bigtableadmin.sandbox.googleapis.com:443")) - .isEqualTo("staging-bigtableadmin.sandbox.googleapis.com:443"); - - assertThat(StaleResourceCleaner.resolveAdminEndpoint(null, null)).isEmpty(); - } - @Test @SuppressWarnings("unchecked") void testCleanUpSharedInstance_viewsTablesAndAuthorizedViews() throws Exception { @@ -360,7 +323,7 @@ void testCleanUpSharedInstance_viewsTablesAndAuthorizedViews() throws Exception mockClusters(SHARED_INSTANCE_ID, ImmutableList.of()); mockInstances(ImmutableList.of()); - StaleResourceCleaner cleaner = createCleaner(ImmutableList.of(SHARED_INSTANCE_ID)); + StaleResourceCleaner cleaner = createCleaner(); cleaner.cleanUp(STALE_PREFIX); // MaterializedViews: stale ones deleted before table deletion; only protected one updated @@ -533,7 +496,7 @@ void testCleanUpSharedInstance_appProfilesClustersAndBackups() throws Exception mockInstances(ImmutableList.of()); - StaleResourceCleaner cleaner = createCleaner(ImmutableList.of(SHARED_INSTANCE_ID)); + StaleResourceCleaner cleaner = createCleaner(); cleaner.cleanUp(STALE_PREFIX); // AppProfiles: only staleAp deleted with ignoreWarnings=true @@ -630,7 +593,14 @@ void testCleanUpStaleInstances() throws Exception { mockBackups(mockListBackupsCallable, staleInstance, cluster0, ImmutableList.of("b0")); mockBackups(mockListBackupsCallable, staleInstance, cluster1, ImmutableList.of("b1")); - StaleResourceCleaner cleaner = createCleaner(ImmutableList.of()); + EnhancedBigtableTableAdminStub sharedTableStub = createTableStubForInstance(SHARED_INSTANCE_ID); + mockMaterializedViews(SHARED_INSTANCE_ID, ImmutableList.of()); + mockLogicalViews(SHARED_INSTANCE_ID, ImmutableList.of()); + mockListTables(sharedTableStub, SHARED_INSTANCE_ID, ImmutableList.of()); + mockAppProfiles(SHARED_INSTANCE_ID, ImmutableList.of()); + mockClusters(SHARED_INSTANCE_ID, ImmutableList.of()); + + StaleResourceCleaner cleaner = createCleaner(); cleaner.cleanUp(STALE_PREFIX); verify(mockDeleteMaterializedViewCallable) @@ -775,7 +745,7 @@ void testConcurrentNotFoundExceptionsAreIgnored() throws Exception { .thenReturn(ApiFutures.immediateFailedFuture(notFound)); mockBackups(mockListBackupsCallable, remainingStaleInst, "c1", ImmutableList.of("b1")); - StaleResourceCleaner cleaner = createCleaner(ImmutableList.of(SHARED_INSTANCE_ID)); + StaleResourceCleaner cleaner = createCleaner(); cleaner.cleanUp(STALE_PREFIX); verify(mockDeleteAuthorizedViewCallable) @@ -802,13 +772,22 @@ void testConcurrentNotFoundExceptionsAreIgnored() throws Exception { .build()); } - private StaleResourceCleaner createCleaner(List sharedInstanceIds) { - return new StaleResourceCleaner( - instanceAdmin, - instanceId -> + private StaleResourceCleaner createCleaner() throws Exception { + AbstractTestEnv mockEnv = mock(AbstractTestEnv.class); + when(mockEnv.getInstanceId()).thenReturn(SHARED_INSTANCE_ID); + when(mockEnv.getInstanceAdminClient()).thenReturn(instanceAdmin); + when(mockEnv.getTableAdminClient()) + .thenReturn( BigtableTableAdminClient.create( - PROJECT_ID, instanceId, tableStubsByInstance.get(instanceId)), - sharedInstanceIds); + PROJECT_ID, SHARED_INSTANCE_ID, tableStubsByInstance.get(SHARED_INSTANCE_ID))); + when(mockEnv.getTableAdminClientForInstance(anyString())) + .thenAnswer( + invocation -> { + String instanceId = invocation.getArgument(0); + return BigtableTableAdminClient.create( + PROJECT_ID, instanceId, tableStubsByInstance.get(instanceId)); + }); + return new StaleResourceCleaner(mockEnv); } private EnhancedBigtableTableAdminStub createTableStubForInstance(String instanceId) From 08367432c4cf8abeb042c23b6c4042f180beaad3 Mon Sep 17 00:00:00 2001 From: Igor Bernstein Date: Tue, 17 Nov 2026 20:24:55 +0000 Subject: [PATCH 4/7] chore(bigtable): drop redundant test-compile and document cleanup properties --- .kokoro/bigtable-cleanup.sh | 2 +- java-bigtable/google-cloud-bigtable/pom.xml | 12 ++++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/.kokoro/bigtable-cleanup.sh b/.kokoro/bigtable-cleanup.sh index 62e883035723..f333a297c728 100755 --- a/.kokoro/bigtable-cleanup.sh +++ b/.kokoro/bigtable-cleanup.sh @@ -37,7 +37,7 @@ RETURN_CODE=0 for instance in google-cloud-bigtable google-cloud-bigtable-session; do echo "Cleaning up stale resources in instance: ${instance}" mvn -B -ntp -pl google-cloud-bigtable \ - test-compile exec:java@cleanup-stale-resources \ + exec:java@cleanup-stale-resources \ -Dbigtable.project="${GOOGLE_CLOUD_PROJECT:-gcloud-devel}" \ -Dbigtable.instance="${instance}" \ -Dbigtable.table=integration-tests || RETURN_CODE=$? diff --git a/java-bigtable/google-cloud-bigtable/pom.xml b/java-bigtable/google-cloud-bigtable/pom.xml index 9a330ef5e178..4d5b9ffac95d 100644 --- a/java-bigtable/google-cloud-bigtable/pom.xml +++ b/java-bigtable/google-cloud-bigtable/pom.xml @@ -811,6 +811,18 @@ org.codehaus.mojo exec-maven-plugin + cleanup-stale-resources From 47b613b16a44afbed585144c47b880bda7a0aceb Mon Sep 17 00:00:00 2001 From: Igor Bernstein Date: Tue, 17 Nov 2026 20:29:01 +0000 Subject: [PATCH 5/7] chore(bigtable): align exec-maven-plugin systemProperties with failsafe profiles --- java-bigtable/google-cloud-bigtable/pom.xml | 25 ++++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/java-bigtable/google-cloud-bigtable/pom.xml b/java-bigtable/google-cloud-bigtable/pom.xml index 4d5b9ffac95d..086cd11656da 100644 --- a/java-bigtable/google-cloud-bigtable/pom.xml +++ b/java-bigtable/google-cloud-bigtable/pom.xml @@ -819,15 +819,34 @@ -Dbigtable.project= -Dbigtable.instance= -Dbigtable.table= - Optional system properties: - -Dbigtable.admin-endpoint= (defaults to client default: bigtableadmin.googleapis.com:443) - -Dbigtable.data-endpoint= (defaults to client default: bigtable.googleapis.com:443) + Optional system properties (consistent with Failsafe IT profiles): + -Dbigtable.cfe-admin-endpoint= (defaults to client default: bigtableadmin.googleapis.com:443) + -Dbigtable.cfe-data-endpoint= (defaults to client default: bigtable.googleapis.com:443) + -Dbigtable.cfe-jwt-audience= (defaults to client default) --> cleanup-stale-resources com.google.cloud.bigtable.test_helpers.env.StaleResourceCleaner test + + + java.util.logging.config.file + ${java-log-config} + + + bigtable.data-endpoint + ${bigtable.cfe-data-endpoint} + + + bigtable.admin-endpoint + ${bigtable.cfe-admin-endpoint} + + + bigtable.data-jwt-audience + ${bigtable.cfe-jwt-audience} + + From 07e2242b91b2a2a5b3666d24c51147c36965e549 Mon Sep 17 00:00:00 2001 From: Igor Bernstein Date: Tue, 17 Nov 2026 20:48:38 +0000 Subject: [PATCH 6/7] fix(bigtable): continue stale resource cleanup on non-NotFoundException errors --- .../env/StaleResourceCleaner.java | 238 +++++++++++++----- .../env/StaleResourceCleanerTest.java | 91 +++++++ 2 files changed, 260 insertions(+), 69 deletions(-) diff --git a/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/test_helpers/env/StaleResourceCleaner.java b/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/test_helpers/env/StaleResourceCleaner.java index 0ed24e89582e..ab067975738b 100644 --- a/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/test_helpers/env/StaleResourceCleaner.java +++ b/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/test_helpers/env/StaleResourceCleaner.java @@ -29,7 +29,6 @@ import com.google.cloud.bigtable.admin.v2.models.UpdateLogicalViewRequest; import com.google.cloud.bigtable.admin.v2.models.UpdateMaterializedViewRequest; import com.google.cloud.bigtable.admin.v2.models.UpdateTableRequest; -import java.io.IOException; import java.time.Instant; import java.time.temporal.ChronoUnit; import java.util.ArrayList; @@ -47,6 +46,7 @@ public class StaleResourceCleaner { private static final Logger LOGGER = Logger.getLogger(StaleResourceCleaner.class.getName()); private final AbstractTestEnv testEnv; + private final List errors = new ArrayList<>(); StaleResourceCleaner(AbstractTestEnv testEnv) { this.testEnv = testEnv; @@ -66,12 +66,13 @@ static boolean isStale(String resourceId, String stalePrefix) { return resourceId.startsWith(PrefixGenerator.PREFIX) && resourceId.compareTo(stalePrefix) < 0; } - public void cleanUp() throws IOException, ExecutionException, InterruptedException { + public void cleanUp() { String stalePrefix = PrefixGenerator.newTimePrefix(Instant.now().minus(1, ChronoUnit.DAYS)); cleanUp(stalePrefix); } - void cleanUp(String stalePrefix) throws IOException, ExecutionException, InterruptedException { + void cleanUp(String stalePrefix) { + errors.clear(); String instanceId = testEnv.getInstanceId(); LOGGER.info( "Starting stale resource cleanup with stalePrefix=" @@ -80,10 +81,19 @@ void cleanUp(String stalePrefix) throws IOException, ExecutionException, Interru + instanceId); cleanUpSharedInstance(instanceId, stalePrefix); cleanUpStaleInstances(stalePrefix); + + if (!errors.isEmpty()) { + RuntimeException exception = + new RuntimeException( + "Stale resource cleanup encountered " + errors.size() + " error(s)", errors.get(0)); + for (int i = 1; i < errors.size(); i++) { + exception.addSuppressed(errors.get(i)); + } + throw exception; + } } - void cleanUpSharedInstance(String instanceId, String stalePrefix) - throws ExecutionException, InterruptedException { + void cleanUpSharedInstance(String instanceId, String stalePrefix) { LOGGER.info("Cleaning up shared instance: " + instanceId); BigtableTableAdminClient tableAdmin = testEnv.getTableAdminClient(); cleanUpStaleMaterializedViews(instanceId, stalePrefix); @@ -95,7 +105,14 @@ void cleanUpSharedInstance(String instanceId, String stalePrefix) private void cleanUpStaleMaterializedViews(String instanceId, String stalePrefix) { BigtableInstanceAdminClient instanceAdmin = testEnv.getInstanceAdminClient(); - for (MaterializedView view : instanceAdmin.listMaterializedViews(instanceId)) { + List views; + try { + views = instanceAdmin.listMaterializedViews(instanceId); + } catch (Exception e) { + recordError("Failed to list materialized views in instance: " + instanceId, e); + return; + } + for (MaterializedView view : views) { if (!isStale(view.getId(), stalePrefix)) { continue; } @@ -114,14 +131,26 @@ private void unprotectAndDeleteMaterializedView(String instanceId, MaterializedV .setDeletionProtection(false)); } instanceAdmin.deleteMaterializedView(instanceId, view.getId()); - } catch (NotFoundException ignored) { - // nothing to clean up, the materialized view was already deleted + } catch (Exception e) { + recordError( + "Failed to delete stale materialized view: " + + view.getId() + + " in instance " + + instanceId, + e); } } private void cleanUpStaleLogicalViews(String instanceId, String stalePrefix) { BigtableInstanceAdminClient instanceAdmin = testEnv.getInstanceAdminClient(); - for (LogicalView view : instanceAdmin.listLogicalViews(instanceId)) { + List views; + try { + views = instanceAdmin.listLogicalViews(instanceId); + } catch (Exception e) { + recordError("Failed to list logical views in instance: " + instanceId, e); + return; + } + for (LogicalView view : views) { if (!isStale(view.getId(), stalePrefix)) { continue; } @@ -138,15 +167,23 @@ private void unprotectAndDeleteLogicalView(String instanceId, LogicalView view) UpdateLogicalViewRequest.of(instanceId, view.getId()).setDeletionProtection(false)); } instanceAdmin.deleteLogicalView(instanceId, view.getId()); - } catch (NotFoundException ignored) { - // nothing to clean up, the logical view was already deleted + } catch (Exception e) { + recordError( + "Failed to delete stale logical view: " + view.getId() + " in instance " + instanceId, e); } } private void cleanUpTablesAndAuthorizedViews( BigtableTableAdminClient tableAdmin, String stalePrefix) { LOGGER.info("Start cleaning up stale tables with stalePrefix=" + stalePrefix); - for (String tableId : tableAdmin.listTables()) { + List tableIds; + try { + tableIds = tableAdmin.listTables(); + } catch (Exception e) { + recordError("Failed to list tables", e); + return; + } + for (String tableId : tableIds) { if (!tableId.startsWith(PrefixGenerator.PREFIX)) { cleanUpStaleAuthorizedViewsOnPersistentTable(tableAdmin, tableId, stalePrefix); continue; @@ -157,8 +194,8 @@ private void cleanUpTablesAndAuthorizedViews( prepTableForDelete(tableAdmin, tableId); LOGGER.info("Deleting stale table: " + tableId); tableAdmin.deleteTable(tableId); - } catch (NotFoundException e) { - LOGGER.log(Level.WARNING, "Deleting stale table failed: " + tableId, e); + } catch (Exception e) { + recordError("Deleting stale table failed: " + tableId, e); } } } @@ -177,58 +214,84 @@ private void prepTableForDelete(BigtableTableAdminClient tableAdmin, String tabl try { tableAdmin.updateAuthorizedView( UpdateAuthorizedViewRequest.of(tableId, viewId).setDeletionProtection(false)); - } catch (NotFoundException ignored) { - // nothing to clean up, the view was already deleted + } catch (Exception e) { + recordError( + "Failed to unprotect authorized view: " + viewId + " on stale table: " + tableId, e); } } } private void cleanUpStaleAuthorizedViewsOnPersistentTable( BigtableTableAdminClient tableAdmin, String tableId, String stalePrefix) { + List viewIds; try { - for (String viewId : tableAdmin.listAuthorizedViews(tableId)) { - if (!isStale(viewId, stalePrefix)) { - continue; - } - LOGGER.info( - "Deleting stale authorized view: " + viewId + " on persistent table: " + tableId); - try { - tableAdmin.updateAuthorizedView( - UpdateAuthorizedViewRequest.of(tableId, viewId).setDeletionProtection(false)); - tableAdmin.deleteAuthorizedView(tableId, viewId); - } catch (NotFoundException ignored) { - // nothing to clean up, the view was already deleted - } + viewIds = tableAdmin.listAuthorizedViews(tableId); + } catch (Exception e) { + recordError("Failed to list authorized views on persistent table: " + tableId, e); + return; + } + for (String viewId : viewIds) { + if (!isStale(viewId, stalePrefix)) { + continue; + } + LOGGER.info("Deleting stale authorized view: " + viewId + " on persistent table: " + tableId); + try { + tableAdmin.updateAuthorizedView( + UpdateAuthorizedViewRequest.of(tableId, viewId).setDeletionProtection(false)); + tableAdmin.deleteAuthorizedView(tableId, viewId); + } catch (Exception e) { + recordError( + "Failed to delete stale authorized view: " + + viewId + + " on persistent table: " + + tableId, + e); } - } catch (NotFoundException ignored) { - // nothing to clean up, the table was already deleted } } private void cleanUpStaleAppProfiles(String instanceId, String stalePrefix) { BigtableInstanceAdminClient instanceAdmin = testEnv.getInstanceAdminClient(); - for (AppProfile appProfile : instanceAdmin.listAppProfiles(instanceId)) { + List appProfiles; + try { + appProfiles = instanceAdmin.listAppProfiles(instanceId); + } catch (Exception e) { + recordError("Failed to list app profiles in instance: " + instanceId, e); + return; + } + for (AppProfile appProfile : appProfiles) { if (!isStale(appProfile.getId(), stalePrefix)) { continue; } try { instanceAdmin.deleteAppProfile(instanceId, appProfile.getId(), true); - } catch (NotFoundException ignored) { - // nothing to clean up, the app profile was already deleted + } catch (Exception e) { + recordError( + "Failed to delete stale app profile: " + + appProfile.getId() + + " in instance " + + instanceId, + e); } } } private void cleanUpClustersAndBackups( - String instanceId, BigtableTableAdminClient tableAdmin, String stalePrefix) - throws ExecutionException, InterruptedException { + String instanceId, BigtableTableAdminClient tableAdmin, String stalePrefix) { BigtableInstanceAdminClient instanceAdmin = testEnv.getInstanceAdminClient(); - for (Cluster cluster : instanceAdmin.listClusters(instanceId)) { + List clusters; + try { + clusters = instanceAdmin.listClusters(instanceId); + } catch (Exception e) { + recordError("Failed to list clusters in instance: " + instanceId, e); + return; + } + for (Cluster cluster : clusters) { if (!cluster.getId().startsWith(PrefixGenerator.PREFIX)) { try { deleteStaleBackups(tableAdmin, cluster.getId(), stalePrefix); - } catch (NotFoundException ignored) { - // nothing to clean up, the cluster or backup was already deleted + } catch (Exception e) { + recordError("Failed to delete stale backups in cluster: " + cluster.getId(), e); } continue; } @@ -238,45 +301,63 @@ private void cleanUpClustersAndBackups( try { deleteAllBackups(tableAdmin, cluster.getId()); - } catch (NotFoundException ignored) { - // nothing to clean up, the backup was already deleted + } catch (Exception e) { + recordError("Failed to delete backups in stale cluster: " + cluster.getId(), e); } try { instanceAdmin.deleteCluster(instanceId, cluster.getId()); - } catch (NotFoundException ignored) { - // nothing to clean up, the cluster was already deleted + } catch (Exception e) { + recordError( + "Failed to delete stale cluster: " + cluster.getId() + " in instance " + instanceId, e); } } } - private void cleanUpStaleInstances(String stalePrefix) - throws IOException, ExecutionException, InterruptedException { + private void cleanUpStaleInstances(String stalePrefix) { BigtableInstanceAdminClient instanceAdmin = testEnv.getInstanceAdminClient(); - for (Instance instance : instanceAdmin.listInstances()) { + List instances; + try { + instances = instanceAdmin.listInstances(); + } catch (Exception e) { + recordError("Failed to list instances", e); + return; + } + for (Instance instance : instances) { if (!isStale(instance.getId(), stalePrefix)) { continue; } try { prepInstanceForDelete(instance.getId()); deleteInstance(instance.getId()); - } catch (NotFoundException ignored) { - // nothing to clean up, the instance was already deleted + } catch (Exception e) { + recordError("Failed to delete stale instance: " + instance.getId(), e); } } } private void prepInstanceForDelete(String instanceId) { BigtableInstanceAdminClient instanceAdmin = testEnv.getInstanceAdminClient(); - for (MaterializedView materializedView : instanceAdmin.listMaterializedViews(instanceId)) { - unprotectAndDeleteMaterializedView(instanceId, materializedView); + try { + for (MaterializedView materializedView : instanceAdmin.listMaterializedViews(instanceId)) { + unprotectAndDeleteMaterializedView(instanceId, materializedView); + } + } catch (NotFoundException e) { + throw e; + } catch (Exception e) { + recordError("Failed to list materialized views in stale instance: " + instanceId, e); } - for (LogicalView logicalView : instanceAdmin.listLogicalViews(instanceId)) { - unprotectAndDeleteLogicalView(instanceId, logicalView); + try { + for (LogicalView logicalView : instanceAdmin.listLogicalViews(instanceId)) { + unprotectAndDeleteLogicalView(instanceId, logicalView); + } + } catch (NotFoundException e) { + throw e; + } catch (Exception e) { + recordError("Failed to list logical views in stale instance: " + instanceId, e); } } - private void deleteInstance(String instanceId) - throws IOException, ExecutionException, InterruptedException { + private void deleteInstance(String instanceId) { BigtableInstanceAdminClient instanceAdmin = testEnv.getInstanceAdminClient(); try (BigtableTableAdminClient tableAdmin = testEnv.getTableAdminClientForInstance(instanceId)) { List clusters = instanceAdmin.listClusters(instanceId); @@ -286,31 +367,39 @@ private void deleteInstance(String instanceId) for (Cluster cluster : clusters) { try { deleteAllBackups(tableAdmin, cluster.getId()); - } catch (NotFoundException ignored) { - // nothing to clean up, the cluster or backup was already deleted + } catch (Exception e) { + recordError( + "Failed to delete backups in cluster " + + cluster.getId() + + " of stale instance " + + instanceId, + e); } // Skip the first cluster so that it can be deleted by deleteInstance (instances can't exist // without clusters) if (!isFirstCluster) { try { instanceAdmin.deleteCluster(instanceId, cluster.getId()); - } catch (NotFoundException ignored) { - // nothing to clean up, the cluster was already deleted + } catch (Exception e) { + recordError( + "Failed to delete cluster " + cluster.getId() + " of stale instance " + instanceId, + e); } } isFirstCluster = false; } + } catch (Exception e) { + recordError("Failed to clean up clusters/backups in stale instance: " + instanceId, e); } try { instanceAdmin.deleteInstance(instanceId); - } catch (NotFoundException ignored) { - // nothing to clean up, the instance was already deleted + } catch (Exception e) { + recordError("Failed to delete stale instance: " + instanceId, e); } } - private void deleteAllBackups(BigtableTableAdminClient tableAdmin, String clusterId) - throws ExecutionException, InterruptedException { + private void deleteAllBackups(BigtableTableAdminClient tableAdmin, String clusterId) { List> futures = new ArrayList<>(); for (String backupId : tableAdmin.listBackups(clusterId)) { @@ -318,12 +407,11 @@ private void deleteAllBackups(BigtableTableAdminClient tableAdmin, String cluste futures.add(f); } - awaitBackupDeletions(futures); + awaitBackupDeletions(futures, clusterId); } private void deleteStaleBackups( - BigtableTableAdminClient tableAdmin, String clusterId, String stalePrefix) - throws ExecutionException, InterruptedException { + BigtableTableAdminClient tableAdmin, String clusterId, String stalePrefix) { List> futures = new ArrayList<>(); for (String backupId : tableAdmin.listBackups(clusterId)) { @@ -333,11 +421,10 @@ private void deleteStaleBackups( } } - awaitBackupDeletions(futures); + awaitBackupDeletions(futures, clusterId); } - private void awaitBackupDeletions(List> futures) - throws ExecutionException, InterruptedException { + private void awaitBackupDeletions(List> futures, String clusterId) { for (ApiFuture future : futures) { try { future.get(); @@ -345,8 +432,21 @@ private void awaitBackupDeletions(List> futures) if (e.getCause() instanceof NotFoundException) { continue; } - throw e; + recordError("Failed to delete backup in cluster: " + clusterId, e); + } catch (Exception e) { + recordError("Failed to delete backup in cluster: " + clusterId, e); } } } + + private void recordError(String message, Exception e) { + if (e instanceof NotFoundException) { + return; + } + if (e instanceof InterruptedException) { + Thread.currentThread().interrupt(); + } + LOGGER.log(Level.WARNING, message, e); + errors.add(e); + } } diff --git a/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/test_helpers/env/StaleResourceCleanerTest.java b/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/test_helpers/env/StaleResourceCleanerTest.java index e03d3671ea89..30dfab30ec34 100644 --- a/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/test_helpers/env/StaleResourceCleanerTest.java +++ b/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/test_helpers/env/StaleResourceCleanerTest.java @@ -15,6 +15,8 @@ */ package com.google.cloud.bigtable.test_helpers.env; +import static com.google.common.truth.Truth.assertThat; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.inOrder; @@ -30,6 +32,7 @@ import com.google.api.gax.longrunning.OperationFuture; import com.google.api.gax.longrunning.OperationFutures; import com.google.api.gax.longrunning.OperationSnapshot; +import com.google.api.gax.rpc.InternalException; import com.google.api.gax.rpc.NotFoundException; import com.google.api.gax.rpc.OperationCallable; import com.google.api.gax.rpc.UnaryCallable; @@ -772,6 +775,94 @@ void testConcurrentNotFoundExceptionsAreIgnored() throws Exception { .build()); } + @Test + @SuppressWarnings("unchecked") + void testNonNotFoundErrorsContinueCleanupAndThrowAtEnd() throws Exception { + EnhancedBigtableTableAdminStub sharedTableStub = createTableStubForInstance(SHARED_INSTANCE_ID); + InternalException internalError = + new InternalException(null, GrpcStatusCode.of(Code.INTERNAL), false); + + // 1. Materialized views: first stale view throws InternalException on delete, second succeeds + String failingMv = STALE_ID_PREFIX + "mv-fail"; + String succeedingMv = STALE_ID_PREFIX + "mv-ok"; + mockMaterializedViews( + SHARED_INSTANCE_ID, + ImmutableList.of( + buildMaterializedView(SHARED_INSTANCE_ID, failingMv, false), + buildMaterializedView(SHARED_INSTANCE_ID, succeedingMv, false))); + when(mockDeleteMaterializedViewCallable.futureCall( + DeleteMaterializedViewRequest.newBuilder() + .setName( + NameUtil.formatMaterializedViewName(PROJECT_ID, SHARED_INSTANCE_ID, failingMv)) + .build())) + .thenReturn(ApiFutures.immediateFailedFuture(internalError)); + + mockLogicalViews(SHARED_INSTANCE_ID, ImmutableList.of()); + mockListTables(sharedTableStub, SHARED_INSTANCE_ID, ImmutableList.of()); + + // 2. App profiles: first stale app profile throws InternalException on delete, second succeeds + String failingAp = STALE_ID_PREFIX + "ap-fail"; + String succeedingAp = STALE_ID_PREFIX + "ap-ok"; + mockAppProfiles(SHARED_INSTANCE_ID, ImmutableList.of(failingAp, succeedingAp)); + when(mockDeleteAppProfileCallable.futureCall( + DeleteAppProfileRequest.newBuilder() + .setName(NameUtil.formatAppProfileName(PROJECT_ID, SHARED_INSTANCE_ID, failingAp)) + .setIgnoreWarnings(true) + .build())) + .thenReturn(ApiFutures.immediateFailedFuture(internalError)); + + mockClusters(SHARED_INSTANCE_ID, ImmutableList.of()); + + // 3. Stale instances: first stale instance fails to delete, second stale instance succeeds + String failingInst = STALE_ID_PREFIX + "inst-fail"; + String succeedingInst = STALE_ID_PREFIX + "inst-ok"; + mockInstances(ImmutableList.of(failingInst, succeedingInst)); + + mockMaterializedViews(failingInst, ImmutableList.of()); + mockLogicalViews(failingInst, ImmutableList.of()); + mockClusters(failingInst, ImmutableList.of()); + createTableStubForInstance(failingInst); + when(mockDeleteInstanceCallable.futureCall( + DeleteInstanceRequest.newBuilder() + .setName(NameUtil.formatInstanceName(PROJECT_ID, failingInst)) + .build())) + .thenReturn(ApiFutures.immediateFailedFuture(internalError)); + + mockMaterializedViews(succeedingInst, ImmutableList.of()); + mockLogicalViews(succeedingInst, ImmutableList.of()); + mockClusters(succeedingInst, ImmutableList.of()); + createTableStubForInstance(succeedingInst); + + StaleResourceCleaner cleaner = createCleaner(); + RuntimeException thrown = + assertThrows(RuntimeException.class, () -> cleaner.cleanUp(STALE_PREFIX)); + + assertThat(thrown.getMessage()).contains("3 error(s)"); + assertThat(thrown.getCause()).isNotNull(); + assertThat(thrown.getSuppressed()).hasLength(2); + + // Subsequent resources in each phase were still cleaned up despite earlier failures + verify(mockDeleteMaterializedViewCallable) + .futureCall( + DeleteMaterializedViewRequest.newBuilder() + .setName( + NameUtil.formatMaterializedViewName( + PROJECT_ID, SHARED_INSTANCE_ID, succeedingMv)) + .build()); + verify(mockDeleteAppProfileCallable) + .futureCall( + DeleteAppProfileRequest.newBuilder() + .setName( + NameUtil.formatAppProfileName(PROJECT_ID, SHARED_INSTANCE_ID, succeedingAp)) + .setIgnoreWarnings(true) + .build()); + verify(mockDeleteInstanceCallable) + .futureCall( + DeleteInstanceRequest.newBuilder() + .setName(NameUtil.formatInstanceName(PROJECT_ID, succeedingInst)) + .build()); + } + private StaleResourceCleaner createCleaner() throws Exception { AbstractTestEnv mockEnv = mock(AbstractTestEnv.class); when(mockEnv.getInstanceId()).thenReturn(SHARED_INSTANCE_ID); From eab22ef9944950c788aa9cf407d93459d6b1c16b Mon Sep 17 00:00:00 2001 From: Igor Bernstein Date: Wed, 18 Nov 2026 16:03:19 +0000 Subject: [PATCH 7/7] test(bigtable): use withoutAnnotations() on all mocks in StaleResourceCleanerTest for JDK 8 --- .../env/StaleResourceCleanerTest.java | 38 ++++++++++++------- 1 file changed, 24 insertions(+), 14 deletions(-) diff --git a/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/test_helpers/env/StaleResourceCleanerTest.java b/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/test_helpers/env/StaleResourceCleanerTest.java index 30dfab30ec34..86a54802fc93 100644 --- a/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/test_helpers/env/StaleResourceCleanerTest.java +++ b/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/test_helpers/env/StaleResourceCleanerTest.java @@ -864,7 +864,7 @@ void testNonNotFoundErrorsContinueCleanupAndThrowAtEnd() throws Exception { } private StaleResourceCleaner createCleaner() throws Exception { - AbstractTestEnv mockEnv = mock(AbstractTestEnv.class); + AbstractTestEnv mockEnv = mock(AbstractTestEnv.class, withSettings().withoutAnnotations()); when(mockEnv.getInstanceId()).thenReturn(SHARED_INSTANCE_ID); when(mockEnv.getInstanceAdminClient()).thenReturn(instanceAdmin); when(mockEnv.getTableAdminClient()) @@ -888,7 +888,8 @@ private EnhancedBigtableTableAdminStub createTableStubForInstance(String instanc Field backgroundResourcesField = GrpcBigtableTableAdminStub.class.getDeclaredField("backgroundResources"); backgroundResourcesField.setAccessible(true); - backgroundResourcesField.set(stub, mock(BackgroundResource.class)); + backgroundResourcesField.set( + stub, mock(BackgroundResource.class, withSettings().withoutAnnotations())); tableStubsByInstance.put(instanceId, stub); return stub; } @@ -912,10 +913,12 @@ private LogicalView buildLogicalView( } private void mockMaterializedViews(String instanceId, List views) { - ListMaterializedViewsPage page = mock(ListMaterializedViewsPage.class); + ListMaterializedViewsPage page = + mock(ListMaterializedViewsPage.class, withSettings().withoutAnnotations()); when(page.getValues()).thenReturn(views); when(page.hasNextPage()).thenReturn(false); - ListMaterializedViewsPagedResponse response = mock(ListMaterializedViewsPagedResponse.class); + ListMaterializedViewsPagedResponse response = + mock(ListMaterializedViewsPagedResponse.class, withSettings().withoutAnnotations()); when(response.getPage()).thenReturn(page); when(mockListMaterializedViewsCallable.futureCall( ListMaterializedViewsRequest.newBuilder() @@ -925,10 +928,12 @@ private void mockMaterializedViews(String instanceId, List vie } private void mockLogicalViews(String instanceId, List views) { - ListLogicalViewsPage page = mock(ListLogicalViewsPage.class); + ListLogicalViewsPage page = + mock(ListLogicalViewsPage.class, withSettings().withoutAnnotations()); when(page.getValues()).thenReturn(views); when(page.hasNextPage()).thenReturn(false); - ListLogicalViewsPagedResponse response = mock(ListLogicalViewsPagedResponse.class); + ListLogicalViewsPagedResponse response = + mock(ListLogicalViewsPagedResponse.class, withSettings().withoutAnnotations()); when(response.getPage()).thenReturn(page); when(mockListLogicalViewsCallable.futureCall( ListLogicalViewsRequest.newBuilder() @@ -951,10 +956,11 @@ private void mockListTables( .setName(NameUtil.formatTableName(PROJECT_ID, instanceId, tableId)) .build()); } - ListTablesPage page = mock(ListTablesPage.class); + ListTablesPage page = mock(ListTablesPage.class, withSettings().withoutAnnotations()); when(page.getValues()).thenReturn(tables); when(page.hasNextPage()).thenReturn(false); - ListTablesPagedResponse response = mock(ListTablesPagedResponse.class); + ListTablesPagedResponse response = + mock(ListTablesPagedResponse.class, withSettings().withoutAnnotations()); when(response.getPage()).thenReturn(page); when(mockListTablesCallable.futureCall( ListTablesRequest.newBuilder() @@ -975,10 +981,12 @@ private void mockAuthorizedViews( .setName(NameUtil.formatAuthorizedViewName(PROJECT_ID, instanceId, tableId, viewId)) .build()); } - ListAuthorizedViewsPage page = mock(ListAuthorizedViewsPage.class); + ListAuthorizedViewsPage page = + mock(ListAuthorizedViewsPage.class, withSettings().withoutAnnotations()); when(page.getValues()).thenReturn(views); when(page.hasNextPage()).thenReturn(false); - ListAuthorizedViewsPagedResponse response = mock(ListAuthorizedViewsPagedResponse.class); + ListAuthorizedViewsPagedResponse response = + mock(ListAuthorizedViewsPagedResponse.class, withSettings().withoutAnnotations()); when(response.getPage()).thenReturn(page); when(callable.futureCall( ListAuthorizedViewsRequest.newBuilder() @@ -997,10 +1005,11 @@ private void mockAppProfiles(String instanceId, List appProfileIds) { AppProfile.MultiClusterRoutingUseAny.getDefaultInstance()) .build()); } - ListAppProfilesPage page = mock(ListAppProfilesPage.class); + ListAppProfilesPage page = mock(ListAppProfilesPage.class, withSettings().withoutAnnotations()); when(page.getValues()).thenReturn(profiles); when(page.hasNextPage()).thenReturn(false); - ListAppProfilesPagedResponse response = mock(ListAppProfilesPagedResponse.class); + ListAppProfilesPagedResponse response = + mock(ListAppProfilesPagedResponse.class, withSettings().withoutAnnotations()); when(response.getPage()).thenReturn(page); when(mockListAppProfilesCallable.futureCall( ListAppProfilesRequest.newBuilder() @@ -1039,10 +1048,11 @@ private void mockBackups( .setName(NameUtil.formatBackupName(PROJECT_ID, instanceId, clusterId, backupId)) .build()); } - ListBackupsPage page = mock(ListBackupsPage.class); + ListBackupsPage page = mock(ListBackupsPage.class, withSettings().withoutAnnotations()); when(page.getValues()).thenReturn(backups); when(page.hasNextPage()).thenReturn(false); - ListBackupsPagedResponse response = mock(ListBackupsPagedResponse.class); + ListBackupsPagedResponse response = + mock(ListBackupsPagedResponse.class, withSettings().withoutAnnotations()); when(response.getPage()).thenReturn(page); when(callable.futureCall( ListBackupsRequest.newBuilder()