From 0f23635653f5220178fd0b1a4424241528a52d9f Mon Sep 17 00:00:00 2001 From: 205762370+Rob9786 <205762370+Rob9786@users.noreply.github.com> Date: Fri, 15 Aug 2025 13:53:07 +0100 Subject: [PATCH 01/14] 5425: Initial commit of new class --- .../cdk/util/AutoStopEcsClusterTasks.java | 67 ++++++ .../sleeper/core/deploy/LambdaHandler.java | 220 +++++++++--------- 2 files changed, 179 insertions(+), 108 deletions(-) create mode 100644 java/cdk/src/main/java/sleeper/cdk/util/AutoStopEcsClusterTasks.java diff --git a/java/cdk/src/main/java/sleeper/cdk/util/AutoStopEcsClusterTasks.java b/java/cdk/src/main/java/sleeper/cdk/util/AutoStopEcsClusterTasks.java new file mode 100644 index 00000000000..56d42b0773e --- /dev/null +++ b/java/cdk/src/main/java/sleeper/cdk/util/AutoStopEcsClusterTasks.java @@ -0,0 +1,67 @@ +/* + * Copyright 2022-2025 Crown Copyright + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package sleeper.cdk.util; + +import software.amazon.awscdk.CustomResource; +import software.amazon.awscdk.Duration; +import software.amazon.awscdk.customresources.Provider; +import software.amazon.awscdk.services.ecs.ICluster; +import software.amazon.awscdk.services.lambda.IFunction; +import software.amazon.awscdk.services.logs.ILogGroup; +import software.constructs.Construct; + +import sleeper.cdk.jars.LambdaCode; +import sleeper.core.deploy.LambdaHandler; +import sleeper.core.properties.instance.InstanceProperties; +import sleeper.core.util.EnvironmentUtils; + +import java.util.Map; + +public class AutoStopEcsClusterTasks { + + private AutoStopEcsClusterTasks() { + } + + public static void autoStopTasksOnEcsCluster( + Construct scope, InstanceProperties instanceProperties, LambdaCode lambdaCode, + ICluster cluster, String clusterName, + ILogGroup logGroup, + ILogGroup providerLogGroup) { + + String id = cluster.getNode().getId() + "-AutoStop"; + String functionName = clusterName + "-autostop"; + + IFunction lambda = lambdaCode.buildFunction(scope, LambdaHandler.AUTO_STOP_ECS_CLUSTER_TASKS, id + "Lambda", builder -> builder + .functionName(functionName) + .memorySize(2048) + .environment(EnvironmentUtils.createDefaultEnvironmentNoConfigBucket(instanceProperties)) + .description("Lambda for auto-stopping ECS tasks") + .logGroup(logGroup) + .timeout(Duration.minutes(10))); + + Provider propertiesWriterProvider = Provider.Builder.create(scope, id + "Provider") + .onEventHandler(lambda) + .logGroup(providerLogGroup) + .build(); + + CustomResource.Builder.create(scope, id) + .resourceType("Custom::AutoStopEcsClusterTasks") + .properties(Map.of("cluster", clusterName)) + .serviceToken(propertiesWriterProvider.getServiceToken()) + .build(); + } + +} diff --git a/java/core/src/main/java/sleeper/core/deploy/LambdaHandler.java b/java/core/src/main/java/sleeper/core/deploy/LambdaHandler.java index 3ac85f7a3a9..5f0a65403e7 100644 --- a/java/core/src/main/java/sleeper/core/deploy/LambdaHandler.java +++ b/java/core/src/main/java/sleeper/core/deploy/LambdaHandler.java @@ -35,149 +35,153 @@ public class LambdaHandler { private static final List ALL = new ArrayList<>(); private static final Map ATHENA_HANDLER_BY_CLASSNAME = new HashMap<>(); public static final LambdaHandler ATHENA_SIMPLE_COMPOSITE = builder() - .jar(LambdaJar.ATHENA) - .handler("sleeper.athena.composite.SimpleCompositeHandler") - .optionalStack(OptionalStack.AthenaStack).add(); + .jar(LambdaJar.ATHENA) + .handler("sleeper.athena.composite.SimpleCompositeHandler") + .optionalStack(OptionalStack.AthenaStack).add(); public static final LambdaHandler ATHENA_ITERATORS_COMPOSITE = builder() - .jar(LambdaJar.ATHENA) - .handler("sleeper.athena.composite.IteratorApplyingCompositeHandler") - .optionalStack(OptionalStack.AthenaStack).add(); + .jar(LambdaJar.ATHENA) + .handler("sleeper.athena.composite.IteratorApplyingCompositeHandler") + .optionalStack(OptionalStack.AthenaStack).add(); public static final LambdaHandler BULK_IMPORT_STARTER = builder() - .jar(LambdaJar.BULK_IMPORT_STARTER) - .handler("sleeper.bulkimport.starter.BulkImportStarterLambda") - .optionalStacks(OptionalStack.BULK_IMPORT_STACKS).add(); + .jar(LambdaJar.BULK_IMPORT_STARTER) + .handler("sleeper.bulkimport.starter.BulkImportStarterLambda") + .optionalStacks(OptionalStack.BULK_IMPORT_STACKS).add(); public static final LambdaHandler INGEST_TASK_CREATOR = builder() - .jar(LambdaJar.INGEST_TASK_CREATOR) - .handler("sleeper.ingest.taskrunner.RunIngestTasksLambda::eventHandler") - .optionalStack(OptionalStack.IngestStack).add(); + .jar(LambdaJar.INGEST_TASK_CREATOR) + .handler("sleeper.ingest.taskrunner.RunIngestTasksLambda::eventHandler") + .optionalStack(OptionalStack.IngestStack).add(); public static final LambdaHandler INGEST_BATCHER_SUBMITTER = builder() - .jar(LambdaJar.INGEST_BATCHER_SUBMITTER) - .handler("sleeper.ingest.batcher.submitter.IngestBatcherSubmitterLambda::handleRequest") - .optionalStack(OptionalStack.IngestBatcherStack).add(); + .jar(LambdaJar.INGEST_BATCHER_SUBMITTER) + .handler("sleeper.ingest.batcher.submitter.IngestBatcherSubmitterLambda::handleRequest") + .optionalStack(OptionalStack.IngestBatcherStack).add(); public static final LambdaHandler INGEST_BATCHER_JOB_CREATOR = builder() - .jar(LambdaJar.INGEST_BATCHER_JOB_CREATOR) - .handler("sleeper.ingest.batcher.job.creator.IngestBatcherJobCreatorLambda::eventHandler") - .optionalStack(OptionalStack.IngestBatcherStack).add(); + .jar(LambdaJar.INGEST_BATCHER_JOB_CREATOR) + .handler("sleeper.ingest.batcher.job.creator.IngestBatcherJobCreatorLambda::eventHandler") + .optionalStack(OptionalStack.IngestBatcherStack).add(); public static final LambdaHandler GARBAGE_COLLECTOR_TRIGGER = builder() - .jar(LambdaJar.GARBAGE_COLLECTOR) - .handler("sleeper.garbagecollector.GarbageCollectorTriggerLambda::handleRequest") - .optionalStack(OptionalStack.GarbageCollectorStack).add(); + .jar(LambdaJar.GARBAGE_COLLECTOR) + .handler("sleeper.garbagecollector.GarbageCollectorTriggerLambda::handleRequest") + .optionalStack(OptionalStack.GarbageCollectorStack).add(); public static final LambdaHandler GARBAGE_COLLECTOR = builder() - .jar(LambdaJar.GARBAGE_COLLECTOR) - .handler("sleeper.garbagecollector.GarbageCollectorLambda::handleRequest") - .optionalStack(OptionalStack.GarbageCollectorStack).add(); + .jar(LambdaJar.GARBAGE_COLLECTOR) + .handler("sleeper.garbagecollector.GarbageCollectorLambda::handleRequest") + .optionalStack(OptionalStack.GarbageCollectorStack).add(); public static final LambdaHandler COMPACTION_JOB_CREATOR_TRIGGER = builder() - .jar(LambdaJar.COMPACTION_JOB_CREATOR) - .handler("sleeper.compaction.job.creation.lambda.CreateCompactionJobsTriggerLambda::handleRequest") - .optionalStack(OptionalStack.CompactionStack).add(); + .jar(LambdaJar.COMPACTION_JOB_CREATOR) + .handler("sleeper.compaction.job.creation.lambda.CreateCompactionJobsTriggerLambda::handleRequest") + .optionalStack(OptionalStack.CompactionStack).add(); public static final LambdaHandler COMPACTION_JOB_CREATOR = builder() - .jar(LambdaJar.COMPACTION_JOB_CREATOR) - .handler("sleeper.compaction.job.creation.lambda.CreateCompactionJobsLambda::handleRequest") - .optionalStack(OptionalStack.CompactionStack).add(); + .jar(LambdaJar.COMPACTION_JOB_CREATOR) + .handler("sleeper.compaction.job.creation.lambda.CreateCompactionJobsLambda::handleRequest") + .optionalStack(OptionalStack.CompactionStack).add(); public static final LambdaHandler COMPACTION_JOB_DISPATCHER = builder() - .jar(LambdaJar.COMPACTION_JOB_CREATOR) - .handler("sleeper.compaction.job.creation.lambda.CompactionJobDispatchLambda::handleRequest") - .optionalStack(OptionalStack.CompactionStack).add(); + .jar(LambdaJar.COMPACTION_JOB_CREATOR) + .handler("sleeper.compaction.job.creation.lambda.CompactionJobDispatchLambda::handleRequest") + .optionalStack(OptionalStack.CompactionStack).add(); public static final LambdaHandler COMPACTION_TASK_CREATOR = builder() - .jar(LambdaJar.COMPACTION_TASK_CREATOR) - .handler("sleeper.compaction.task.creation.RunCompactionTasksLambda::eventHandler") - .optionalStack(OptionalStack.CompactionStack).add(); + .jar(LambdaJar.COMPACTION_TASK_CREATOR) + .handler("sleeper.compaction.task.creation.RunCompactionTasksLambda::eventHandler") + .optionalStack(OptionalStack.CompactionStack).add(); public static final LambdaHandler COMPACTION_TASK_TERMINATOR = builder() - .jar(LambdaJar.COMPACTION_TASK_CREATOR) - .handler("sleeper.compaction.task.creation.SafeTerminationLambda::handleRequest") - .optionalStack(OptionalStack.CompactionStack).add(); + .jar(LambdaJar.COMPACTION_TASK_CREATOR) + .handler("sleeper.compaction.task.creation.SafeTerminationLambda::handleRequest") + .optionalStack(OptionalStack.CompactionStack).add(); public static final LambdaHandler FIND_PARTITIONS_TO_SPLIT_TRIGGER = builder() - .jar(LambdaJar.PARTITION_SPLITTER) - .handler("sleeper.splitter.lambda.FindPartitionsToSplitTriggerLambda::handleRequest") - .optionalStack(OptionalStack.PartitionSplittingStack).add(); + .jar(LambdaJar.PARTITION_SPLITTER) + .handler("sleeper.splitter.lambda.FindPartitionsToSplitTriggerLambda::handleRequest") + .optionalStack(OptionalStack.PartitionSplittingStack).add(); public static final LambdaHandler FIND_PARTITIONS_TO_SPLIT = builder() - .jar(LambdaJar.PARTITION_SPLITTER) - .handler("sleeper.splitter.lambda.FindPartitionsToSplitLambda::handleRequest") - .optionalStack(OptionalStack.PartitionSplittingStack).add(); + .jar(LambdaJar.PARTITION_SPLITTER) + .handler("sleeper.splitter.lambda.FindPartitionsToSplitLambda::handleRequest") + .optionalStack(OptionalStack.PartitionSplittingStack).add(); public static final LambdaHandler SPLIT_PARTITION = builder() - .jar(LambdaJar.PARTITION_SPLITTER) - .handler("sleeper.splitter.lambda.SplitPartitionLambda::handleRequest") - .optionalStack(OptionalStack.PartitionSplittingStack).add(); + .jar(LambdaJar.PARTITION_SPLITTER) + .handler("sleeper.splitter.lambda.SplitPartitionLambda::handleRequest") + .optionalStack(OptionalStack.PartitionSplittingStack).add(); public static final LambdaHandler KEEP_QUERY_WARM = builder() - .jar(LambdaJar.QUERY) - .handler("sleeper.query.lambda.WarmQueryExecutorLambda::handleRequest") - .optionalStack(OptionalStack.KeepLambdaWarmStack).add(); + .jar(LambdaJar.QUERY) + .handler("sleeper.query.lambda.WarmQueryExecutorLambda::handleRequest") + .optionalStack(OptionalStack.KeepLambdaWarmStack).add(); public static final LambdaHandler BULK_EXPORT_PLANNER = builder() - .jar(LambdaJar.BULK_EXPORT_PLANNER) - .handler("sleeper.bulkexport.planner.SqsBulkExportProcessorLambda::handleRequest") - .optionalStack(BulkExportStack).add(); + .jar(LambdaJar.BULK_EXPORT_PLANNER) + .handler("sleeper.bulkexport.planner.SqsBulkExportProcessorLambda::handleRequest") + .optionalStack(BulkExportStack).add(); public static final LambdaHandler BULK_EXPORT_TASK_CREATOR = builder() - .jar(LambdaJar.BULK_EXPORT_TASK_CREATOR) - .handler("sleeper.bulkexport.taskcreator.SqsTriggeredBulkExportTaskRunnerLambda::handleRequest") - .optionalStack(BulkExportStack).add(); + .jar(LambdaJar.BULK_EXPORT_TASK_CREATOR) + .handler("sleeper.bulkexport.taskcreator.SqsTriggeredBulkExportTaskRunnerLambda::handleRequest") + .optionalStack(BulkExportStack).add(); public static final LambdaHandler QUERY_EXECUTOR = builder() - .jar(LambdaJar.QUERY) - .handler("sleeper.query.lambda.SqsQueryProcessorLambda::handleRequest") - .optionalStack(OptionalStack.QueryStack).add(); + .jar(LambdaJar.QUERY) + .handler("sleeper.query.lambda.SqsQueryProcessorLambda::handleRequest") + .optionalStack(OptionalStack.QueryStack).add(); public static final LambdaHandler QUERY_LEAF_PARTITION = builder() - .jar(LambdaJar.QUERY) - .handler("sleeper.query.lambda.SqsLeafPartitionQueryLambda::handleRequest") - .optionalStack(OptionalStack.QueryStack).add(); + .jar(LambdaJar.QUERY) + .handler("sleeper.query.lambda.SqsLeafPartitionQueryLambda::handleRequest") + .optionalStack(OptionalStack.QueryStack).add(); public static final LambdaHandler WEB_SOCKET_QUERY = builder() - .jar(LambdaJar.QUERY) - .handler("sleeper.query.lambda.WebSocketQueryProcessorLambda::handleRequest") - .optionalStack(OptionalStack.WebSocketQueryStack).add(); + .jar(LambdaJar.QUERY) + .handler("sleeper.query.lambda.WebSocketQueryProcessorLambda::handleRequest") + .optionalStack(OptionalStack.WebSocketQueryStack).add(); public static final LambdaHandler AUTO_DELETE_S3_OBJECTS = builder() - .jar(LambdaJar.CUSTOM_RESOURCES) - .handler("sleeper.cdk.custom.AutoDeleteS3ObjectsLambda::handleEvent") - .core().add(); + .jar(LambdaJar.CUSTOM_RESOURCES) + .handler("sleeper.cdk.custom.AutoDeleteS3ObjectsLambda::handleEvent") + .core().add(); + public static final LambdaHandler AUTO_STOP_ECS_CLUSTER_TASKS = builder() + .jar(LambdaJar.CUSTOM_RESOURCES) + .handler("sleeper.cdk.custom.AutoStopEcsClusterTasksLambda::handleEvent") + .core().add(); public static final LambdaHandler PROPERTIES_WRITER = builder() - .jar(LambdaJar.CUSTOM_RESOURCES) - .handler("sleeper.cdk.custom.PropertiesWriterLambda::handleEvent") - .core().add(); + .jar(LambdaJar.CUSTOM_RESOURCES) + .handler("sleeper.cdk.custom.PropertiesWriterLambda::handleEvent") + .core().add(); public static final LambdaHandler VPC_CHECK = builder() - .jar(LambdaJar.CUSTOM_RESOURCES) - .handler("sleeper.cdk.custom.VpcCheckLambda::handleEvent") - .core().add(); + .jar(LambdaJar.CUSTOM_RESOURCES) + .handler("sleeper.cdk.custom.VpcCheckLambda::handleEvent") + .core().add(); public static final LambdaHandler METRICS_TRIGGER = builder() - .jar(LambdaJar.METRICS) - .handler("sleeper.metrics.TableMetricsTriggerLambda::handleRequest") - .optionalStack(OptionalStack.TableMetricsStack).add(); + .jar(LambdaJar.METRICS) + .handler("sleeper.metrics.TableMetricsTriggerLambda::handleRequest") + .optionalStack(OptionalStack.TableMetricsStack).add(); public static final LambdaHandler METRICS = builder() - .jar(LambdaJar.METRICS) - .handler("sleeper.metrics.TableMetricsLambda::handleRequest") - .optionalStack(OptionalStack.TableMetricsStack).add(); + .jar(LambdaJar.METRICS) + .handler("sleeper.metrics.TableMetricsLambda::handleRequest") + .optionalStack(OptionalStack.TableMetricsStack).add(); public static final LambdaHandler STATESTORE_COMMITTER = builder() - .jar(LambdaJar.STATESTORE) - .handler("sleeper.statestore.lambda.committer.StateStoreCommitterLambda::handleRequest") - .core().add(); + .jar(LambdaJar.STATESTORE) + .handler("sleeper.statestore.lambda.committer.StateStoreCommitterLambda::handleRequest") + .core().add(); public static final LambdaHandler COMPACTION_COMMIT_BATCHER = builder() - .jar(LambdaJar.STATESTORE) - .handler("sleeper.statestore.lambda.committer.CompactionCommitBatcherLambda::handleRequest") - .core().add(); + .jar(LambdaJar.STATESTORE) + .handler("sleeper.statestore.lambda.committer.CompactionCommitBatcherLambda::handleRequest") + .core().add(); public static final LambdaHandler SNAPSHOT_CREATION_TRIGGER = builder() - .jar(LambdaJar.STATESTORE) - .handler("sleeper.statestore.lambda.snapshot.TransactionLogSnapshotCreationTriggerLambda::handleRequest") - .core().add(); + .jar(LambdaJar.STATESTORE) + .handler("sleeper.statestore.lambda.snapshot.TransactionLogSnapshotCreationTriggerLambda::handleRequest") + .core().add(); public static final LambdaHandler SNAPSHOT_CREATION = builder() - .jar(LambdaJar.STATESTORE) - .handler("sleeper.statestore.lambda.snapshot.TransactionLogSnapshotCreationLambda::handleRequest") - .core().add(); + .jar(LambdaJar.STATESTORE) + .handler("sleeper.statestore.lambda.snapshot.TransactionLogSnapshotCreationLambda::handleRequest") + .core().add(); public static final LambdaHandler SNAPSHOT_DELETION_TRIGGER = builder() - .jar(LambdaJar.STATESTORE) - .handler("sleeper.statestore.lambda.snapshot.TransactionLogSnapshotDeletionTriggerLambda::handleRequest") - .core().add(); + .jar(LambdaJar.STATESTORE) + .handler("sleeper.statestore.lambda.snapshot.TransactionLogSnapshotDeletionTriggerLambda::handleRequest") + .core().add(); public static final LambdaHandler SNAPSHOT_DELETION = builder() - .jar(LambdaJar.STATESTORE) - .handler("sleeper.statestore.lambda.snapshot.TransactionLogSnapshotDeletionLambda::handleRequest") - .core().add(); + .jar(LambdaJar.STATESTORE) + .handler("sleeper.statestore.lambda.snapshot.TransactionLogSnapshotDeletionLambda::handleRequest") + .core().add(); public static final LambdaHandler TRANSACTION_DELETION_TRIGGER = builder() - .jar(LambdaJar.STATESTORE) - .handler("sleeper.statestore.lambda.transaction.TransactionLogTransactionDeletionTriggerLambda::handleRequest") - .core().add(); + .jar(LambdaJar.STATESTORE) + .handler("sleeper.statestore.lambda.transaction.TransactionLogTransactionDeletionTriggerLambda::handleRequest") + .core().add(); public static final LambdaHandler TRANSACTION_DELETION = builder() - .jar(LambdaJar.STATESTORE) - .handler("sleeper.statestore.lambda.transaction.TransactionLogTransactionDeletionLambda::handleRequest") - .core().add(); + .jar(LambdaJar.STATESTORE) + .handler("sleeper.statestore.lambda.transaction.TransactionLogTransactionDeletionLambda::handleRequest") + .core().add(); public static final LambdaHandler TRANSACTION_FOLLOWER = builder() - .jar(LambdaJar.STATESTORE) - .handler("sleeper.statestore.lambda.transaction.TransactionLogFollowerLambda::handleRequest") - .core().add(); + .jar(LambdaJar.STATESTORE) + .handler("sleeper.statestore.lambda.transaction.TransactionLogFollowerLambda::handleRequest") + .core().add(); private final LambdaJar jar; private final String handler; From 1fc6218ed46e4b0933af709d6d11c9af64b3ae33 Mon Sep 17 00:00:00 2001 From: 205762370+Rob9786 <205762370+Rob9786@users.noreply.github.com> Date: Wed, 20 Aug 2025 10:00:28 +0000 Subject: [PATCH 02/14] 5425: Commit of logging and update to stack --- .../sleeper/cdk/stack/core/LoggingStack.java | 2 ++ .../sleeper/cdk/stack/ingest/IngestStack.java | 17 +++++++++++++++-- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/java/cdk/src/main/java/sleeper/cdk/stack/core/LoggingStack.java b/java/cdk/src/main/java/sleeper/cdk/stack/core/LoggingStack.java index 998c11a7663..4cad47cfc31 100644 --- a/java/cdk/src/main/java/sleeper/cdk/stack/core/LoggingStack.java +++ b/java/cdk/src/main/java/sleeper/cdk/stack/core/LoggingStack.java @@ -91,6 +91,8 @@ public enum LogGroupRef { BULK_IMPORT_AUTODELETE("bulk-import-autodelete"), BULK_IMPORT_AUTODELETE_PROVIDER("bulk-import-autodelete-provider"), INGEST_TASKS("IngestTasks"), + INGEST_TASKS_AUTODELETE("ingest-tasks-autodelete"), + INGEST_TASKS_AUTODELETE_PROVIDER("ingest-tasks-autodelete-provider"), INGEST_CREATE_TASKS("ingest-create-tasks"), INGEST_BATCHER_SUBMIT_FILES("ingest-batcher-submit-files"), INGEST_BATCHER_CREATE_JOBS("ingest-batcher-create-jobs"), diff --git a/java/cdk/src/main/java/sleeper/cdk/stack/ingest/IngestStack.java b/java/cdk/src/main/java/sleeper/cdk/stack/ingest/IngestStack.java index 7966766b872..534120db708 100644 --- a/java/cdk/src/main/java/sleeper/cdk/stack/ingest/IngestStack.java +++ b/java/cdk/src/main/java/sleeper/cdk/stack/ingest/IngestStack.java @@ -20,6 +20,7 @@ import software.amazon.awscdk.CfnOutputProps; import software.amazon.awscdk.Duration; import software.amazon.awscdk.NestedStack; +import software.amazon.awscdk.RemovalPolicy; import software.amazon.awscdk.services.cloudwatch.IMetric; import software.amazon.awscdk.services.ec2.IVpc; import software.amazon.awscdk.services.ec2.Vpc; @@ -49,6 +50,7 @@ import sleeper.cdk.jars.LambdaCode; import sleeper.cdk.stack.core.CoreStacks; import sleeper.cdk.stack.core.LoggingStack.LogGroupRef; +import sleeper.cdk.util.AutoStopEcsClusterTasks; import sleeper.cdk.util.Utils; import sleeper.core.deploy.DockerDeployment; import sleeper.core.deploy.LambdaHandler; @@ -61,6 +63,7 @@ import java.util.Objects; import static sleeper.cdk.util.Utils.createAlarmForDlq; +import static sleeper.cdk.util.Utils.removalPolicy; import static sleeper.cdk.util.Utils.shouldDeployPaused; import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.INGEST_CLOUDWATCH_RULE; import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.INGEST_CLUSTER; @@ -109,6 +112,7 @@ public IngestStack( // - A lambda that periodically checks the number of running ingest tasks // and if there are not enough (i.e. there is a backlog on the queue // then it creates more tasks). + // - A lambda that stops task when a delete cluster event is triggered. IBucket jarsBucket = Bucket.fromBucketName(this, "JarsBucket", jars.bucketName()); LambdaCode lambdaCode = jars.lambdaCode(jarsBucket); @@ -117,7 +121,7 @@ public IngestStack( sqsQueueForIngestJobs(coreStacks, topic, errorMetrics); // ECS cluster for ingest tasks - ecsClusterForIngestTasks(jarsBucket, coreStacks, ingestJobQueue); + ecsClusterForIngestTasks(jarsBucket, coreStacks, ingestJobQueue, lambdaCode); // Lambda to create ingest tasks lambdaToCreateIngestTasks(coreStacks, ingestJobQueue, lambdaCode); @@ -181,7 +185,9 @@ private Queue sqsQueueForIngestJobs(CoreStacks coreStacks, Topic topic, List Date: Wed, 20 Aug 2025 11:56:54 +0000 Subject: [PATCH 03/14] 5425: Add permissions to ECS --- .../sleeper/cdk/util/AutoStopEcsClusterTasks.java | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/java/cdk/src/main/java/sleeper/cdk/util/AutoStopEcsClusterTasks.java b/java/cdk/src/main/java/sleeper/cdk/util/AutoStopEcsClusterTasks.java index 56d42b0773e..3d148c865e8 100644 --- a/java/cdk/src/main/java/sleeper/cdk/util/AutoStopEcsClusterTasks.java +++ b/java/cdk/src/main/java/sleeper/cdk/util/AutoStopEcsClusterTasks.java @@ -19,6 +19,9 @@ import software.amazon.awscdk.Duration; import software.amazon.awscdk.customresources.Provider; import software.amazon.awscdk.services.ecs.ICluster; +import software.amazon.awscdk.services.iam.IRole; +import software.amazon.awscdk.services.iam.ManagedPolicy; +import software.amazon.awscdk.services.iam.PolicyStatement; import software.amazon.awscdk.services.lambda.IFunction; import software.amazon.awscdk.services.logs.ILogGroup; import software.constructs.Construct; @@ -28,7 +31,9 @@ import sleeper.core.properties.instance.InstanceProperties; import sleeper.core.util.EnvironmentUtils; +import java.util.List; import java.util.Map; +import java.util.Objects; public class AutoStopEcsClusterTasks { @@ -52,6 +57,16 @@ public static void autoStopTasksOnEcsCluster( .logGroup(logGroup) .timeout(Duration.minutes(10))); + // Grant this function permission to list tasks and stop tasks + PolicyStatement policyStatement = PolicyStatement.Builder + .create() + .resources(List.of("*")) + .actions(List.of("ecs:ListTasks", "ecs:StopTask", "iam:PassRole")) + .build(); + IRole role = Objects.requireNonNull(lambda.getRole()); + role.addToPrincipalPolicy(policyStatement); + role.addManagedPolicy(ManagedPolicy.fromAwsManagedPolicyName("service-role/AmazonECSTaskExecutionRolePolicy")); + Provider propertiesWriterProvider = Provider.Builder.create(scope, id + "Provider") .onEventHandler(lambda) .logGroup(providerLogGroup) From aa5c8d11992134cebdf68abd0b24c518782fe875 Mon Sep 17 00:00:00 2001 From: 205762370+Rob9786 <205762370+Rob9786@users.noreply.github.com> Date: Wed, 20 Aug 2025 13:16:59 +0000 Subject: [PATCH 04/14] 4525: Added spotbugs suppression from IngestStack --- .../src/main/java/sleeper/cdk/util/AutoStopEcsClusterTasks.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/java/cdk/src/main/java/sleeper/cdk/util/AutoStopEcsClusterTasks.java b/java/cdk/src/main/java/sleeper/cdk/util/AutoStopEcsClusterTasks.java index 3d148c865e8..f914c5db223 100644 --- a/java/cdk/src/main/java/sleeper/cdk/util/AutoStopEcsClusterTasks.java +++ b/java/cdk/src/main/java/sleeper/cdk/util/AutoStopEcsClusterTasks.java @@ -15,6 +15,7 @@ */ package sleeper.cdk.util; +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import software.amazon.awscdk.CustomResource; import software.amazon.awscdk.Duration; import software.amazon.awscdk.customresources.Provider; @@ -35,6 +36,7 @@ import java.util.Map; import java.util.Objects; +@SuppressFBWarnings("NP_NULL_ON_SOME_PATH_FROM_RETURN_VALUE") public class AutoStopEcsClusterTasks { private AutoStopEcsClusterTasks() { From fce79824fee90b754fdad697f8fdb76733433937 Mon Sep 17 00:00:00 2001 From: 205762370+Rob9786 <205762370+Rob9786@users.noreply.github.com> Date: Thu, 21 Aug 2025 07:36:21 +0000 Subject: [PATCH 05/14] 5425: Try with no sleep --- .../java/sleeper/cdk/custom/AutoStopEcsClusterTasksLambda.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/java/cdk-custom-resources/src/main/java/sleeper/cdk/custom/AutoStopEcsClusterTasksLambda.java b/java/cdk-custom-resources/src/main/java/sleeper/cdk/custom/AutoStopEcsClusterTasksLambda.java index 5501c3d7341..9a9bbf08240 100644 --- a/java/cdk-custom-resources/src/main/java/sleeper/cdk/custom/AutoStopEcsClusterTasksLambda.java +++ b/java/cdk-custom-resources/src/main/java/sleeper/cdk/custom/AutoStopEcsClusterTasksLambda.java @@ -78,7 +78,7 @@ private void stopTasks(String clusterName) { LOGGER.info("Stopping task {} in cluster {} ", task, clusterName); // Rate limit for ECS StopTask is 100 burst, 40 sustained: // https://docs.aws.amazon.com/AmazonECS/latest/APIReference/request-throttling.html - sleepForSustainedRatePerSecond(30, sleep); + sleepForSustainedRatePerSecond(30); ecsClient.stopTask(builder -> builder.cluster(clusterName).task(task)); }); } From f2b641f09e9999de4e3785e948ffa38efceed0d2 Mon Sep 17 00:00:00 2001 From: 205762370+Rob9786 <205762370+Rob9786@users.noreply.github.com> Date: Thu, 21 Aug 2025 09:14:18 +0000 Subject: [PATCH 06/14] 5425: Changed code to use ShutdownSystemProcesses version --- .../custom/AutoStopEcsClusterTasksLambda.java | 43 +++++++++++++------ 1 file changed, 31 insertions(+), 12 deletions(-) diff --git a/java/cdk-custom-resources/src/main/java/sleeper/cdk/custom/AutoStopEcsClusterTasksLambda.java b/java/cdk-custom-resources/src/main/java/sleeper/cdk/custom/AutoStopEcsClusterTasksLambda.java index 9a9bbf08240..fa92bb981ef 100644 --- a/java/cdk-custom-resources/src/main/java/sleeper/cdk/custom/AutoStopEcsClusterTasksLambda.java +++ b/java/cdk-custom-resources/src/main/java/sleeper/cdk/custom/AutoStopEcsClusterTasksLambda.java @@ -24,6 +24,7 @@ import sleeper.core.util.ThreadSleep; import java.util.Map; +import java.util.function.Consumer; import static sleeper.core.util.RateLimitUtils.sleepForSustainedRatePerSecond; @@ -62,24 +63,42 @@ public void handleEvent( case "Update": break; case "Delete": - stopTasks(clusterName); + stopTasks(ecsClient, clusterName); break; default: throw new IllegalArgumentException("Invalid request type: " + event.getRequestType()); } } - private void stopTasks(String clusterName) { - ecsClient.listTasksPaginator(builder -> builder.cluster(clusterName).maxResults(maxResults)) - .taskArns() + private static void stopTasks(EcsClient ecs, String clusterName) { + LOGGER.info("Stopping tasks for ECS cluster {}", clusterName); + forEachTaskArn(ecs, clusterName, taskArn -> { + // Rate limit for ECS StopTask is 100 burst, 40 sustained: + // https://docs.aws.amazon.com/AmazonECS/latest/APIReference/request-throttling.html + sleepForSustainedRatePerSecond(30); + ecs.stopTask(builder -> builder.cluster(clusterName).task(taskArn) + .reason("Cleaning up before cdk destroy")); + }); + + // ecsClient.listTasksPaginator(builder -> builder.cluster(clusterName).maxResults(maxResults)) + // .taskArns() + // .stream() + // .forEach( + // task -> { + // LOGGER.info("Stopping task {} in cluster {} ", task, clusterName); + // // Rate limit for ECS StopTask is 100 burst, 40 sustained: + // // https://docs.aws.amazon.com/AmazonECS/latest/APIReference/request-throttling.html + // sleepForSustainedRatePerSecond(30); + // ecsClient.stopTask(builder -> builder.cluster(clusterName).task(task)); + // }); + } + + private static void forEachTaskArn(EcsClient ecs, String clusterName, Consumer consumer) { + ecs.listTasksPaginator(builder -> builder.cluster(clusterName)) .stream() - .forEach( - task -> { - LOGGER.info("Stopping task {} in cluster {} ", task, clusterName); - // Rate limit for ECS StopTask is 100 burst, 40 sustained: - // https://docs.aws.amazon.com/AmazonECS/latest/APIReference/request-throttling.html - sleepForSustainedRatePerSecond(30); - ecsClient.stopTask(builder -> builder.cluster(clusterName).task(task)); - }); + .peek(response -> LOGGER.info("Found {} tasks", response.taskArns().size())) + .flatMap(response -> response.taskArns().stream()) + .forEach(consumer); } + } From 9081aedd7c5e32606ab9c3328e328c446d10a5f2 Mon Sep 17 00:00:00 2001 From: 205762370+Rob9786 <205762370+Rob9786@users.noreply.github.com> Date: Thu, 21 Aug 2025 12:34:54 +0000 Subject: [PATCH 07/14] 4525: Re-instate sleep and maxResults --- .../custom/AutoStopEcsClusterTasksLambda.java | 24 +++++-------------- 1 file changed, 6 insertions(+), 18 deletions(-) diff --git a/java/cdk-custom-resources/src/main/java/sleeper/cdk/custom/AutoStopEcsClusterTasksLambda.java b/java/cdk-custom-resources/src/main/java/sleeper/cdk/custom/AutoStopEcsClusterTasksLambda.java index fa92bb981ef..8572283cb64 100644 --- a/java/cdk-custom-resources/src/main/java/sleeper/cdk/custom/AutoStopEcsClusterTasksLambda.java +++ b/java/cdk-custom-resources/src/main/java/sleeper/cdk/custom/AutoStopEcsClusterTasksLambda.java @@ -63,38 +63,26 @@ public void handleEvent( case "Update": break; case "Delete": - stopTasks(ecsClient, clusterName); + stopTasks(ecsClient, clusterName, maxResults, sleep); break; default: throw new IllegalArgumentException("Invalid request type: " + event.getRequestType()); } } - private static void stopTasks(EcsClient ecs, String clusterName) { + private static void stopTasks(EcsClient ecs, String clusterName, int maxResults, ThreadSleep sleep) { LOGGER.info("Stopping tasks for ECS cluster {}", clusterName); - forEachTaskArn(ecs, clusterName, taskArn -> { + forEachTaskArn(ecs, clusterName, maxResults, taskArn -> { // Rate limit for ECS StopTask is 100 burst, 40 sustained: // https://docs.aws.amazon.com/AmazonECS/latest/APIReference/request-throttling.html - sleepForSustainedRatePerSecond(30); + sleepForSustainedRatePerSecond(30, sleep); ecs.stopTask(builder -> builder.cluster(clusterName).task(taskArn) .reason("Cleaning up before cdk destroy")); }); - - // ecsClient.listTasksPaginator(builder -> builder.cluster(clusterName).maxResults(maxResults)) - // .taskArns() - // .stream() - // .forEach( - // task -> { - // LOGGER.info("Stopping task {} in cluster {} ", task, clusterName); - // // Rate limit for ECS StopTask is 100 burst, 40 sustained: - // // https://docs.aws.amazon.com/AmazonECS/latest/APIReference/request-throttling.html - // sleepForSustainedRatePerSecond(30); - // ecsClient.stopTask(builder -> builder.cluster(clusterName).task(task)); - // }); } - private static void forEachTaskArn(EcsClient ecs, String clusterName, Consumer consumer) { - ecs.listTasksPaginator(builder -> builder.cluster(clusterName)) + private static void forEachTaskArn(EcsClient ecs, String clusterName, int maxResults, Consumer consumer) { + ecs.listTasksPaginator(builder -> builder.cluster(clusterName).maxResults(maxResults)) .stream() .peek(response -> LOGGER.info("Found {} tasks", response.taskArns().size())) .flatMap(response -> response.taskArns().stream()) From ce968d1756e2e2bfa0e7637419f493993f99cc3b Mon Sep 17 00:00:00 2001 From: 205762370+Rob9786 <205762370+Rob9786@users.noreply.github.com> Date: Thu, 21 Aug 2025 13:37:03 +0000 Subject: [PATCH 08/14] 4525: Tidied up logrefs and added extra logging --- .../sleeper/cdk/custom/AutoStopEcsClusterTasksLambda.java | 1 + .../src/main/java/sleeper/cdk/stack/core/LoggingStack.java | 4 ++-- .../src/main/java/sleeper/cdk/stack/ingest/IngestStack.java | 4 ++-- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/java/cdk-custom-resources/src/main/java/sleeper/cdk/custom/AutoStopEcsClusterTasksLambda.java b/java/cdk-custom-resources/src/main/java/sleeper/cdk/custom/AutoStopEcsClusterTasksLambda.java index 8572283cb64..5cdbc6e75c5 100644 --- a/java/cdk-custom-resources/src/main/java/sleeper/cdk/custom/AutoStopEcsClusterTasksLambda.java +++ b/java/cdk-custom-resources/src/main/java/sleeper/cdk/custom/AutoStopEcsClusterTasksLambda.java @@ -78,6 +78,7 @@ private static void stopTasks(EcsClient ecs, String clusterName, int maxResults, sleepForSustainedRatePerSecond(30, sleep); ecs.stopTask(builder -> builder.cluster(clusterName).task(taskArn) .reason("Cleaning up before cdk destroy")); + LOGGER.info("Stopped task {} in ECS cluster {}", taskArn, clusterName); }); } diff --git a/java/cdk/src/main/java/sleeper/cdk/stack/core/LoggingStack.java b/java/cdk/src/main/java/sleeper/cdk/stack/core/LoggingStack.java index 4cad47cfc31..2867af50986 100644 --- a/java/cdk/src/main/java/sleeper/cdk/stack/core/LoggingStack.java +++ b/java/cdk/src/main/java/sleeper/cdk/stack/core/LoggingStack.java @@ -91,8 +91,8 @@ public enum LogGroupRef { BULK_IMPORT_AUTODELETE("bulk-import-autodelete"), BULK_IMPORT_AUTODELETE_PROVIDER("bulk-import-autodelete-provider"), INGEST_TASKS("IngestTasks"), - INGEST_TASKS_AUTODELETE("ingest-tasks-autodelete"), - INGEST_TASKS_AUTODELETE_PROVIDER("ingest-tasks-autodelete-provider"), + INGEST_TASKS_AUTOSTOP("ingest-tasks-autostop"), + INGEST_TASKS_AUTOSTOP_PROVIDER("ingest-tasks-autostop-provider"), INGEST_CREATE_TASKS("ingest-create-tasks"), INGEST_BATCHER_SUBMIT_FILES("ingest-batcher-submit-files"), INGEST_BATCHER_CREATE_JOBS("ingest-batcher-create-jobs"), diff --git a/java/cdk/src/main/java/sleeper/cdk/stack/ingest/IngestStack.java b/java/cdk/src/main/java/sleeper/cdk/stack/ingest/IngestStack.java index 534120db708..91b0ff6835a 100644 --- a/java/cdk/src/main/java/sleeper/cdk/stack/ingest/IngestStack.java +++ b/java/cdk/src/main/java/sleeper/cdk/stack/ingest/IngestStack.java @@ -245,8 +245,8 @@ private Cluster ecsClusterForIngestTasks( if (removalPolicy == RemovalPolicy.DESTROY) { AutoStopEcsClusterTasks.autoStopTasksOnEcsCluster(this, instanceProperties, lambdaCode, cluster, clusterName, - coreStacks.getLogGroup(LogGroupRef.INGEST_TASKS_AUTODELETE), - coreStacks.getLogGroup(LogGroupRef.INGEST_TASKS_AUTODELETE_PROVIDER)); + coreStacks.getLogGroup(LogGroupRef.INGEST_TASKS_AUTOSTOP), + coreStacks.getLogGroup(LogGroupRef.INGEST_TASKS_AUTOSTOP_PROVIDER)); } return cluster; From 12f552b3f464d3460506e6f95a8a9173feac3037 Mon Sep 17 00:00:00 2001 From: 205762370+Rob9786 <205762370+Rob9786@users.noreply.github.com> Date: Fri, 22 Aug 2025 08:18:55 +0000 Subject: [PATCH 09/14] 5425: Removed any form of sleeping or throttling. --- .../sleeper/cdk/custom/AutoStopEcsClusterTasksLambda.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/java/cdk-custom-resources/src/main/java/sleeper/cdk/custom/AutoStopEcsClusterTasksLambda.java b/java/cdk-custom-resources/src/main/java/sleeper/cdk/custom/AutoStopEcsClusterTasksLambda.java index 5cdbc6e75c5..755818dd718 100644 --- a/java/cdk-custom-resources/src/main/java/sleeper/cdk/custom/AutoStopEcsClusterTasksLambda.java +++ b/java/cdk-custom-resources/src/main/java/sleeper/cdk/custom/AutoStopEcsClusterTasksLambda.java @@ -26,8 +26,6 @@ import java.util.Map; import java.util.function.Consumer; -import static sleeper.core.util.RateLimitUtils.sleepForSustainedRatePerSecond; - /** Deletes an ECS cluster. */ public class AutoStopEcsClusterTasksLambda { public static final Logger LOGGER = LoggerFactory.getLogger(AutoStopEcsClusterTasksLambda.class); @@ -75,7 +73,7 @@ private static void stopTasks(EcsClient ecs, String clusterName, int maxResults, forEachTaskArn(ecs, clusterName, maxResults, taskArn -> { // Rate limit for ECS StopTask is 100 burst, 40 sustained: // https://docs.aws.amazon.com/AmazonECS/latest/APIReference/request-throttling.html - sleepForSustainedRatePerSecond(30, sleep); + //sleepForSustainedRatePerSecond(30, sleep); ecs.stopTask(builder -> builder.cluster(clusterName).task(taskArn) .reason("Cleaning up before cdk destroy")); LOGGER.info("Stopped task {} in ECS cluster {}", taskArn, clusterName); From 74994b8dac7f5e3dc45c1319b480f21b64064597 Mon Sep 17 00:00:00 2001 From: 205762370+Rob9786 <205762370+Rob9786@users.noreply.github.com> Date: Fri, 22 Aug 2025 09:31:44 +0000 Subject: [PATCH 10/14] 5425: Re-instated sleep --- .../sleeper/cdk/custom/AutoStopEcsClusterTasksLambda.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/java/cdk-custom-resources/src/main/java/sleeper/cdk/custom/AutoStopEcsClusterTasksLambda.java b/java/cdk-custom-resources/src/main/java/sleeper/cdk/custom/AutoStopEcsClusterTasksLambda.java index 755818dd718..5cdbc6e75c5 100644 --- a/java/cdk-custom-resources/src/main/java/sleeper/cdk/custom/AutoStopEcsClusterTasksLambda.java +++ b/java/cdk-custom-resources/src/main/java/sleeper/cdk/custom/AutoStopEcsClusterTasksLambda.java @@ -26,6 +26,8 @@ import java.util.Map; import java.util.function.Consumer; +import static sleeper.core.util.RateLimitUtils.sleepForSustainedRatePerSecond; + /** Deletes an ECS cluster. */ public class AutoStopEcsClusterTasksLambda { public static final Logger LOGGER = LoggerFactory.getLogger(AutoStopEcsClusterTasksLambda.class); @@ -73,7 +75,7 @@ private static void stopTasks(EcsClient ecs, String clusterName, int maxResults, forEachTaskArn(ecs, clusterName, maxResults, taskArn -> { // Rate limit for ECS StopTask is 100 burst, 40 sustained: // https://docs.aws.amazon.com/AmazonECS/latest/APIReference/request-throttling.html - //sleepForSustainedRatePerSecond(30, sleep); + sleepForSustainedRatePerSecond(30, sleep); ecs.stopTask(builder -> builder.cluster(clusterName).task(taskArn) .reason("Cleaning up before cdk destroy")); LOGGER.info("Stopped task {} in ECS cluster {}", taskArn, clusterName); From b9f674387fd8b8515c85edca8e2e042f2646ce2f Mon Sep 17 00:00:00 2001 From: 205762370+Rob9786 <205762370+Rob9786@users.noreply.github.com> Date: Thu, 28 Aug 2025 12:25:36 +0000 Subject: [PATCH 11/14] 5425: Add dependency to custom resource --- .../main/java/sleeper/cdk/util/AutoStopEcsClusterTasks.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/java/cdk/src/main/java/sleeper/cdk/util/AutoStopEcsClusterTasks.java b/java/cdk/src/main/java/sleeper/cdk/util/AutoStopEcsClusterTasks.java index f914c5db223..5492c32b7c7 100644 --- a/java/cdk/src/main/java/sleeper/cdk/util/AutoStopEcsClusterTasks.java +++ b/java/cdk/src/main/java/sleeper/cdk/util/AutoStopEcsClusterTasks.java @@ -74,11 +74,14 @@ public static void autoStopTasksOnEcsCluster( .logGroup(providerLogGroup) .build(); - CustomResource.Builder.create(scope, id) + CustomResource customResource = CustomResource.Builder.create(scope, id) .resourceType("Custom::AutoStopEcsClusterTasks") .properties(Map.of("cluster", clusterName)) .serviceToken(propertiesWriterProvider.getServiceToken()) .build(); + + customResource.getNode().addDependency(cluster); + } } From cd711d05e77f4076ae44fd3e07f3c4c8d466842d Mon Sep 17 00:00:00 2001 From: 205762370+Rob9786 <205762370+Rob9786@users.noreply.github.com> Date: Fri, 29 Aug 2025 07:30:04 +0000 Subject: [PATCH 12/14] 5425: Test without removal policy --- .../sleeper/cdk/stack/ingest/IngestStack.java | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/java/cdk/src/main/java/sleeper/cdk/stack/ingest/IngestStack.java b/java/cdk/src/main/java/sleeper/cdk/stack/ingest/IngestStack.java index 91b0ff6835a..dce2a98c227 100644 --- a/java/cdk/src/main/java/sleeper/cdk/stack/ingest/IngestStack.java +++ b/java/cdk/src/main/java/sleeper/cdk/stack/ingest/IngestStack.java @@ -20,7 +20,6 @@ import software.amazon.awscdk.CfnOutputProps; import software.amazon.awscdk.Duration; import software.amazon.awscdk.NestedStack; -import software.amazon.awscdk.RemovalPolicy; import software.amazon.awscdk.services.cloudwatch.IMetric; import software.amazon.awscdk.services.ec2.IVpc; import software.amazon.awscdk.services.ec2.Vpc; @@ -63,7 +62,6 @@ import java.util.Objects; import static sleeper.cdk.util.Utils.createAlarmForDlq; -import static sleeper.cdk.util.Utils.removalPolicy; import static sleeper.cdk.util.Utils.shouldDeployPaused; import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.INGEST_CLOUDWATCH_RULE; import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.INGEST_CLUSTER; @@ -187,7 +185,7 @@ private Cluster ecsClusterForIngestTasks( CoreStacks coreStacks, Queue ingestJobQueue, LambdaCode lambdaCode) { - RemovalPolicy removalPolicy = removalPolicy(instanceProperties); + //RemovalPolicy removalPolicy = removalPolicy(instanceProperties); VpcLookupOptions vpcLookupOptions = VpcLookupOptions.builder() .vpcId(instanceProperties.get(VPC_ID)) .build(); @@ -242,12 +240,12 @@ private Cluster ecsClusterForIngestTasks( .build(); new CfnOutput(this, INGEST_CONTAINER_ROLE_ARN, ingestRoleARNProps); - if (removalPolicy == RemovalPolicy.DESTROY) { - AutoStopEcsClusterTasks.autoStopTasksOnEcsCluster(this, instanceProperties, lambdaCode, - cluster, clusterName, - coreStacks.getLogGroup(LogGroupRef.INGEST_TASKS_AUTOSTOP), - coreStacks.getLogGroup(LogGroupRef.INGEST_TASKS_AUTOSTOP_PROVIDER)); - } + //if (removalPolicy == RemovalPolicy.DESTROY) { + AutoStopEcsClusterTasks.autoStopTasksOnEcsCluster(this, instanceProperties, lambdaCode, + cluster, clusterName, + coreStacks.getLogGroup(LogGroupRef.INGEST_TASKS_AUTOSTOP), + coreStacks.getLogGroup(LogGroupRef.INGEST_TASKS_AUTOSTOP_PROVIDER)); + //} return cluster; } From bd31bb28a6aa0dcee4d826123d430bb37b09bd2f Mon Sep 17 00:00:00 2001 From: 205762370+Rob9786 <205762370+Rob9786@users.noreply.github.com> Date: Fri, 29 Aug 2025 08:16:18 +0000 Subject: [PATCH 13/14] 5425: Removed commented out code --- .../src/main/java/sleeper/cdk/stack/ingest/IngestStack.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/java/cdk/src/main/java/sleeper/cdk/stack/ingest/IngestStack.java b/java/cdk/src/main/java/sleeper/cdk/stack/ingest/IngestStack.java index dce2a98c227..070c3a7e199 100644 --- a/java/cdk/src/main/java/sleeper/cdk/stack/ingest/IngestStack.java +++ b/java/cdk/src/main/java/sleeper/cdk/stack/ingest/IngestStack.java @@ -185,7 +185,7 @@ private Cluster ecsClusterForIngestTasks( CoreStacks coreStacks, Queue ingestJobQueue, LambdaCode lambdaCode) { - //RemovalPolicy removalPolicy = removalPolicy(instanceProperties); + VpcLookupOptions vpcLookupOptions = VpcLookupOptions.builder() .vpcId(instanceProperties.get(VPC_ID)) .build(); @@ -240,12 +240,10 @@ private Cluster ecsClusterForIngestTasks( .build(); new CfnOutput(this, INGEST_CONTAINER_ROLE_ARN, ingestRoleARNProps); - //if (removalPolicy == RemovalPolicy.DESTROY) { AutoStopEcsClusterTasks.autoStopTasksOnEcsCluster(this, instanceProperties, lambdaCode, cluster, clusterName, coreStacks.getLogGroup(LogGroupRef.INGEST_TASKS_AUTOSTOP), coreStacks.getLogGroup(LogGroupRef.INGEST_TASKS_AUTOSTOP_PROVIDER)); - //} return cluster; } From 14e19ff9d04f41cda3491bb939b6b90e06e54237 Mon Sep 17 00:00:00 2001 From: 205762370+Rob9786 <205762370+Rob9786@users.noreply.github.com> Date: Mon, 1 Sep 2025 08:14:57 +0000 Subject: [PATCH 14/14] 5425: Re-saved LambdaHandler --- .../sleeper/core/deploy/LambdaHandler.java | 222 +++++++++--------- 1 file changed, 111 insertions(+), 111 deletions(-) diff --git a/java/core/src/main/java/sleeper/core/deploy/LambdaHandler.java b/java/core/src/main/java/sleeper/core/deploy/LambdaHandler.java index 5f0a65403e7..6c28fc283d3 100644 --- a/java/core/src/main/java/sleeper/core/deploy/LambdaHandler.java +++ b/java/core/src/main/java/sleeper/core/deploy/LambdaHandler.java @@ -35,153 +35,153 @@ public class LambdaHandler { private static final List ALL = new ArrayList<>(); private static final Map ATHENA_HANDLER_BY_CLASSNAME = new HashMap<>(); public static final LambdaHandler ATHENA_SIMPLE_COMPOSITE = builder() - .jar(LambdaJar.ATHENA) - .handler("sleeper.athena.composite.SimpleCompositeHandler") - .optionalStack(OptionalStack.AthenaStack).add(); + .jar(LambdaJar.ATHENA) + .handler("sleeper.athena.composite.SimpleCompositeHandler") + .optionalStack(OptionalStack.AthenaStack).add(); public static final LambdaHandler ATHENA_ITERATORS_COMPOSITE = builder() - .jar(LambdaJar.ATHENA) - .handler("sleeper.athena.composite.IteratorApplyingCompositeHandler") - .optionalStack(OptionalStack.AthenaStack).add(); + .jar(LambdaJar.ATHENA) + .handler("sleeper.athena.composite.IteratorApplyingCompositeHandler") + .optionalStack(OptionalStack.AthenaStack).add(); public static final LambdaHandler BULK_IMPORT_STARTER = builder() - .jar(LambdaJar.BULK_IMPORT_STARTER) - .handler("sleeper.bulkimport.starter.BulkImportStarterLambda") - .optionalStacks(OptionalStack.BULK_IMPORT_STACKS).add(); + .jar(LambdaJar.BULK_IMPORT_STARTER) + .handler("sleeper.bulkimport.starter.BulkImportStarterLambda") + .optionalStacks(OptionalStack.BULK_IMPORT_STACKS).add(); public static final LambdaHandler INGEST_TASK_CREATOR = builder() - .jar(LambdaJar.INGEST_TASK_CREATOR) - .handler("sleeper.ingest.taskrunner.RunIngestTasksLambda::eventHandler") - .optionalStack(OptionalStack.IngestStack).add(); + .jar(LambdaJar.INGEST_TASK_CREATOR) + .handler("sleeper.ingest.taskrunner.RunIngestTasksLambda::eventHandler") + .optionalStack(OptionalStack.IngestStack).add(); public static final LambdaHandler INGEST_BATCHER_SUBMITTER = builder() - .jar(LambdaJar.INGEST_BATCHER_SUBMITTER) - .handler("sleeper.ingest.batcher.submitter.IngestBatcherSubmitterLambda::handleRequest") - .optionalStack(OptionalStack.IngestBatcherStack).add(); + .jar(LambdaJar.INGEST_BATCHER_SUBMITTER) + .handler("sleeper.ingest.batcher.submitter.IngestBatcherSubmitterLambda::handleRequest") + .optionalStack(OptionalStack.IngestBatcherStack).add(); public static final LambdaHandler INGEST_BATCHER_JOB_CREATOR = builder() - .jar(LambdaJar.INGEST_BATCHER_JOB_CREATOR) - .handler("sleeper.ingest.batcher.job.creator.IngestBatcherJobCreatorLambda::eventHandler") - .optionalStack(OptionalStack.IngestBatcherStack).add(); + .jar(LambdaJar.INGEST_BATCHER_JOB_CREATOR) + .handler("sleeper.ingest.batcher.job.creator.IngestBatcherJobCreatorLambda::eventHandler") + .optionalStack(OptionalStack.IngestBatcherStack).add(); public static final LambdaHandler GARBAGE_COLLECTOR_TRIGGER = builder() - .jar(LambdaJar.GARBAGE_COLLECTOR) - .handler("sleeper.garbagecollector.GarbageCollectorTriggerLambda::handleRequest") - .optionalStack(OptionalStack.GarbageCollectorStack).add(); + .jar(LambdaJar.GARBAGE_COLLECTOR) + .handler("sleeper.garbagecollector.GarbageCollectorTriggerLambda::handleRequest") + .optionalStack(OptionalStack.GarbageCollectorStack).add(); public static final LambdaHandler GARBAGE_COLLECTOR = builder() - .jar(LambdaJar.GARBAGE_COLLECTOR) - .handler("sleeper.garbagecollector.GarbageCollectorLambda::handleRequest") - .optionalStack(OptionalStack.GarbageCollectorStack).add(); + .jar(LambdaJar.GARBAGE_COLLECTOR) + .handler("sleeper.garbagecollector.GarbageCollectorLambda::handleRequest") + .optionalStack(OptionalStack.GarbageCollectorStack).add(); public static final LambdaHandler COMPACTION_JOB_CREATOR_TRIGGER = builder() - .jar(LambdaJar.COMPACTION_JOB_CREATOR) - .handler("sleeper.compaction.job.creation.lambda.CreateCompactionJobsTriggerLambda::handleRequest") - .optionalStack(OptionalStack.CompactionStack).add(); + .jar(LambdaJar.COMPACTION_JOB_CREATOR) + .handler("sleeper.compaction.job.creation.lambda.CreateCompactionJobsTriggerLambda::handleRequest") + .optionalStack(OptionalStack.CompactionStack).add(); public static final LambdaHandler COMPACTION_JOB_CREATOR = builder() - .jar(LambdaJar.COMPACTION_JOB_CREATOR) - .handler("sleeper.compaction.job.creation.lambda.CreateCompactionJobsLambda::handleRequest") - .optionalStack(OptionalStack.CompactionStack).add(); + .jar(LambdaJar.COMPACTION_JOB_CREATOR) + .handler("sleeper.compaction.job.creation.lambda.CreateCompactionJobsLambda::handleRequest") + .optionalStack(OptionalStack.CompactionStack).add(); public static final LambdaHandler COMPACTION_JOB_DISPATCHER = builder() - .jar(LambdaJar.COMPACTION_JOB_CREATOR) - .handler("sleeper.compaction.job.creation.lambda.CompactionJobDispatchLambda::handleRequest") - .optionalStack(OptionalStack.CompactionStack).add(); + .jar(LambdaJar.COMPACTION_JOB_CREATOR) + .handler("sleeper.compaction.job.creation.lambda.CompactionJobDispatchLambda::handleRequest") + .optionalStack(OptionalStack.CompactionStack).add(); public static final LambdaHandler COMPACTION_TASK_CREATOR = builder() - .jar(LambdaJar.COMPACTION_TASK_CREATOR) - .handler("sleeper.compaction.task.creation.RunCompactionTasksLambda::eventHandler") - .optionalStack(OptionalStack.CompactionStack).add(); + .jar(LambdaJar.COMPACTION_TASK_CREATOR) + .handler("sleeper.compaction.task.creation.RunCompactionTasksLambda::eventHandler") + .optionalStack(OptionalStack.CompactionStack).add(); public static final LambdaHandler COMPACTION_TASK_TERMINATOR = builder() - .jar(LambdaJar.COMPACTION_TASK_CREATOR) - .handler("sleeper.compaction.task.creation.SafeTerminationLambda::handleRequest") - .optionalStack(OptionalStack.CompactionStack).add(); + .jar(LambdaJar.COMPACTION_TASK_CREATOR) + .handler("sleeper.compaction.task.creation.SafeTerminationLambda::handleRequest") + .optionalStack(OptionalStack.CompactionStack).add(); public static final LambdaHandler FIND_PARTITIONS_TO_SPLIT_TRIGGER = builder() - .jar(LambdaJar.PARTITION_SPLITTER) - .handler("sleeper.splitter.lambda.FindPartitionsToSplitTriggerLambda::handleRequest") - .optionalStack(OptionalStack.PartitionSplittingStack).add(); + .jar(LambdaJar.PARTITION_SPLITTER) + .handler("sleeper.splitter.lambda.FindPartitionsToSplitTriggerLambda::handleRequest") + .optionalStack(OptionalStack.PartitionSplittingStack).add(); public static final LambdaHandler FIND_PARTITIONS_TO_SPLIT = builder() - .jar(LambdaJar.PARTITION_SPLITTER) - .handler("sleeper.splitter.lambda.FindPartitionsToSplitLambda::handleRequest") - .optionalStack(OptionalStack.PartitionSplittingStack).add(); + .jar(LambdaJar.PARTITION_SPLITTER) + .handler("sleeper.splitter.lambda.FindPartitionsToSplitLambda::handleRequest") + .optionalStack(OptionalStack.PartitionSplittingStack).add(); public static final LambdaHandler SPLIT_PARTITION = builder() - .jar(LambdaJar.PARTITION_SPLITTER) - .handler("sleeper.splitter.lambda.SplitPartitionLambda::handleRequest") - .optionalStack(OptionalStack.PartitionSplittingStack).add(); + .jar(LambdaJar.PARTITION_SPLITTER) + .handler("sleeper.splitter.lambda.SplitPartitionLambda::handleRequest") + .optionalStack(OptionalStack.PartitionSplittingStack).add(); public static final LambdaHandler KEEP_QUERY_WARM = builder() - .jar(LambdaJar.QUERY) - .handler("sleeper.query.lambda.WarmQueryExecutorLambda::handleRequest") - .optionalStack(OptionalStack.KeepLambdaWarmStack).add(); + .jar(LambdaJar.QUERY) + .handler("sleeper.query.lambda.WarmQueryExecutorLambda::handleRequest") + .optionalStack(OptionalStack.KeepLambdaWarmStack).add(); public static final LambdaHandler BULK_EXPORT_PLANNER = builder() - .jar(LambdaJar.BULK_EXPORT_PLANNER) - .handler("sleeper.bulkexport.planner.SqsBulkExportProcessorLambda::handleRequest") - .optionalStack(BulkExportStack).add(); + .jar(LambdaJar.BULK_EXPORT_PLANNER) + .handler("sleeper.bulkexport.planner.SqsBulkExportProcessorLambda::handleRequest") + .optionalStack(BulkExportStack).add(); public static final LambdaHandler BULK_EXPORT_TASK_CREATOR = builder() - .jar(LambdaJar.BULK_EXPORT_TASK_CREATOR) - .handler("sleeper.bulkexport.taskcreator.SqsTriggeredBulkExportTaskRunnerLambda::handleRequest") - .optionalStack(BulkExportStack).add(); + .jar(LambdaJar.BULK_EXPORT_TASK_CREATOR) + .handler("sleeper.bulkexport.taskcreator.SqsTriggeredBulkExportTaskRunnerLambda::handleRequest") + .optionalStack(BulkExportStack).add(); public static final LambdaHandler QUERY_EXECUTOR = builder() - .jar(LambdaJar.QUERY) - .handler("sleeper.query.lambda.SqsQueryProcessorLambda::handleRequest") - .optionalStack(OptionalStack.QueryStack).add(); + .jar(LambdaJar.QUERY) + .handler("sleeper.query.lambda.SqsQueryProcessorLambda::handleRequest") + .optionalStack(OptionalStack.QueryStack).add(); public static final LambdaHandler QUERY_LEAF_PARTITION = builder() - .jar(LambdaJar.QUERY) - .handler("sleeper.query.lambda.SqsLeafPartitionQueryLambda::handleRequest") - .optionalStack(OptionalStack.QueryStack).add(); + .jar(LambdaJar.QUERY) + .handler("sleeper.query.lambda.SqsLeafPartitionQueryLambda::handleRequest") + .optionalStack(OptionalStack.QueryStack).add(); public static final LambdaHandler WEB_SOCKET_QUERY = builder() - .jar(LambdaJar.QUERY) - .handler("sleeper.query.lambda.WebSocketQueryProcessorLambda::handleRequest") - .optionalStack(OptionalStack.WebSocketQueryStack).add(); + .jar(LambdaJar.QUERY) + .handler("sleeper.query.lambda.WebSocketQueryProcessorLambda::handleRequest") + .optionalStack(OptionalStack.WebSocketQueryStack).add(); public static final LambdaHandler AUTO_DELETE_S3_OBJECTS = builder() - .jar(LambdaJar.CUSTOM_RESOURCES) - .handler("sleeper.cdk.custom.AutoDeleteS3ObjectsLambda::handleEvent") - .core().add(); + .jar(LambdaJar.CUSTOM_RESOURCES) + .handler("sleeper.cdk.custom.AutoDeleteS3ObjectsLambda::handleEvent") + .core().add(); public static final LambdaHandler AUTO_STOP_ECS_CLUSTER_TASKS = builder() - .jar(LambdaJar.CUSTOM_RESOURCES) - .handler("sleeper.cdk.custom.AutoStopEcsClusterTasksLambda::handleEvent") - .core().add(); + .jar(LambdaJar.CUSTOM_RESOURCES) + .handler("sleeper.cdk.custom.AutoStopEcsClusterTasksLambda::handleEvent") + .core().add(); public static final LambdaHandler PROPERTIES_WRITER = builder() - .jar(LambdaJar.CUSTOM_RESOURCES) - .handler("sleeper.cdk.custom.PropertiesWriterLambda::handleEvent") - .core().add(); + .jar(LambdaJar.CUSTOM_RESOURCES) + .handler("sleeper.cdk.custom.PropertiesWriterLambda::handleEvent") + .core().add(); public static final LambdaHandler VPC_CHECK = builder() - .jar(LambdaJar.CUSTOM_RESOURCES) - .handler("sleeper.cdk.custom.VpcCheckLambda::handleEvent") - .core().add(); + .jar(LambdaJar.CUSTOM_RESOURCES) + .handler("sleeper.cdk.custom.VpcCheckLambda::handleEvent") + .core().add(); public static final LambdaHandler METRICS_TRIGGER = builder() - .jar(LambdaJar.METRICS) - .handler("sleeper.metrics.TableMetricsTriggerLambda::handleRequest") - .optionalStack(OptionalStack.TableMetricsStack).add(); + .jar(LambdaJar.METRICS) + .handler("sleeper.metrics.TableMetricsTriggerLambda::handleRequest") + .optionalStack(OptionalStack.TableMetricsStack).add(); public static final LambdaHandler METRICS = builder() - .jar(LambdaJar.METRICS) - .handler("sleeper.metrics.TableMetricsLambda::handleRequest") - .optionalStack(OptionalStack.TableMetricsStack).add(); + .jar(LambdaJar.METRICS) + .handler("sleeper.metrics.TableMetricsLambda::handleRequest") + .optionalStack(OptionalStack.TableMetricsStack).add(); public static final LambdaHandler STATESTORE_COMMITTER = builder() - .jar(LambdaJar.STATESTORE) - .handler("sleeper.statestore.lambda.committer.StateStoreCommitterLambda::handleRequest") - .core().add(); + .jar(LambdaJar.STATESTORE) + .handler("sleeper.statestore.lambda.committer.StateStoreCommitterLambda::handleRequest") + .core().add(); public static final LambdaHandler COMPACTION_COMMIT_BATCHER = builder() - .jar(LambdaJar.STATESTORE) - .handler("sleeper.statestore.lambda.committer.CompactionCommitBatcherLambda::handleRequest") - .core().add(); + .jar(LambdaJar.STATESTORE) + .handler("sleeper.statestore.lambda.committer.CompactionCommitBatcherLambda::handleRequest") + .core().add(); public static final LambdaHandler SNAPSHOT_CREATION_TRIGGER = builder() - .jar(LambdaJar.STATESTORE) - .handler("sleeper.statestore.lambda.snapshot.TransactionLogSnapshotCreationTriggerLambda::handleRequest") - .core().add(); + .jar(LambdaJar.STATESTORE) + .handler("sleeper.statestore.lambda.snapshot.TransactionLogSnapshotCreationTriggerLambda::handleRequest") + .core().add(); public static final LambdaHandler SNAPSHOT_CREATION = builder() - .jar(LambdaJar.STATESTORE) - .handler("sleeper.statestore.lambda.snapshot.TransactionLogSnapshotCreationLambda::handleRequest") - .core().add(); + .jar(LambdaJar.STATESTORE) + .handler("sleeper.statestore.lambda.snapshot.TransactionLogSnapshotCreationLambda::handleRequest") + .core().add(); public static final LambdaHandler SNAPSHOT_DELETION_TRIGGER = builder() - .jar(LambdaJar.STATESTORE) - .handler("sleeper.statestore.lambda.snapshot.TransactionLogSnapshotDeletionTriggerLambda::handleRequest") - .core().add(); + .jar(LambdaJar.STATESTORE) + .handler("sleeper.statestore.lambda.snapshot.TransactionLogSnapshotDeletionTriggerLambda::handleRequest") + .core().add(); public static final LambdaHandler SNAPSHOT_DELETION = builder() - .jar(LambdaJar.STATESTORE) - .handler("sleeper.statestore.lambda.snapshot.TransactionLogSnapshotDeletionLambda::handleRequest") - .core().add(); + .jar(LambdaJar.STATESTORE) + .handler("sleeper.statestore.lambda.snapshot.TransactionLogSnapshotDeletionLambda::handleRequest") + .core().add(); public static final LambdaHandler TRANSACTION_DELETION_TRIGGER = builder() - .jar(LambdaJar.STATESTORE) - .handler("sleeper.statestore.lambda.transaction.TransactionLogTransactionDeletionTriggerLambda::handleRequest") - .core().add(); + .jar(LambdaJar.STATESTORE) + .handler("sleeper.statestore.lambda.transaction.TransactionLogTransactionDeletionTriggerLambda::handleRequest") + .core().add(); public static final LambdaHandler TRANSACTION_DELETION = builder() - .jar(LambdaJar.STATESTORE) - .handler("sleeper.statestore.lambda.transaction.TransactionLogTransactionDeletionLambda::handleRequest") - .core().add(); + .jar(LambdaJar.STATESTORE) + .handler("sleeper.statestore.lambda.transaction.TransactionLogTransactionDeletionLambda::handleRequest") + .core().add(); public static final LambdaHandler TRANSACTION_FOLLOWER = builder() - .jar(LambdaJar.STATESTORE) - .handler("sleeper.statestore.lambda.transaction.TransactionLogFollowerLambda::handleRequest") - .core().add(); + .jar(LambdaJar.STATESTORE) + .handler("sleeper.statestore.lambda.transaction.TransactionLogFollowerLambda::handleRequest") + .core().add(); private final LambdaJar jar; private final String handler;