Skip to content

Commit 76c2bb1

Browse files
rlhagermrlhagerm
committed
Updating tests and metadata
1 parent 32cdede commit 76c2bb1

File tree

7 files changed

+335
-146
lines changed

7 files changed

+335
-146
lines changed

‎.doc_gen/metadata/cloudwatch-logs_metadata.yaml‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -288,6 +288,14 @@ cloudwatch-logs_PutSubscriptionFilter:
288288
cloudwatch-logs: {PutSubscriptionFilter}
289289
cloudwatch-logs_GetQueryResults:
290290
languages:
291+
.NET:
292+
versions:
293+
- sdk_version: 3
294+
github: dotnetv4/CloudWatchLogs/LargeQuery
295+
excerpts:
296+
- description:
297+
snippet_tags:
298+
- CloudWatchLogs.dotnetv4.GetQueryResults
291299
JavaScript:
292300
versions:
293301
- sdk_version: 3
@@ -306,8 +314,28 @@ cloudwatch-logs_GetQueryResults:
306314
- python.example_code.cloudwatch_logs.get_query_results
307315
services:
308316
cloudwatch-logs: {GetQueryResults}
317+
cloudwatch-logs_PutLogEvents:
318+
languages:
319+
.NET:
320+
versions:
321+
- sdk_version: 3
322+
github: dotnetv4/CloudWatchLogs/LargeQuery
323+
excerpts:
324+
- description:
325+
snippet_tags:
326+
- CloudWatchLogs.dotnetv4.PutLogEvents
327+
services:
328+
cloudwatch-logs: {PutLogEvents}
309329
cloudwatch-logs_StartQuery:
310330
languages:
331+
.NET:
332+
versions:
333+
- sdk_version: 3
334+
github: dotnetv4/CloudWatchLogs/LargeQuery
335+
excerpts:
336+
- description:
337+
snippet_tags:
338+
- CloudWatchLogs.dotnetv4.StartQuery
311339
JavaScript:
312340
versions:
313341
- sdk_version: 3
@@ -332,6 +360,14 @@ cloudwatch-logs_Scenario_BigQuery:
332360
synopsis: use &CWL; to query more than 10,000 records.
333361
category: Scenarios
334362
languages:
363+
.NET:
364+
versions:
365+
- sdk_version: 3
366+
github: dotnetv4/CloudWatchLogs/LargeQuery
367+
excerpts:
368+
- description: This is the main workflow that demonstrates the large query scenario.
369+
snippet_tags:
370+
- CloudWatchLogs.dotnetv4.LargeQueryWorkflow
335371
JavaScript:
336372
versions:
337373
- sdk_version: 3

‎dotnetv3/CloudWatchLogs/README.md‎

Lines changed: 23 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,16 @@ Code excerpts that show you how to call individual service functions.
4141
- [DeleteLogGroup](DeleteLogGroupExample/DeleteLogGroup.cs#L6)
4242
- [DescribeExportTasks](DescribeExportTasksExample/DescribeExportTasks.cs#L6)
4343
- [DescribeLogGroups](DescribeLogGroupsExample/DescribeLogGroups.cs#L6)
44+
- [GetQueryResults](../../dotnetv4/CloudWatchLogs/LargeQuery/Actions/CloudWatchLogsWrapper.cs#L79)
45+
- [PutLogEvents](../../dotnetv4/CloudWatchLogs/LargeQuery/Actions/CloudWatchLogsWrapper.cs#L110)
46+
- [StartQuery](../../dotnetv4/CloudWatchLogs/LargeQuery/Actions/CloudWatchLogsWrapper.cs#L30)
47+
48+
### Scenarios
49+
50+
Code examples that show you how to accomplish a specific task by calling multiple
51+
functions within the same service.
52+
53+
- [Run a large query](../../dotnetv4/CloudWatchLogs/LargeQuery/Scenarios/LargeQueryWorkflow.cs)
4454

4555

4656
<!--custom.examples.start-->
@@ -73,6 +83,18 @@ Alternatively, you can run the example from within your IDE.
7383

7484

7585

86+
#### Run a large query
87+
88+
This example shows you how to use CloudWatch Logs to query more than 10,000 records.
89+
90+
91+
<!--custom.scenario_prereqs.cloudwatch-logs_Scenario_BigQuery.start-->
92+
<!--custom.scenario_prereqs.cloudwatch-logs_Scenario_BigQuery.end-->
93+
94+
95+
<!--custom.scenarios.cloudwatch-logs_Scenario_BigQuery.start-->
96+
<!--custom.scenarios.cloudwatch-logs_Scenario_BigQuery.end-->
97+
7698
### Tests
7799

78100
⚠ Running tests might result in charges to your AWS account.
@@ -99,4 +121,4 @@ in the `dotnetv3` folder.
99121

100122
Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
101123

102-
SPDX-License-Identifier: Apache-2.0
124+
SPDX-License-Identifier: Apache-2.0

‎dotnetv4/CloudWatchLogs/LargeQuery/README.md‎

Lines changed: 87 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,17 @@
22

33
This folder contains a .NET feature scenario that demonstrates how to perform large-scale queries on Amazon CloudWatch Logs using recursive binary search to retrieve more than the 10,000 result limit.
44

5+
## Overview
6+
7+
CloudWatch Logs Insights queries have a maximum result limit of 10,000 records per query. This example demonstrates how to overcome this limitation by using a recursive binary search algorithm that splits the time range into smaller segments when the limit is reached.
8+
9+
The scenario performs the following steps:
10+
11+
1. **Setup**: Deploys a CloudFormation stack with a log group and log stream
12+
2. **Data Generation**: Creates and uploads 50,000 sample log entries
13+
3. **Query Execution**: Performs recursive queries to retrieve all logs using binary search
14+
4. **Cleanup**: Removes all created resources
15+
516
## Project Structure
617

718
```
@@ -14,7 +25,7 @@ LargeQuery/
1425
│ ├── README.md # Detailed scenario documentation
1526
│ └── CloudWatchLogsScenario.csproj # Scenario project file
1627
├── Tests/
17-
│ ├── LargeQueryWorkflowTests.cs # Unit tests
28+
│ ├── LargeQueryWorkflowTests.cs # Integration tests
1829
│ ├── Usings.cs # Global usings
1930
│ └── CloudWatchLogsTests.csproj # Test project file
2031
└── CloudWatchLogsLargeQuery.sln # Solution file
@@ -23,13 +34,16 @@ LargeQuery/
2334
## What This Example Demonstrates
2435

2536
- Deploying AWS resources using CloudFormation
26-
- Generating and ingesting large volumes of log data
27-
- Performing CloudWatch Logs Insights queries
37+
- Generating and ingesting large volumes of log data using PutLogEvents
38+
- Performing CloudWatch Logs Insights queries with StartQuery and GetQueryResults
2839
- Using recursive binary search to retrieve more than 10,000 results
40+
- Handling timestamp precision for accurate query splitting
2941
- Cleaning up resources after completion
3042

3143
## Running the Example
3244

45+
### Interactive Mode
46+
3347
1. Navigate to the solution directory:
3448
```
3549
cd dotnetv4/CloudWatchLogs/LargeQuery
@@ -45,22 +59,87 @@ LargeQuery/
4559
dotnet run --project Scenarios/CloudWatchLogsScenario.csproj
4660
```
4761

48-
4. Run the tests:
49-
```
50-
dotnet test
51-
```
62+
4. Follow the prompts to:
63+
- Deploy the CloudFormation stack
64+
- Generate sample logs
65+
- Execute the recursive query
66+
- View sample results
67+
- Clean up resources
68+
69+
### Non-Interactive Mode (Testing)
70+
71+
Run the integration tests to execute the scenario without user prompts:
72+
73+
```
74+
dotnet test
75+
```
76+
77+
The test verifies that the scenario completes without errors and successfully retrieves all 50,000 log entries.
5278

5379
## Prerequisites
5480

5581
- .NET 8.0 or later
5682
- AWS credentials configured
57-
- Permissions for CloudWatch Logs and CloudFormation
83+
- Permissions for:
84+
- CloudWatch Logs (CreateLogGroup, CreateLogStream, PutLogEvents, StartQuery, GetQueryResults, DeleteLogGroup)
85+
- CloudFormation (CreateStack, DescribeStacks, DeleteStack)
86+
87+
## How It Works
88+
89+
### Recursive Query Algorithm
90+
91+
The key to retrieving more than 10,000 results is the recursive binary search algorithm:
92+
93+
1. Execute a query with the full date range
94+
2. If results < 10,000, return them (we have all logs in this range)
95+
3. If results = 10,000, there may be more logs:
96+
- Get the timestamp of the last result
97+
- Calculate the midpoint between the last timestamp and end date
98+
- Recursively query the first half (last timestamp to midpoint)
99+
- Recursively query the second half (midpoint to end date)
100+
- Combine all results
101+
102+
This approach ensures all logs are retrieved by progressively narrowing the time ranges until each segment contains fewer than 10,000 results.
103+
104+
### Timestamp Precision
105+
106+
The algorithm uses millisecond precision for timestamps to ensure accurate splitting and prevent duplicate or missing log entries. Each query adjusts the start time by 1 millisecond to avoid overlapping results.
107+
108+
## Expected Output
109+
110+
When running the scenario, you'll see output similar to:
111+
112+
```
113+
--------------------------------------------------------------------------------
114+
Welcome to the CloudWatch Logs Large Query Scenario.
115+
--------------------------------------------------------------------------------
116+
Preparing the application...
117+
Deploying CloudFormation stack: CloudWatchLargeQueryStack
118+
CloudFormation stack creation started: CloudWatchLargeQueryStack
119+
Waiting for CloudFormation stack creation to complete...
120+
CloudFormation stack creation complete.
121+
Stack output RoleARN: arn:aws:iam::123456789012:role/...
122+
Generating 50,000 sample log entries...
123+
Batch 1/5: Created 10,000 log entries
124+
Batch 2/5: Created 10,000 log entries
125+
...
126+
Waiting 5 minutes for logs to be fully ingested...
127+
--------------------------------------------------------------------------------
128+
Starting recursive query to retrieve all logs...
129+
Query date range: 2024-01-15T10:00:00.000Z to 2024-01-15T10:05:00.000Z. Found 10000 logs.
130+
Query date range: 2024-01-15T10:02:30.000Z to 2024-01-15T10:03:45.000Z. Found 10000 logs.
131+
...
132+
Queries finished in 8.234 seconds.
133+
Total logs found: 50000
134+
--------------------------------------------------------------------------------
135+
```
58136

59137
## Related Resources
60138

61139
- [CloudWatch Logs Documentation](https://docs.aws.amazon.com/AmazonCloudWatch/latest/logs/)
62140
- [CloudWatch Logs Insights Query Syntax](https://docs.aws.amazon.com/AmazonCloudWatch/latest/logs/CWL_QuerySyntax.html)
63141
- [AWS SDK for .NET](https://aws.amazon.com/sdk-for-net/)
142+
- [CloudWatch Logs API Reference](https://docs.aws.amazon.com/AmazonCloudWatchLogs/latest/APIReference/)
64143

65144
---
66145

‎dotnetv4/CloudWatchLogs/LargeQuery/Scenarios/LargeQueryWorkflow.cs‎

Lines changed: 41 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,7 @@ public class LargeQueryWorkflow
4747
private static long _queryEndDate;
4848

4949
public static bool _interactive = true;
50-
private static string _stackName = "CloudWatchLargeQueryStack";
50+
public static string _stackName = "CloudWatchLargeQueryStack";
5151
private static string _stackResourcePath = "../../../../../../../scenarios/features/cloudwatch_logs_large_query/resources/stack.yaml";
5252

5353
public static async Task Main(string[] args)
@@ -107,6 +107,46 @@ public static async Task Main(string[] args)
107107
Console.WriteLine("CloudWatch Logs Large Query scenario completed.");
108108
}
109109

110+
/// <summary>
111+
/// Runs the scenario workflow. Used for testing.
112+
/// </summary>
113+
public static async Task RunScenario()
114+
{
115+
Console.WriteLine(new string('-', 80));
116+
Console.WriteLine("Welcome to the CloudWatch Logs Large Query Scenario.");
117+
Console.WriteLine(new string('-', 80));
118+
Console.WriteLine("This scenario demonstrates how to perform large-scale queries on");
119+
Console.WriteLine("CloudWatch Logs using recursive binary search to retrieve more than");
120+
Console.WriteLine("the 10,000 result limit.");
121+
Console.WriteLine();
122+
123+
try
124+
{
125+
Console.WriteLine(new string('-', 80));
126+
var prepareSuccess = await PrepareApplication();
127+
Console.WriteLine(new string('-', 80));
128+
129+
if (prepareSuccess)
130+
{
131+
Console.WriteLine(new string('-', 80));
132+
await ExecuteLargeQuery();
133+
Console.WriteLine(new string('-', 80));
134+
}
135+
136+
Console.WriteLine(new string('-', 80));
137+
await Cleanup();
138+
Console.WriteLine(new string('-', 80));
139+
}
140+
catch (Exception ex)
141+
{
142+
_logger.LogError(ex, "There was a problem with the scenario, initiating cleanup...");
143+
_interactive = false;
144+
await Cleanup();
145+
}
146+
147+
Console.WriteLine("CloudWatch Logs Large Query scenario completed.");
148+
}
149+
110150
/// <summary>
111151
/// Prepares the application by creating the necessary resources.
112152
/// </summary>

0 commit comments

Comments
 (0)