feat: Use DataDog to monitor lambda for proxy and limit API Gateway access only to GitHub CIDRs

Author: cfernhout

main.tf

@@ -300,3 +300,21 @@ resource "random_password" "redis_password" {
   length           = 12
   special          = false
 }
+
+module "github_reverse_proxy" {
+  count = var.deploy_github_reverse_proxy ? 1 : 0
+
+  source = "./modules/github_reverse_proxy"
+
+  deployment_name          = var.deployment_name
+  environment              = var.environment
+  region                   = var.provider_region
+  vpc_cidr                 = local.vpc_cidr
+  vpc_id                   = local.vpc_id
+  vpc_private_subnets      = local.vpc_private_subnets
+  github_cidrs             = var.github_cidrs
+  datadog_api_key          = var.datadog_api_key
+  use_private_egress       = var.lb_internal
+
+  private_system_endpoint  = module.load_balancer.load_balancer_dns
+}

modules/github_reverse_proxy/README.md

@@ -2,34 +2,10 @@
 
 To facilitate an internal load balancer, GitHub webhooks still need to have a way to reach the application. This is achieved by creating a reverse proxy for GitHub webhooks. This Terraform module deploys an AWS Lambda function inside a VPC, integrates it with API Gateway, and ensures least privilege access. The Lambda function receives webhooks from GitHub, processes them, and forwards them to a private system inside a VPC.
 
-The module includes:
-- A Lambda function attached to a VPC.
-- Integration with API Gateway to expose the Lambda function as an HTTP endpoint.
-- A deny policy to follow the principle of least privilege, preventing the Lambda function code from making certain Amazon EC2 API calls.
-- Required IAM roles and permissions for VPC access and logging.
-- A Security Group to allow access to the private endpoint in the VPC.
-
 ## Features
 - Deploys an AWS Lambda function inside a VPC.
 - Integrates the Lambda function with API Gateway for receiving webhooks.
+- Limits access to the API Gateway only to request from GitHub CIDR ranges.
 - Configures the necessary IAM roles, including the `AWSLambdaVPCAccessExecutionRole`.
 - Implements a custom deny policy to prevent the Lambda function from making EC2 network-related API calls.
-- CloudWatch logging for monitoring and troubleshooting.
-
-## Usage
-
-```hcl
-module "lambda_vpc_webhook" {
-  source = "./path_to_your_module" # Replace with your module path
-
-  # Variables
-  vpc_id                   = "vpc-12345678"
-  vpc_private_subnets      = ["subnet-abcdefgh", "subnet-ijklmnop"]
-  github_secret            = "your-github-webhook-secret"
-  private_system_endpoint  = "the ip of the internal load balancer"
-}
-
-output "api_gateway_url" {
-  value = module.lambda_vpc_webhook.api_gateway_url
-}
-```
+- DataDog Lambda extension for logging and monitoring.

modules/github_reverse_proxy/iam.tf

@@ -1,5 +1,3 @@
-data "aws_caller_identity" "current" {}
-
 resource "aws_iam_role" "lambda_role" {
   name = "${var.deployment_name}-lambda-github-webhook-role"
 
@@ -15,6 +13,26 @@ resource "aws_iam_role" "lambda_role" {
   })
 }
 
+resource "aws_iam_policy" "lambda_secrets_policy" {
+  name   = "lambda-secrets-policy"
+  policy = jsonencode({
+    Version: "2012-10-17",
+    Statement: [
+      {
+        Effect: "Allow",
+        Action: "secretsmanager:GetSecretValue",
+        Resource: aws_secretsmanager_secret.datadog_api_key.arn
+      }
+    ]
+  })
+}
+
+# Attach the policy to the Lambda execution role
+resource "aws_iam_role_policy_attachment" "lambda_secrets_policy_attachment" {
+  role       = aws_iam_role.lambda_role.name
+  policy_arn = aws_iam_policy.lambda_secrets_policy.arn
+}
+
 resource "aws_iam_role_policy_attachment" "lambda_vpc_access_policy" {
   role       = aws_iam_role.lambda_role.name
   policy_arn = "arn:aws:iam::aws:policy/service-role/AWSLambdaVPCAccessExecutionRole"
@@ -68,7 +86,7 @@ resource "aws_iam_policy" "lambda_deny_ec2_policy" {
         Condition = {
           "ArnEquals": {
             "lambda:SourceFunctionArn": [
-              "arn:aws:lambda:${data.aws_caller_identity.current.account_id}:function:${aws_lambda_function.github_webhook_handler.function_name}"
+              "arn:aws:lambda:${data.aws_caller_identity.current.account_id}:function:${aws_lambda_alias.prod_alias.function_name}"
             ]
           }
         }
@@ -80,4 +98,4 @@ resource "aws_iam_policy" "lambda_deny_ec2_policy" {
 resource "aws_iam_role_policy_attachment" "lambda_deny_ec2_attachment" {
   role       = aws_iam_role.lambda_role.name
   policy_arn = aws_iam_policy.lambda_deny_ec2_policy.arn
-}
+}

modules/github_reverse_proxy/lambda_function.py

@@ -1,71 +1,52 @@
 import json
-import hashlib
-import hmac
+import logging
 import os
 import urllib3
 
-def verify_github_signature(headers, body, secret):
-    """
-    Verifies the GitHub webhook signature using the provided secret.
-    """
-    signature = headers.get('X-Hub-Signature-256')
+from datadog_lambda.logger import initialize_logging
 
-    if not signature:
-        print("No signature found in headers.")
-        return False
 
-    # Create the HMAC digest
-    hmac_gen = hmac.new(key=secret.encode(), msg=body, digestmod=hashlib.sha256)
-    expected_signature = f'sha256={hmac_gen.hexdigest()}'
+initialize_logging(__name__)
+logger = logging.getLogger(__name__)
+# logger.setLevel(logging.INFO)
 
-    return hmac.compare_digest(signature, expected_signature)
 
-def forward_to_private_system(data, private_endpoint):
+def forward_to_private_system(data, private_endpoint, headers):
     """
     Forward the data to the private system endpoint.
     """
-    http = urllib3.PoolManager()
-    headers = {'Content-Type': 'application/json'}
-
-    response = http.request('POST', private_endpoint, body=json.dumps(data), headers=headers)
-
+    http = urllib3.PoolManager(cert_reqs='CERT_NONE')  # Disable SSL certificate verification
+    response = http.request(
+        'POST', private_endpoint, body=data, headers=headers, assert_same_host=False
+    )
     return response.status, response.data
 
 def lambda_handler(event, context):
-    github_secret = os.getenv('GITHUB_SECRET')
     private_system_endpoint = os.getenv('PRIVATE_SYSTEM_ENDPOINT')
+    logger.info(f"Private system endpoint: {private_system_endpoint}")
 
-    headers = event.get('headers', {})
     body = event.get('body', '')
-
-    # Verify GitHub signature
-    if not verify_github_signature(headers, body.encode('utf-8'), github_secret):
-        print("Signature verification failed.")
-        return {
-            'statusCode': 403,
-            'body': json.dumps('Forbidden: Signature verification failed')
-        }
-
-    # Parse the body as JSON
-    try:
-        payload = json.loads(body)
-    except json.JSONDecodeError as e:
-        print(f"JSON parsing error: {e}")
+    if not body:
+        logger.error("No body found in the event")
         return {
             'statusCode': 400,
-            'body': json.dumps('Bad Request: Invalid JSON payload')
+            'body': json.dumps('Bad Request: No payload found')
         }
+    incoming_headers = event.get('headers', {})
 
-    # Forward the payload to the private system
-    status, response = forward_to_private_system(payload, private_system_endpoint)
+    logger.info("Starting to forward the payload")
+    status, response = forward_to_private_system(
+        body, private_system_endpoint, incoming_headers
+    )
+    logger.info(f"Forwarding status: {status}")
 
     if status == 200:
         return {
             'statusCode': 200,
             'body': json.dumps('Webhook processed and forwarded')
         }
     else:
-        print(f"Error forwarding to private system: {response}")
+        logger.error(f"Error forwarding to private system ({status}): {response}")
         return {
             'statusCode': 500,
             'body': json.dumps('Internal Server Error: Failed to forward webhook')

modules/github_reverse_proxy/lambda_function.zip

@@ -1,36 +1,100 @@
-resource "null_resource" "zip_lambda_function" {
-  provisioner "local-exec" {
-    command = "zip -j lambda_function.zip lambda_function.py"
-  }
+data "aws_caller_identity" "current" {}
+data "aws_region" "current" {}
 
-  triggers = {
-    py_source = filemd5("lambda_function.py")
-  }
+# resource "null_resource" "zip_lambda_function" {
+#   provisioner "local-exec" {
+#     command = "zip -j ${path.module}/lambda_function.zip ${path.module}/lambda_function.py"
+#   }
+
+#   triggers = {
+#     py_source = filemd5("${path.module}/lambda_function.py")
+#   }
+# }
+
+data "archive_file" "zip_lambda_function" {
+  type        = "zip"
+  source_file = "${path.module}/lambda_function.py"
+  output_path = "${path.module}/lambda_function.zip"
+}
+
+
+resource "aws_secretsmanager_secret" "datadog_api_key" {
+  name        = "datadog_api_key"
+  description = "Datadog API Key used for monitoring Lambda"
 }
 
-resource "aws_lambda_function" "github_webhook_handler" {
+resource "aws_secretsmanager_secret_version" "datadog_api_key_version" {
+  secret_id     = aws_secretsmanager_secret.datadog_api_key.id
+  secret_string = var.datadog_api_key  # This should be the Datadog API key (input as a variable)
+}
+
+module "lambda_datadog" {
+  source  = "DataDog/lambda-datadog/aws"
+  version = "1.4.0"
+
   function_name = "${var.deployment_name}-github-webhook-handler"
   role          = aws_iam_role.lambda_role.arn
-  handler       = "lambda_function.lambda_handler" # Python handler
+  handler       = "lambda_function.lambda_handler"
   runtime       = "python3.12"
+  memory_size   = 256
+  timeout       = 30
 
-  filename         = "${path.module}/lambda_function.zip"
-  source_code_hash = filebase64sha256("${path.module}/lambda_function.zip")
+  publish = true
 
-  environment {
-    variables = {
-      GITHUB_SECRET           = var.github_secret
-      PRIVATE_SYSTEM_ENDPOINT = var.private_system_endpoint
-    }
-  }
+  filename         = data.archive_file.zip_lambda_function.output_path
+  source_code_hash = data.archive_file.zip_lambda_function.output_base64sha256
 
-  vpc_config {
-    subnet_ids         = var.vpc_private_subnets
-    security_group_ids = length(var.security_group_ids) > 0 ? var.security_group_ids : [aws_security_group.lambda_sg.id]
+  environment_variables = {
+    "DD_API_KEY_SECRET_ARN" : aws_secretsmanager_secret.datadog_api_key.arn
+    "DD_ENV" : var.environment
+    "DD_SERVICE" : "github-webhook-service"
+    "DD_SITE": "datadoghq.com"
+    "DD_VERSION" : "1.0.0"
+    "DD_EXTENSION_VERSION": "next"
+    "DD_SERVERLESS_LOGS_ENABLED": "true"
+    "DD_LOG_LEVEL": "INFO"
+    "DD_TAGS": "deployment:${var.deployment_name}"
+    "PRIVATE_SYSTEM_ENDPOINT" : "https://${var.private_system_endpoint}/integrations/github/v1/app_hook"
   }
 
+  vpc_config_subnet_ids         = var.vpc_private_subnets
+  vpc_config_security_group_ids = length(var.security_group_ids) > 0 ? var.security_group_ids : [aws_security_group.lambda_sg.id]
+
+  datadog_extension_layer_version = 63
+  datadog_python_layer_version = 98
+
   # Depend on the zip operation
-  depends_on = [null_resource.zip_lambda_function]
+  depends_on = [data.archive_file.zip_lambda_function]
+}
+
+resource "aws_lambda_alias" "prod_alias" {
+  name             = "prod"
+  function_name    = module.lambda_datadog.function_name
+  function_version = module.lambda_datadog.version
+}
+
+resource "aws_lambda_provisioned_concurrency_config" "example" {
+  function_name                     = aws_lambda_alias.prod_alias.function_name
+  provisioned_concurrent_executions = 1
+  qualifier                         = aws_lambda_alias.prod_alias.name
+}
+
+resource "aws_lambda_function_event_invoke_config" "concurrency_limit" {
+  function_name          = aws_lambda_alias.prod_alias.function_name
+  qualifier              = aws_lambda_alias.prod_alias.name
+  maximum_retry_attempts = 2
+}
+
+resource "aws_lambda_permission" "lambda_permission" {
+  statement_id  = "AllowExecutionFromAPIGateway"
+  action        = "lambda:InvokeFunction"
+  function_name = aws_lambda_alias.prod_alias.function_name
+  qualifier     = aws_lambda_alias.prod_alias.name
+  principal     = "apigateway.amazonaws.com"
+
+  # The /* part allows invocation from any stage, method and resource path
+  # within API Gateway.
+  source_arn = "${aws_api_gateway_rest_api.webhook_api.execution_arn}/*"
 }
 
 # API Gateway to act as a reverse proxy
@@ -60,7 +124,7 @@ resource "aws_api_gateway_integration" "lambda_integration" {
   http_method             = aws_api_gateway_method.post_webhook.http_method
   integration_http_method = "POST"
   type                    = "AWS_PROXY"
-  uri                     = aws_lambda_function.github_webhook_handler.invoke_arn
+  uri                     = aws_lambda_alias.prod_alias.invoke_arn
 }
 
 # Deployment of API Gateway
@@ -69,4 +133,31 @@ resource "aws_api_gateway_deployment" "api_deployment" {
   stage_name  = "prod"
 
   depends_on = [aws_api_gateway_integration.lambda_integration]
-}
+}
+
+resource "aws_api_gateway_rest_api_policy" "github_ip_restriction" {
+  rest_api_id = aws_api_gateway_rest_api.webhook_api.id
+
+  policy = jsonencode({
+    Version = "2012-10-17",
+    Statement = [
+            {
+        Effect: "Allow",
+        Principal: "*",
+        Action: "execute-api:Invoke",
+        Resource: "arn:aws:execute-api:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:${aws_api_gateway_rest_api.webhook_api.id}/*/POST/*",
+      },
+      {
+        Effect: "Deny",
+        Principal: "*",
+        Action: "execute-api:Invoke",
+        Resource: "arn:aws:execute-api:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:${aws_api_gateway_rest_api.webhook_api.id}/*/POST/*",
+        Condition: {
+          "NotIpAddress": {
+            "aws:SourceIp": var.github_cidrs
+          }
+        }
+      }
+    ]
+  })
+}

modules/github_reverse_proxy/main.tf

@@ -1,4 +1,4 @@
 output "api_gateway_url" {
-  value = aws_api_gateway_deployment.api_deployment.invoke_url
+  value = "${aws_api_gateway_deployment.api_deployment.invoke_url}/${aws_api_gateway_resource.webhook_resource.path_part}"
 }