14/07/2023 - AWS, GO, TERRAFORM
In this example we use Terraform to configure our AWS account to send alarm emails to our inbox. The logic is simple. AWS EventBridge Scheduler runs every minute to call Lambda to log something random. Logs will contain INFO, WARN and ERROR levels. This is just for simulating application logs. Now, the main part is just starting. We configure CloudWatch to watch our specific log group every 30 seconds to see if there is a new WARN or ERROR logs entry. If there is, it tells SNS to send alarm emails. This is the whole logic.
├── cmd
│ └── greeter
│ └── main.go
├── .gitignore
├── go.mod
├── go.sum
├── main.go
└── terraform
└── development
├── consumer.tf
├── main.tf
└── producer.tf
package main
import (
"context"
"os"
"time"
"github.com/aws/aws-lambda-go/events"
"github.com/aws/aws-lambda-go/lambda"
"golang.org/x/exp/slog"
)
func main() {
lambda.Start(handler)
}
func handler(ctx context.Context, event events.CloudWatchEvent) {
logger := slog.New(slog.NewJSONHandler(os.Stdout, nil))
logger.Info("scheduler is calling")
if sec := time.Now().UTC().Nanosecond(); sec%2 == 0 {
logger.Warn("an even event is detected",
slog.Any("event", event),
)
return
}
logger.Error("an uneven event is detected",
slog.Any("event", event),
)
}
.terraform/
terraform.tfstate*
tmp/
bin/
terraform {
required_version = "~> 1.4.4"
required_providers {
aws = {
source = "hashicorp/aws"
version = "~> 4.41.0"
}
archive = {
source = "hashicorp/archive"
version = "~> 2.3.0"
}
null = {
source = "hashicorp/null"
version = "~> 3.2.1"
}
}
}
provider "aws" {
profile = "development"
region = "eu-west-1"
}
# -- VARS ----------------------------------------------------------------------
locals {
lambda_function_name = "greeter"
go_source_path = "${path.module}/../../cmd/${local.lambda_function_name}/..."
go_binary_path = "${path.module}/../../bin/${local.lambda_function_name}"
go_zip_path = "${path.module}/../../tmp/${local.lambda_function_name}.zip"
}
# -- LAMBDA --------------------------------------------------------------------
resource "null_resource" "lambda_go_binary" {
provisioner "local-exec" {
command = "GOOS=linux GOARCH=amd64 CGO_ENABLED=0 GOFLAGS=-trimpath go build -mod=readonly -ldflags='-s -w' -o ${local.go_binary_path} ${local.go_source_path}"
}
}
data "archive_file" "lambda_go_zip" {
type = "zip"
source_file = local.go_binary_path
output_path = local.go_zip_path
depends_on = [
null_resource.lambda_go_binary,
]
}
resource "aws_lambda_function" "greeter" {
function_name = local.lambda_function_name
handler = local.lambda_function_name
filename = local.go_zip_path
package_type = "Zip"
runtime = "go1.x"
timeout = 30
memory_size = 128
role = aws_iam_role.lambda_executor.arn
source_code_hash = data.archive_file.lambda_go_zip.output_base64sha256
depends_on = [
aws_cloudwatch_log_group.lambda_log_group,
]
}
resource "aws_iam_role" "lambda_executor" {
name = "greeter-lambda-executor"
managed_policy_arns = [aws_iam_policy.lambda_log.arn]
assume_role_policy = jsonencode({
Version = "2012-10-17"
Statement = [
{
Action = "sts:AssumeRole"
Effect = "Allow"
Principal = {
Service = "lambda.amazonaws.com"
}
},
]
})
}
resource "aws_iam_policy" "lambda_log" {
name = "greeter-lambda-log"
policy = jsonencode({
Version = "2012-10-17"
Statement = [
{
Action = [
"logs:CreateLogGroup",
"logs:CreateLogStream",
"logs:PutLogEvents",
]
Effect = "Allow"
Resource = [
"arn:aws:logs:*:*:*",
]
},
]
})
}
resource "aws_cloudwatch_log_group" "lambda_log_group" {
name = "/aws/lambda/${local.lambda_function_name}"
retention_in_days = 5
}
# -- CLOUDWATCH ----------------------------------------------------------------
resource "aws_scheduler_schedule" "schedule" {
name = "greeter-lambda-schedule"
flexible_time_window {
mode = "OFF"
}
schedule_expression = "rate(1 minutes)"
target {
arn = aws_lambda_function.greeter.arn
role_arn = aws_iam_role.schedule_executor.arn
}
}
resource "aws_iam_role" "schedule_executor" {
name = "greeter-lambda-schedule-executor"
managed_policy_arns = [aws_iam_policy.schedule_invoker.arn]
assume_role_policy = jsonencode({
Version = "2012-10-17"
Statement = [
{
Action = "sts:AssumeRole"
Effect = "Allow"
Principal = {
Service = "scheduler.amazonaws.com"
}
},
]
})
}
resource "aws_iam_policy" "schedule_invoker" {
name = "greeter-lambda-schedule-invoker"
policy = jsonencode({
Version = "2012-10-17"
Statement = [
{
Action = "lambda:InvokeFunction"
Effect = "Allow"
Resource = aws_lambda_function.greeter.arn
},
]
})
}
# -- CLOUDWATCH ----------------------------------------------------------------
resource "aws_cloudwatch_log_metric_filter" "lambda_log_filter" {
name = "greeter-lambda-log-filter"
pattern = "{ $.level = WARN || $.level = ERROR }"
log_group_name = aws_cloudwatch_log_group.lambda_log_group.name
metric_transformation {
name = "greeter-lambda-warning-and-error-logs"
namespace = "greeter-lambda-issues"
value = "1"
}
}
resource "aws_cloudwatch_metric_alarm" "lambda_log_alarm" {
alarm_name = "greeter-lambda-log-alarm"
alarm_description = "There is an issue with the greeter lambda function"
comparison_operator = "GreaterThanThreshold"
threshold = 0
evaluation_periods = 1
datapoints_to_alarm = 1
period = 30
statistic = "Sum"
alarm_actions = [aws_sns_topic.lambda_log_alarm.arn]
metric_name = aws_cloudwatch_log_metric_filter.lambda_log_filter.metric_transformation[0].name
namespace = aws_cloudwatch_log_metric_filter.lambda_log_filter.metric_transformation[0].namespace
}
# -- SNS -----------------------------------------------------------------------
resource "aws_sns_topic" "lambda_log_alarm" {
name = "greeter-lambda-log-alarm"
}
resource "aws_sns_topic_subscription" "lambda_log_alarm_receiver" {
topic_arn = aws_sns_topic.lambda_log_alarm.arn
protocol = "email"
endpoint = "you@example.com"
}
me:~/aws/terraform/development$ terraform apply \
-replace="null_resource.lambda_go_binary" \
-replace="archive_file.lambda_go_zip" \
-replace="aws_lambda_function.greeter"
You are receiving this email because your Amazon CloudWatch Alarm "greeter-lambda-log-alarm" in the EU (Ireland) region has entered the ALARM state, because "Threshold Crossed: 1 out of the last 1 datapoints [1.0 (14/07/23 20:51:00)] was greater than the threshold (0.0) (minimum 1 datapoint for OK -> ALARM transition)." at "Friday 14 July, 2023 20:51:52 UTC".
View this alarm in the AWS Management Console:
https://eu-west-1.console.aws.amazon.com/cloudwatch/......
Alarm Details:
- Name: greeter-lambda-log-alarm
- Description: There is an issue with the greeter lambda function
- State Change: INSUFFICIENT_DATA -> ALARM
- Reason for State Change: Threshold Crossed: 1 out of the last 1 datapoints [1.0 (14/07/23 20:51:00)] was greater than the threshold (0.0) (minimum 1 datapoint for OK -> ALARM transition).
- Timestamp: Friday 14 July, 2023 20:51:52 UTC
- AWS Account: 124789012345
- Alarm Arn: arn:aws:cloudwatch:eu-west-1:124789012345:alarm:greeter-lambda-log-alarm
Threshold:
- The alarm is in the ALARM state when the metric is GreaterThanThreshold 0.0 for at least 1 of the last 1 period(s) of 30 seconds.
Monitored Metric:
- MetricNamespace: greeter-lambda-issues
- MetricName: greeter-lambda-warning-and-error-logs
- Dimensions:
- Period: 30 seconds
- Statistic: Sum
- Unit: not specified
- TreatMissingData: missing
State Change Actions:
- OK:
- ALARM: [arn:aws:sns:eu-west-1:124789012345:greeter-lambda-log-alarm]
- INSUFFICIENT_DATA: