Skip to content

Commit

Permalink
Merge pull request #6 from baikonur-oss/0.12-upgrade
Browse files Browse the repository at this point in the history
0.12 upgrade
  • Loading branch information
prog893 authored Nov 14, 2019
2 parents be20b5b + 369c378 commit 58b3764
Show file tree
Hide file tree
Showing 6 changed files with 47 additions and 40 deletions.
2 changes: 1 addition & 1 deletion .pre-commit-config.yaml
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
repos:
- repo: git://github.com/antonbabenko/pre-commit-terraform
rev: v1.7.3
rev: v1.16.0
hooks:
- id: terraform_fmt
- id: terraform_docs
12 changes: 7 additions & 5 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

Terraform module and Lambda for saving JSON log records from Kinesis Data Streams to Elasticsearch Service.

![terraform v0.11.x](https://img.shields.io/badge/terraform-v0.11.x-brightgreen.svg)
![terraform v0.12.x](https://img.shields.io/badge/terraform-v0.12.x-brightgreen.svg)

## Prerequisites
1. Records in Kinesis stream must be valid JSON data. Non-JSON data will be saved with `unknown` prefix.
Expand All @@ -22,18 +22,20 @@ resource "aws_kinesis_stream" "stream" {
module "kinesis_to_elasticsearch" {
source = "baikonur-oss/lambda-kinesis-to-es/aws"
version = "2.0.0"
lambda_package_url = "https://github.com/baikonur-oss/terraform-aws-lambda-kinesis-to-es/releases/download/v1.0.0/lambda_package.zip"
lambda_package_url = "https://github.com/baikonur-oss/terraform-aws-lambda-kinesis-to-es/releases/download/v2.0.0/lambda_package.zip"
name = "kinesis_to_es"
kinesis_stream_arn = "${aws_kinesis_stream.stream.arn}"
kinesis_stream_arn = aws_kinesis_stream.stream.arn
elasticsearch_host = "search-dev-elasticsearch-xxxxxxxx.ap-northeast-1.es.amazonaws.com"
elasticsearch_arn = "arn:aws:es:ap-northeast-1:0123456789:domain/elasticsearch"
failed_log_s3_bucket = "failed-log"
failed_log_s3_prefix = "elasticsearch/"
index_name_prefix = "dev-logs"
max_batch_size = 100
}
```

Warning: use same module and package versions!
Expand Down Expand Up @@ -70,13 +72,13 @@ For more information on module version pinning, see [Selecting a Revision](https
| log\_retention\_in\_days | Lambda Function log retention in days | string | `"30"` | no |
| log\_timestamp\_field | Key name for log timestamp | string | `"time"` | no |
| log\_type\_field | Key name for log type | string | `"log_type"` | no |
| log\_type\_field\_whitelist | Log type whitelist (if empty, all types will be processed) | list | `<list>` | no |
| log\_type\_field\_whitelist | Log type whitelist (if empty, all types will be processed) | list(string) | `[]` | no |
| log\_type\_unknown\_prefix | Log type prefix for logs without log type field | string | `"unknown"` | no |
| memory | Lambda Function memory in megabytes | string | `"256"` | no |
| name | Resource name | string | n/a | yes |
| runtime | Lambda Function runtime | string | `"python3.7"` | no |
| starting\_position | Kinesis ShardIterator type (see: https://docs.aws.amazon.com/kinesis/latest/APIReference/API_GetShardIterator.html ) | string | `"TRIM_HORIZON"` | no |
| tags | Tags for Lambda Function | map | `<map>` | no |
| tags | Tags for Lambda Function | map(string) | `{}` | no |
| timeout | Lambda Function timeout in seconds | string | `"60"` | no |
| timezone | tz database timezone name (e.g. Asia/Tokyo) | string | `"UTC"` | no |
| tracing\_mode | X-Ray tracing mode (see: https://docs.aws.amazon.com/lambda/latest/dg/API_TracingConfig.html ) | string | `"PassThrough"` | no |
Expand Down
Empty file added example.tf
Empty file.
64 changes: 32 additions & 32 deletions main.tf
Original file line number Diff line number Diff line change
Expand Up @@ -7,73 +7,73 @@ data "external" "package" {
}

data "aws_s3_bucket" "failed_log_bucket" {
bucket = "${var.failed_log_s3_bucket}"
bucket = var.failed_log_s3_bucket
}

resource "aws_cloudwatch_log_group" "logs" {
name = "/aws/lambda/${var.name}"

retention_in_days = "${var.log_retention_in_days}"
retention_in_days = var.log_retention_in_days
}

resource "aws_lambda_function" "function" {
function_name = "${var.name}"
handler = "${var.handler}"
role = "${module.iam.arn}"
runtime = "${var.runtime}"
memory_size = "${var.memory}"
timeout = "${var.timeout}"
function_name = var.name
handler = var.handler
role = module.iam.arn
runtime = var.runtime
memory_size = var.memory
timeout = var.timeout

filename = "${local.package_filename}"
filename = local.package_filename

# Below is a very dirty hack to force base64sha256 to wait until
# package download in data.external.package finishes.
#
# WARNING: explicit depends_on from this resource to data.external.package
# does not help

source_code_hash = "${base64sha256(file("${jsonencode(data.external.package.result) == "{}" ? local.package_filename : ""}"))}"
source_code_hash = filebase64sha256(
jsonencode(data.external.package.result) == "{}" ? local.package_filename : "",
)
tracing_config {
mode = "${var.tracing_mode}"
mode = var.tracing_mode
}
environment {
variables {
TZ = "${var.timezone}"

LOG_ID_FIELD = "${var.log_id_field}"
LOG_TYPE_FIELD = "${var.log_type_field}"
LOG_TYPE_UNKNOWN_PREFIX = "${var.log_type_unknown_prefix}"
LOG_TIMESTAMP_FIELD = "${var.log_timestamp_field}"
LOG_TYPE_WHITELIST = "${join(",", var.log_type_field_whitelist)}"

ES_HOST = "${var.elasticsearch_host}"
FAILED_LOG_S3_BUCKET = "${var.failed_log_s3_bucket}"
FAILED_LOG_S3_PREFIX = "${var.failed_log_s3_prefix}"
INDEX_NAME_PREFIX = "${var.index_name_prefix}"
variables = {
TZ = var.timezone
LOG_ID_FIELD = var.log_id_field
LOG_TYPE_FIELD = var.log_type_field
LOG_TYPE_UNKNOWN_PREFIX = var.log_type_unknown_prefix
LOG_TIMESTAMP_FIELD = var.log_timestamp_field
LOG_TYPE_WHITELIST = join(",", var.log_type_field_whitelist)
ES_HOST = var.elasticsearch_host
FAILED_LOG_S3_BUCKET = var.failed_log_s3_bucket
FAILED_LOG_S3_PREFIX = var.failed_log_s3_prefix
INDEX_NAME_PREFIX = var.index_name_prefix
}
}
tags = "${var.tags}"
tags = var.tags
}

resource "aws_lambda_event_source_mapping" "kinesis_mapping" {
batch_size = "${var.batch_size}"
event_source_arn = "${var.kinesis_stream_arn}"
batch_size = var.batch_size
event_source_arn = var.kinesis_stream_arn
enabled = true
function_name = "${aws_lambda_function.function.arn}"
starting_position = "${var.starting_position}"
function_name = aws_lambda_function.function.arn
starting_position = var.starting_position
}

resource "aws_iam_role_policy_attachment" "xray_access" {
policy_arn = "arn:aws:iam::aws:policy/AWSXrayWriteOnlyAccess"
role = "${module.iam.name}"
role = module.iam.name
}

module "iam" {
source = "baikonur-oss/iam-nofile/aws"
version = "v1.0.1"

type = "lambda"
name = "${var.name}"
name = var.name

policy_json = <<EOF
{
Expand Down Expand Up @@ -124,8 +124,8 @@ module "iam" {
]
}
EOF

}

# Note: there is no data resource for AWS ES domains
# memo: https://github.com/terraform-providers/terraform-provider-aws/pull/7932

5 changes: 3 additions & 2 deletions variables.tf
Original file line number Diff line number Diff line change
Expand Up @@ -87,7 +87,7 @@ variable "log_timestamp_field" {
variable "log_type_field_whitelist" {
description = "Log type whitelist (if empty, all types will be processed)"
default = []
type = "list"
type = list(string)
}

variable "tracing_mode" {
Expand All @@ -97,11 +97,12 @@ variable "tracing_mode" {

variable "tags" {
description = "Tags for Lambda Function"
type = "map"
type = map(string)
default = {}
}

variable "log_retention_in_days" {
description = "Lambda Function log retention in days"
default = 30
}

4 changes: 4 additions & 0 deletions versions.tf
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@

terraform {
required_version = ">= 0.12"
}

0 comments on commit 58b3764

Please sign in to comment.