■ はじめに

休み明け、どうせ時間がない見込み。。。
ただ、休み明けで、空いた時間を利用して
AWS MSK Cluster & MSK Connector を構築したい。
なので、Terraformで構築できる準備をしておく。

MSK = Managed Streaming for apache Kafka

Amazon MSK ～入門編～
https://dk521123.hatenablog.com/entry/2023/04/21/000000

【１】公式ドキュメント

１）MSK Clusters

Resource: aws_msk_cluster
https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/msk_cluster
Resource: aws_msk_serverless_cluster
https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/msk_serverless_cluster

２）MSK Clusters configuration

Resource: aws_msk_configuration
https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/msk_configuration

【２】サンプル

０）Github sample

MSK Cluster
https://github.com/angelabad/terraform-aws-msk-cluster/tree/master/examples
https://github.com/cloudposse/terraform-aws-msk-apache-kafka-cluster/tree/master/examples/complete

１）MSK Cluster

For Non-serverless version
https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/msk_cluster#with-volume_throughput-argument

# For Network

# VPC
resource "aws_vpc" "vpc" {
  cidr_block = "192.168.0.0/22"
}

data "aws_availability_zones" "azs" {
  state = "available"
}

# Subnet

resource "aws_subnet" "subnet_az1" {
  availability_zone = data.aws_availability_zones.azs.names[0]
  cidr_block        = "192.168.0.0/24"
  vpc_id            = aws_vpc.vpc.id
}

resource "aws_subnet" "subnet_az2" {
  availability_zone = data.aws_availability_zones.azs.names[1]
  cidr_block        = "192.168.1.0/24"
  vpc_id            = aws_vpc.vpc.id
}

resource "aws_subnet" "subnet_az3" {
  availability_zone = data.aws_availability_zones.azs.names[2]
  cidr_block        = "192.168.2.0/24"
  vpc_id            = aws_vpc.vpc.id
}

# Security Group
resource "aws_security_group" "demo_msk_sg" {
  name        = "demo-msk-sg"
  vpc_id = aws_vpc.vpc.id
  description = "Allow All trafic"

  ingress {
    description      = "TLS from VPC"
    from_port        = 0
    to_port          = 0
    protocol         = "-1"
    cidr_blocks      = ["0.0.0.0/0"]
    ipv6_cidr_blocks = ["::/0"]
  }

  egress {
    from_port        = 0
    to_port          = 0
    protocol         = "-1"
    cidr_blocks      = ["0.0.0.0/0"]
    ipv6_cidr_blocks = ["::/0"]
  }

  tags = {
    Name = "demo-msk-sg"
  }
}

# ★Main★
# MSK cluster
resource "aws_msk_cluster" "demo_msk_cluster" {
  cluster_name           = "demo-msk-cluster"
  kafka_version          = "2.7.1"
  number_of_broker_nodes = 3
  description = "This is a main aws_msk_cluster for this blog"

  broker_node_group_info {
    instance_type = "kafka.m5.4xlarge"
    client_subnets = [
      aws_subnet.subnet_az1.id,
      aws_subnet.subnet_az2.id,
      aws_subnet.subnet_az3.id,
    ]
    storage_info {
      ebs_storage_info {
        provisioned_throughput {
          enabled           = true
          volume_throughput = 250
        }
        volume_size = 1000
      }
    }
    security_groups = [aws_security_group.demo_msk_sg.id]
  }

  # To set timeout (Default is 20min)
  timeouts {
    create = "1h"
    update = "1h"
    delete = "1h"
  }
}

# Zookeeper
output "zookeeper_connect_string" {
  value = aws_msk_cluster.demo_msk_cluster.zookeeper_connect_string
}

# Broker
output "bootstrap_brokers_tls" {
  description = "TLS connection host:port pairs"
  value       = aws_msk_cluster.demo_msk_cluster.bootstrap_brokers_tls
}

２）MSK Custom - Serverless

https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/msk_serverless_cluster

# For Network

# VPC
resource "aws_vpc" "vpc" {
  cidr_block = "192.168.0.0/22"
}

data "aws_availability_zones" "azs" {
  state = "available"
}

# Subnet

resource "aws_subnet" "subnet_az1" {
  availability_zone = data.aws_availability_zones.azs.names[0]
  cidr_block        = "192.168.0.0/24"
  vpc_id            = aws_vpc.vpc.id
}

resource "aws_subnet" "subnet_az2" {
  availability_zone = data.aws_availability_zones.azs.names[1]
  cidr_block        = "192.168.1.0/24"
  vpc_id            = aws_vpc.vpc.id
}

resource "aws_subnet" "subnet_az3" {
  availability_zone = data.aws_availability_zones.azs.names[2]
  cidr_block        = "192.168.2.0/24"
  vpc_id            = aws_vpc.vpc.id
}

# Security Group
resource "aws_security_group" "demo_msk_sg" {
  name        = "demo-msk-sg"
  vpc_id = aws_vpc.vpc.id
  description = "Allow All trafic"

  ingress {
    description      = "TLS from VPC"
    from_port        = 0
    to_port          = 0
    protocol         = "-1"
    cidr_blocks      = ["0.0.0.0/0"]
    ipv6_cidr_blocks = ["::/0"]
  }

  egress {
    from_port        = 0
    to_port          = 0
    protocol         = "-1"
    cidr_blocks      = ["0.0.0.0/0"]
    ipv6_cidr_blocks = ["::/0"]
  }

  tags = {
    Name = "demo-msk-sg"
  }
}

# ★Main★
resource "aws_msk_serverless_cluster" "demo_msk_serverless_cluster" {
  cluster_name = "demo-msk-serverless-cluster"
  description = "This is a main for this blog"

  vpc_config {
    subnet_ids = [
      aws_subnet.subnet_az1.id,
      aws_subnet.subnet_az2.id,
      aws_subnet.subnet_az3.id,
    ]
    security_group_ids = [aws_security_group.demo_msk_sg.id]
  }

  client_authentication {
    sasl {
      iam {
        enabled = true
      }
    }
  }
}

【３】使用上の注意

１）Timeout値について

https://docs.aws.amazon.com/ja_jp/msk/latest/developerguide/troubleshooting.html#troubleshooting-cluster-stuck

より抜粋
~~~~~~~~~~~~~~
クラスターが [作成中] 状態のまま停止しているように見える
クラスターの作成には、最大 30 分かかる場合があります。
30 分間待ってから、クラスターの状態を再度確認します
~~~~~~~~~~~~~~

ただ、Terraform の Timeout値は、（どうやら）20分なので
伸ばしておいた方がいいかも。

プログラムの超個人的なメモ

Memo for Programming.

【Terraform】Terraform ～ AWS MSK ～

■ はじめに

目次

【１】公式ドキュメント

１）MSK Clusters

２）MSK Clusters configuration

【２】サンプル

０）Github sample

１）MSK Cluster

２）MSK Custom - Serverless

【３】使用上の注意

１）Timeout値について

関連記事