19탄-26. 테라폼-AWS-무중단 배포
다음은 주말 CloudNet 테라폼 스터디 내용 참고하여 정리한 부분입니다.
https://gasidaseo.notion.site/gasidaseo/CloudNet-Blog-c9dfa44a27ff431dafdd2edacc8a1863
<1> 백앤드 구축
<2> Staging RDS 배포
<3> 모듈을 활용하여 Staging 웹서버 클러스터 배포 후 무중단 업그레이드
<1> 백앤드 구축
1
# 환경변수에 지정
export TF_VAR_bucket_name=masterseo-t101-tfstate
export TF_VAR_table_name=masterseo-t101-locks
# 환경변수 확인
export | grep TF_VAR_
2
배포
#
cd 03-terraform-state/file-layout-example/global/s3
cat main.tf variables.tf
[root@ip-172-31-61-209 s3]# cat main.tf variables.tf
terraform {
required_version = ">= 1.0.0, < 2.0.0"
required_providers {
aws = {
source = "hashicorp/aws"
version = "~> 4.0"
}
}
}
provider "aws" {
region = "us-east-2"
}
resource "aws_s3_bucket" "terraform_state" {
bucket = var.bucket_name
// This is only here so we can destroy the bucket as part of automated tests. You should not copy this for production
// usage
force_destroy = true
}
# Enable versioning so you can see the full revision history of your
# state files
resource "aws_s3_bucket_versioning" "enabled" {
bucket = aws_s3_bucket.terraform_state.id
versioning_configuration {
status = "Enabled"
}
}
# Enable server-side encryption by default
resource "aws_s3_bucket_server_side_encryption_configuration" "default" {
bucket = aws_s3_bucket.terraform_state.id
rule {
apply_server_side_encryption_by_default {
sse_algorithm = "AES256"
}
}
}
# Explicitly block all public access to the S3 bucket
resource "aws_s3_bucket_public_access_block" "public_access" {
bucket = aws_s3_bucket.terraform_state.id
block_public_acls = true
block_public_policy = true
ignore_public_acls = true
restrict_public_buckets = true
}
resource "aws_dynamodb_table" "terraform_locks" {
name = var.table_name
billing_mode = "PAY_PER_REQUEST"
hash_key = "LockID"
attribute {
name = "LockID"
type = "S"
}
}
variable "bucket_name" {
description = "The name of the S3 bucket. Must be globally unique."
type = string
}
variable "table_name" {
description = "The name of the DynamoDB table. Must be unique in this AWS account."
type = string
}
[root@ip-172-31-61-209 s3]#
# 초기화 및 검증 : 환경변수 적용 확인
terraform init && terraform plan
# 배포
terraform apply -auto-approve
# 확인
aws s3 ls
2022-12-07 08:20:10 masterseo-t101-tfstate
aws dynamodb list-tables --output text
[root@ip-172-31-61-209 s3]# aws dynamodb list-tables --output text
TABLENAMES masterseo-t101-locks
cd ..
cd ..
<2> Staging RDS 배포
1
# [터미널1] RDS 생성 모니터링
while true; do aws rds describe-db-instances --query "*[].[Endpoint.Address,Endpoint.Port,MasterUsername]" --output text ; echo "------------------------------" ; sleep 1; done
2
# [터미널2]
cd /root/terraform-up-and-running-code/code/terraform/04-terraform-module/module-example/stage/data-stores/mysql
cat main.tf variables.tf
[root@ip-172-31-61-209 mysql]# cat main.tf variables.tf
terraform {
required_version = ">= 1.0.0, < 2.0.0"
required_providers {
aws = {
source = "hashicorp/aws"
version = "~> 4.0"
}
}
backend "s3" {
# This backend configuration is filled in automatically at test time by Terratest. If you wish to run this example
# manually, uncomment and fill in the config below.
bucket = "masterseo-t101-tfstate"
key = "stage/data-stores/mysql/terraform.tfstate"
region = "us-east-2"
dynamodb_table = "masterseo-t101-locks"
# encrypt = true
}
}
provider "aws" {
region = "us-east-2"
}
resource "aws_db_instance" "example" {
identifier_prefix = "terraform-up-and-running"
engine = "mysql"
allocated_storage = 10
instance_class = "db.t2.micro"
db_name = var.db_name
username = var.db_username
password = var.db_password
skip_final_snapshot = true
}
# ---------------------------------------------------------------------------------------------------------------------
# REQUIRED PARAMETERS
# You must provide a value for each of these parameters.
# ---------------------------------------------------------------------------------------------------------------------
variable "db_username" {
description = "The username for the database"
type = string
sensitive = true
}
variable "db_password" {
description = "The password for the database"
type = string
sensitive = true
}
# ---------------------------------------------------------------------------------------------------------------------
# OPTIONAL PARAMETERS
# These parameters have reasonable defaults.
# ---------------------------------------------------------------------------------------------------------------------
variable "db_name" {
description = "The name to use for the database"
type = string
default = "example_database_stage"
}
[root@ip-172-31-61-209 mysql]#
# 환경변수에 지정
export TF_VAR_db_username='cloudneta'
export TF_VAR_db_password='cloudnetaQ!'
# 환경변수 확인
export | grep TF_VAR_
# main.tf 에 백엔드 부분 수정
vi main.tf
backend "s3" {
# This backend configuration is filled in automatically at test time by Terratest. If you wish to run this example
# manually, uncomment and fill in the config below.
bucket = "masterseo-t101-tfstate"
key = "stage/data-stores/mysql/terraform.tfstate"
region = "us-east-2"
dynamodb_table = "masterseo-t101-locks"
# encrypt = true
}
# 초기화 및 검증 : 환경변수 적용 확인
terraform init && terraform plan
# 배포 : RDS는 생성 시 6분 정도 시간 소요
terraform apply -auto-approve
터미널2에서 모니터링 결과
terraform-up-and-running20221xxxxxxxx400000001.c7aitcdywcyg.us-east-2.rds.amazonaws.com 3306 cloudneta
------------------------------
terraform-up-and-running202212xxxxxxx400000001.c7aitcdywcyg.us-east-2.rds.amazonaws.com 3306 cloudneta
------------------------------
terraform output
oot@ip-172-31-61-209 mysql]# terraform output
address = "terraform-up-and-running2094400000001.c7aitcdywcyg.us-east-2.rds.amazonaws.com"
port = 3306
aws s3 ls s3://$TF_VAR_bucket_name --recursive --human-readable --summarize
-31-61-209 mysql]# aws s3 ls s3://$TF_VAR_bucket_name --recursive --human-readable --summarize
2022-12-07 08:31:11 4.3 KiB stage/data-stores/mysql/terraform.tfstate
Total Objects: 1
Total Size: 4.3 KiB
# 이동
cd ..
cd ..
<3> 모듈을 활용하여 Staging 웹서버 클러스터 배포 후 무중단 업그레이드
1
#
cd /root/terraform-up-and-running-code/code/terraform/05-tips-and-tricks/zero-downtime-deployment/modules/services/webserver-cluster
cat main.tf variables.tf
[root@ip-172-31-61-209 webserver-cluster]# cat main.tf variables.tf
terraform {
required_version = ">= 1.0.0, < 2.0.0"
required_providers {
aws = {
source = "hashicorp/aws"
version = "~> 4.0"
}
}
}
resource "aws_launch_configuration" "example" {
image_id = var.ami
instance_type = var.instance_type
security_groups = [aws_security_group.instance.id]
user_data = templatefile("${path.module}/user-data.sh", {
server_port = var.server_port
db_address = data.terraform_remote_state.db.outputs.address
db_port = data.terraform_remote_state.db.outputs.port
server_text = var.server_text
})
# Required when using a launch configuration with an auto scaling group.
lifecycle {
create_before_destroy = true
}
}
resource "aws_autoscaling_group" "example" {
# Explicitly depend on the launch configuration's name so each time it's
# replaced, this ASG is also replaced
name = "${var.cluster_name}-${aws_launch_configuration.example.name}"
launch_configuration = aws_launch_configuration.example.name
vpc_zone_identifier = data.aws_subnets.default.ids
target_group_arns = [aws_lb_target_group.asg.arn]
health_check_type = "ELB"
min_size = var.min_size
max_size = var.max_size
# Wait for at least this many instances to pass health checks before
# considering the ASG deployment complete
min_elb_capacity = var.min_size
# When replacing this ASG, create the replacement first, and only delete the
# original after
lifecycle {
create_before_destroy = true
}
tag {
key = "Name"
value = var.cluster_name
propagate_at_launch = true
}
dynamic "tag" {
for_each = {
for key, value in var.custom_tags:
key => upper(value)
if key != "Name"
}
content {
key = tag.key
value = tag.value
propagate_at_launch = true
}
}
}
resource "aws_autoscaling_schedule" "scale_out_during_business_hours" {
count = var.enable_autoscaling ? 1 : 0
scheduled_action_name = "${var.cluster_name}-scale-out-during-business-hours"
min_size = 2
max_size = 10
desired_capacity = 10
recurrence = "0 9 * * *"
autoscaling_group_name = aws_autoscaling_group.example.name
}
resource "aws_autoscaling_schedule" "scale_in_at_night" {
count = var.enable_autoscaling ? 1 : 0
scheduled_action_name = "${var.cluster_name}-scale-in-at-night"
min_size = 2
max_size = 10
desired_capacity = 2
recurrence = "0 17 * * *"
autoscaling_group_name = aws_autoscaling_group.example.name
}
resource "aws_security_group" "instance" {
name = "${var.cluster_name}-instance"
}
resource "aws_security_group_rule" "allow_server_http_inbound" {
type = "ingress"
security_group_id = aws_security_group.instance.id
from_port = var.server_port
to_port = var.server_port
protocol = local.tcp_protocol
cidr_blocks = local.all_ips
}
data "aws_vpc" "default" {
default = true
}
data "aws_subnets" "default" {
filter {
name = "vpc-id"
values = [data.aws_vpc.default.id]
}
}
resource "aws_lb" "example" {
name = var.cluster_name
load_balancer_type = "application"
subnets = data.aws_subnets.default.ids
security_groups = [aws_security_group.alb.id]
}
resource "aws_lb_listener" "http" {
load_balancer_arn = aws_lb.example.arn
port = local.http_port
protocol = "HTTP"
# By default, return a simple 404 page
default_action {
type = "fixed-response"
fixed_response {
content_type = "text/plain"
message_body = "404: page not found"
status_code = 404
}
}
}
resource "aws_lb_target_group" "asg" {
name = var.cluster_name
port = var.server_port
protocol = "HTTP"
vpc_id = data.aws_vpc.default.id
health_check {
path = "/"
protocol = "HTTP"
matcher = "200"
interval = 15
timeout = 3
healthy_threshold = 2
unhealthy_threshold = 2
}
}
resource "aws_lb_listener_rule" "asg" {
listener_arn = aws_lb_listener.http.arn
priority = 100
condition {
path_pattern {
values = ["*"]
}
}
action {
type = "forward"
target_group_arn = aws_lb_target_group.asg.arn
}
}
resource "aws_security_group" "alb" {
name = "${var.cluster_name}-alb"
}
resource "aws_security_group_rule" "allow_http_inbound" {
type = "ingress"
security_group_id = aws_security_group.alb.id
from_port = local.http_port
to_port = local.http_port
protocol = local.tcp_protocol
cidr_blocks = local.all_ips
}
resource "aws_security_group_rule" "allow_all_outbound" {
type = "egress"
security_group_id = aws_security_group.alb.id
from_port = local.any_port
to_port = local.any_port
protocol = local.any_protocol
cidr_blocks = local.all_ips
}
data "terraform_remote_state" "db" {
backend = "s3"
config = {
bucket = var.db_remote_state_bucket
key = var.db_remote_state_key
region = "us-east-2"
}
}
resource "aws_cloudwatch_metric_alarm" "high_cpu_utilization" {
alarm_name = "${var.cluster_name}-high-cpu-utilization"
namespace = "AWS/EC2"
metric_name = "CPUUtilization"
dimensions = {
AutoScalingGroupName = aws_autoscaling_group.example.name
}
comparison_operator = "GreaterThanThreshold"
evaluation_periods = 1
period = 300
statistic = "Average"
threshold = 90
unit = "Percent"
}
resource "aws_cloudwatch_metric_alarm" "low_cpu_credit_balance" {
count = format("%.1s", var.instance_type) == "t" ? 1 : 0
alarm_name = "${var.cluster_name}-low-cpu-credit-balance"
namespace = "AWS/EC2"
metric_name = "CPUCreditBalance"
dimensions = {
AutoScalingGroupName = aws_autoscaling_group.example.name
}
comparison_operator = "LessThanThreshold"
evaluation_periods = 1
period = 300
statistic = "Minimum"
threshold = 10
unit = "Count"
}
locals {
http_port = 80
any_port = 0
any_protocol = "-1"
tcp_protocol = "tcp"
all_ips = ["0.0.0.0/0"]
}# ---------------------------------------------------------------------------------------------------------------------
# REQUIRED PARAMETERS
# You must provide a value for each of these parameters.
# ---------------------------------------------------------------------------------------------------------------------
variable "cluster_name" {
description = "The name to use for all the cluster resources"
type = string
}
variable "db_remote_state_bucket" {
description = "The name of the S3 bucket for the database's remote state"
type = string
}
variable "db_remote_state_key" {
description = "The path for the database's remote state in S3"
type = string
}
variable "instance_type" {
description = "The type of EC2 Instances to run (e.g. t2.micro)"
type = string
}
variable "min_size" {
description = "The minimum number of EC2 Instances in the ASG"
type = number
}
variable "max_size" {
description = "The maximum number of EC2 Instances in the ASG"
type = number
}
variable "enable_autoscaling" {
description = "If set to true, enable auto scaling"
type = bool
}
# ---------------------------------------------------------------------------------------------------------------------
# OPTIONAL PARAMETERS
# These parameters have reasonable defaults.
# ---------------------------------------------------------------------------------------------------------------------
variable "ami" {
description = "The AMI to run in the cluster"
type = string
default = "ami-0fb653ca2d3203ac1"
}
variable "server_text" {
description = "The text the web server should return"
type = string
default = "Hello, World"
}
variable "custom_tags" {
description = "Custom tags to set on the Instances in the ASG"
type = map(string)
default = {}
}
variable "server_port" {
description = "The port the server will use for HTTP requests"
type = number
default = 8080
}
[root@ip-172-31-61-209 webserver-cluster]#
2
# 환경변수에 지정
export TF_VAR_db_remote_state_bucket=$TF_VAR_bucket_name # description = "The name of the S3 bucket used for the database's remote state storage"
export TF_VAR_db_remote_state_key='stage/data-stores/mysql/terraform.tfstate' # description = "The name of the key in the S3 bucket used for the database's remote state storage"
# 환경변수 확인
export | grep TF_VAR_
3
# 초기화 및 검증 : 환경변수 적용 확인
terraform init && terraform plan -var server_text='Old server text'
# 배포
terraform apply -auto-approve -var server_text='Old server text'
# ALB DNS주소로 curl 접속 확인
ALBDNS=$(terraform output -raw alb_dns_name)
while true; do curl --connect-timeout 1 http://$ALBDNS ; echo; echo "------------------------------"; date; sleep 1; done
curl -s http://$ALBDNS
4
# [터미널1]
ALBDNS=<직접입력>
while true; do curl --connect-timeout 1 http://$ALBDNS ; echo; echo "------------------------------"; date; sleep 1; done
5
# [터미널2]
while true; do aws ec2 describe-instances --query "Reservations[*].Instances[*].{PublicIPAdd:PublicIpAddress,InstanceName:Tags[?Key=='Name']|[0].Value,Status:State.Name}" --filters Name=instance-state-name,Values=running --output text ; echo "------------------------------" ; sleep 1; done
# 무중단 업그레이드 배포 : 환경변수 수정 적용 >> 오토스케일링그룹 2개중 기존 그룹은 5분 정도 후에 EC2가 삭제됨(오래 걸리니 맘 편히 기다리자)
terraform plan -var server_text='NEW server text'
terraform apply -auto-approve -var server_text='NEW server text'
6
# 삭제
# 각 폴더에서 리소스 삭제
cd /root/terraform-up-and-running-code/code/terraform/04-terraform-module/module-example/stage/data-stores/mysql
웹 , 디비 , 백엔드 저장소 순으로 삭제한다.
stage/services/webserver-cluster$ terraform destroy -auto-approve
stage/data-stores/mysql$ terraform destroy -auto-approve
03-terraform-state/file-layout-example/global/s3$ terraform destroy -auto-approve
다음
https://brunch.co.kr/@topasvga/2850
https://brunch.co.kr/@topasvga/2421
감사합니다.
