43탄-2.테라폼-Amazon EKS클러스터 생성2
참고 사이트
https://jennifersoft.com/ko/blog/tech/2023-08-23-jennifer-kubernetes-2/
<1> 명령 EC2 1개 생성
<2> 테라폼으로 EKS 생성,126 버전
<3> 삭제 = NAT GW 삭제 필수!!!
<1> 명령 EC2 1개 생성
https://brunch.co.kr/@topasvga/3585
<2> 테라폼으로 EKS 생성,126 버전
1
참고 사이트 내용을 조금 수정했습니다.
2
main.tf 파일 1개로 만들었습니다~
ls
main.tf outputs.tf terraform.tfstate
3
[root@myeks2-bastion-EC2 terraform-eks]# more *.tf
::::::::::::::
main.tf
::::::::::::::
terraform {
required_version = "~> 1.4"
required_providers {
aws = {
source = "hashicorp/aws"
version = "~> 4.63"
}
}
/*
backend "s3" {
bucket = "jerry-test-tfstate"
key = "jerry-test.tfstate"
region = "ap-northeast-2"
profile = "jerry-test"
dynamodb_table = "TerraformStateLock"
}
*/
}
provider "aws" {
region = local.region
# shared_config_files=["~/.aws/config"] # Or $HOME/.aws/config
# shared_credentials_files = ["~/.aws/credentials"] # Or $HOME/.aws/credentials
# profile = "jerry-test"
}
# Error handling with "The configmap "aws-auth" does not exist"
# https://github.com/terraform-aws-modules/terraform-aws-eks/issues/2009
/*
data "aws_eks_cluster" "default" {
name = module.eks.cluster_name
}
*/
data "aws_eks_cluster_auth" "default" {
name = module.eks.cluster_name
}
provider "kubernetes" {
host = module.eks.cluster_endpoint
cluster_ca_certificate = base64decode(module.eks.cluster_certificate_authority_data)
token = data.aws_eks_cluster_auth.default.token
exec {
api_version = "client.authentication.k8s.io/v1beta1"
command = "aws"
# This requires the awscli to be installed locally where Terraform is executed
args = ["eks", "get-token", "--cluster-name", module.eks.cluster_name]
}
}
data "aws_availability_zones" "available" {}
data "aws_caller_identity" "current" {}
locals {
name = "jerry-test"
cluster_version = "1.26"
region = "ap-northeast-2"
vpc_cidr = "10.110.0.0/16"
azs = slice(data.aws_availability_zones.available.names, 0, 3)
tags = {
env = "test"
user = "jerry"
}
}
resource "aws_iam_policy" "additional" {
name = "${local.name}-additional"
policy = jsonencode({
Version = "2012-10-17"
Statement = [
{
Action = [
"ec2:Describe*",
]
Effect = "Allow"
Resource = "*"
},
]
})
}
module "kms" {
source = "terraform-aws-modules/kms/aws"
version = "1.1.0"
aliases = ["eks/${local.name}"]
description = "${local.name} cluster encryption key"
enable_default_policy = true
key_owners = [data.aws_caller_identity.current.arn]
tags = local.tags
}
/*
resource "aws_eks_addon" "aws_ebs_csi_driver" {
cluster_name = module.eks.cluster_name
addon_name = "aws-ebs-csi-driver"
service_account_role_arn = "arn:aws:iam::${var.aws_account_id}:role/ebs-csi-switch-jerry-test"
addon_version = "v1.22.0-eksbuild.2"
resolve_conflicts = "OVERWRITE"
}
*/
module "irsa-ebs-csi" {
source = "terraform-aws-modules/iam/aws//modules/iam-assumable-role-with-oidc"
version = "4.7.0"
create_role = true
role_name = "AmazonEKSTFEBSCSIRole-${module.eks.cluster_name}"
provider_url = module.eks.oidc_provider
role_policy_arns = [data.aws_iam_policy.ebs_csi_policy.arn]
oidc_fully_qualified_subjects = ["system:serviceaccount:kube-system:ebs-csi-controller-sa"]
}
resource "aws_eks_addon" "ebs-csi" {
cluster_name = module.eks.cluster_name
addon_name = "aws-ebs-csi-driver"
addon_version = "v1.20.0-eksbuild.1"
service_account_role_arn = module.irsa-ebs-csi.iam_role_arn
tags = {
"eks_addon" = "ebs-csi"
"terraform" = "true"
}
}
data "aws_iam_policy" "ebs_csi_policy" {
arn = "arn:aws:iam::aws:policy/service-role/AmazonEBSCSIDriverPolicy"
}
module "eks" {
source = "terraform-aws-modules/eks/aws"
version = "~> 19.0"
cluster_name = local.name
cluster_version = local.cluster_version
cluster_endpoint_public_access = true
cluster_addons = {
coredns = {
preserve = true
most_recent = true
timeouts = {
create = "25m"
delete = "10m"
}
}
kube-proxy = {
most_recent = true
}
vpc-cni = {
most_recent = true
}
}
# coreDNS , kube-proxy,vpc-cni등 미리 add-on으로 관리하면 향후 EKS 업그레이드 작어비 신규 버전 정보만 명시하여 편리하게 업그레이드 할수 있습니다.
create_kms_key = false
cluster_encryption_config = {
resources = ["secrets"]
provider_key_arn = module.kms.key_arn
}
manage_aws_auth_configmap = true
aws_auth_roles = [
# We need to add in the Karpenter node IAM role for nodes launched by Karpenter
{
rolearn = module.karpenter.role_arn
username = "system:node:{{EC2PrivateDNSName}}"
groups = [
"system:bootstrappers",
"system:nodes",
]
},
]
iam_role_additional_policies = {
additional = aws_iam_policy.additional.arn
}
vpc_id = module.vpc.vpc_id
subnet_ids = module.vpc.private_subnets
control_plane_subnet_ids = module.vpc.intra_subnets
# 컨트롤 플래인 서브넷 대역은 별도로 한다.
eks_managed_node_group_defaults = {
ami_type = "AL2_x86_64"
iam_role_additional_policies = {
additional = aws_iam_policy.additional.arn
}
ebs_optimized = true
block_device_mappings = {
xvda = {
device_name = "/dev/xvda"
ebs = {
volume_size = 100
volume_type = "gp3"
iops = 3000
throughput = 150
# encrypted = true
# kms_key_id = aws_kms_key.ebs.arn
delete_on_termination = true
}
}
}
tags = local.tags
}
# SPOT 사용
eks_managed_node_groups = {
base = {
name = "karpenter"
use_name_prefix = false
instance_types = ["t3.large", "t3a.large", "m6i.large"]
capacity_type = "SPOT"
min_size = 1
max_size = 2
desired_size = 1
subnet_ids = module.vpc.private_subnets
}
}
tags = merge(local.tags, {
# NOTE - if creating multiple security groups with this module, only tag the
# security group that Karpenter should utilize with the following tag
# (i.e. - at most, only one security group should have this tag in your account)
"karpenter.sh/discovery" = local.name
})
}
module "load_balancer_controller_irsa_role" {
source = "terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks"
role_name = "load-balancer-controller"
attach_load_balancer_controller_policy = true
oidc_providers = {
ex = {
provider_arn = module.eks.oidc_provider_arn
namespace_service_accounts = ["kube-system:aws-load-balancer-controller"]
}
}
tags = local.tags
}
/*
module "external_dns_irsa_role" {
source = "terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks"
role_name = "external-dns"
attach_external_dns_policy = true
external_dns_hosted_zone_arns = ["arn:aws:route53:::hostedzone/${var.hosted_zone_id}"]
oidc_providers = {
ex = {
provider_arn = module.eks.oidc_provider_arn
namespace_service_accounts = ["kube-system:external-dns"]
}
}
tags = local.tags
}
*/
module "ebs_csi_irsa_role" {
source = "terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks"
role_name = "ebs-csi-switch-jerry-test"
attach_ebs_csi_policy = true
oidc_providers = {
ex = {
provider_arn = module.eks.oidc_provider_arn
namespace_service_accounts = ["kube-system:ebs-csi-controller-sa"]
}
}
tags = local.tags
}
module "karpenter" {
source = "terraform-aws-modules/eks/aws//modules/karpenter"
cluster_name = module.eks.cluster_name
irsa_oidc_provider_arn = module.eks.oidc_provider_arn
policies = {
AmazonSSMManagedInstanceCore = "arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore"
}
tags = local.tags
}
module "vpc" {
source = "terraform-aws-modules/vpc/aws"
version = "~>3.12"
name = local.name
cidr = local.vpc_cidr
azs = local.azs
private_subnets = [for k, v in local.azs : cidrsubnet(local.vpc_cidr, 4, k)]
public_subnets = [for k, v in local.azs : cidrsubnet(local.vpc_cidr, 8, k + 48)]
intra_subnets = [for k, v in local.azs : cidrsubnet(local.vpc_cidr, 8, k + 52)]
# 비트 계산
vpc가 16비트 임. 10.110.0.0 /16
16+4 = 20 비트 , C-class 16개 = 0~15 , 16~31 , 32~47 = 10.110.0.0/20 , 10.110.16.0/20, 10.110.32.0/20
16+8 = 24 비트 , C-Class 1개 = 48, 49, 50 = 10.110.48.0/24 , 10.110.49.0/24 , 10.110.50.0/24
16+8= 24 비트 , C-Class 1개 = 52, 53, 54 = 10.110.52.0/24 , 10.110.53.0/24 , 10.110.54.0/24
enable_nat_gateway = true
enable_dns_hostnames = true
enable_flow_log = true
create_flow_log_cloudwatch_iam_role = true
create_flow_log_cloudwatch_log_group = true
public_subnet_tags = {
"kubernetes.io/role/elb" = 1
"karpenter.sh/discovery" = local.name
}
private_subnet_tags = {
"kubernetes.io/role/internal-elb" = 1
"karpenter.sh/discovery" = local.name
}
tags = local.tags
}
::::::::::::::
outputs.tf
::::::::::::::
################################################################################
# Cluster
################################################################################
output "cluster_arn" {
description = "The Amazon Resource Name (ARN) of the cluster"
value = module.eks.cluster_arn
}
output "cluster_certificate_authority_data" {
description = "Base64 encoded certificate data required to communicate with the cluster"
value = module.eks.cluster_certificate_authority_data
}
output "cluster_endpoint" {
description = "Endpoint for your Kubernetes API server"
value = module.eks.cluster_endpoint
}
output "cluster_id" {
description = "The ID of the EKS cluster. Note: currently a value is returned only for local EKS clusters created on Outposts"
value = module.eks.cluster_id
}
output "cluster_name" {
description = "The name of the EKS cluster"
value = module.eks.cluster_name
}
output "cluster_oidc_issuer_url" {
description = "The URL on the EKS cluster for the OpenID Connect identity provider"
value = module.eks.cluster_oidc_issuer_url
}
output "cluster_platform_version" {
description = "Platform version for the cluster"
value = module.eks.cluster_platform_version
}
output "cluster_status" {
description = "Status of the EKS cluster. One of `CREATING`, `ACTIVE`, `DELETING`, `FAILED`"
value = module.eks.cluster_status
}
output "cluster_primary_security_group_id" {
description = "Cluster security group that was created by Amazon EKS for the cluster. Managed node groups use this security group for control-plane-to-data-plane communication. Referred to as 'Cluster security group' in the EKS console
"
value = module.eks.cluster_primary_security_group_id
}
################################################################################
# Security Group
################################################################################
output "cluster_security_group_arn" {
description = "Amazon Resource Name (ARN) of the cluster security group"
value = module.eks.cluster_security_group_arn
}
output "cluster_security_group_id" {
description = "ID of the cluster security group"
value = module.eks.cluster_security_group_id
}
################################################################################
# Node Security Group
################################################################################
output "node_security_group_arn" {
description = "Amazon Resource Name (ARN) of the node shared security group"
value = module.eks.node_security_group_arn
}
output "node_security_group_id" {
description = "ID of the node shared security group"
value = module.eks.node_security_group_id
}
################################################################################
# IRSA
################################################################################
output "oidc_provider" {
description = "The OpenID Connect identity provider (issuer URL without leading `https://`)"
value = module.eks.oidc_provider
}
output "oidc_provider_arn" {
description = "The ARN of the OIDC Provider if `enable_irsa = true`"
value = module.eks.oidc_provider_arn
}
output "cluster_tls_certificate_sha1_fingerprint" {
description = "The SHA1 fingerprint of the public key of the cluster's certificate"
value = module.eks.cluster_tls_certificate_sha1_fingerprint
}
################################################################################
# IAM Role
################################################################################
output "cluster_iam_role_name" {
description = "IAM role name of the EKS cluster"
value = module.eks.cluster_iam_role_name
}
output "cluster_iam_role_arn" {
description = "IAM role ARN of the EKS cluster"
value = module.eks.cluster_iam_role_arn
}
output "cluster_iam_role_unique_id" {
description = "Stable and unique string identifying the IAM role"
value = module.eks.cluster_iam_role_unique_id
}
################################################################################
# EKS Addons
################################################################################
output "cluster_addons" {
description = "Map of attribute maps for all EKS cluster addons enabled"
value = module.eks.cluster_addons
}
################################################################################
# EKS Identity Provider
################################################################################
output "cluster_identity_providers" {
description = "Map of attribute maps for all EKS identity providers enabled"
value = module.eks.cluster_identity_providers
}
################################################################################
# CloudWatch Log Group
################################################################################
output "cloudwatch_log_group_name" {
description = "Name of cloudwatch log group created"
value = module.eks.cloudwatch_log_group_name
}
output "cloudwatch_log_group_arn" {
description = "Arn of cloudwatch log group created"
value = module.eks.cloudwatch_log_group_arn
}
################################################################################
# Fargate Profile
################################################################################
output "fargate_profiles" {
description = "Map of attribute maps for all EKS Fargate Profiles created"
value = module.eks.fargate_profiles
}
################################################################################
# EKS Managed Node Group
################################################################################
output "eks_managed_node_groups" {
description = "Map of attribute maps for all EKS managed node groups created"
value = module.eks.eks_managed_node_groups
}
output "eks_managed_node_groups_autoscaling_group_names" {
description = "List of the autoscaling group names created by EKS managed node groups"
value = module.eks.eks_managed_node_groups_autoscaling_group_names
}
################################################################################
# Self Managed Node Group
################################################################################
output "self_managed_node_groups" {
description = "Map of attribute maps for all self managed node groups created"
value = module.eks.self_managed_node_groups
}
output "self_managed_node_groups_autoscaling_group_names" {
description = "List of the autoscaling group names created by self-managed node groups"
value = module.eks.self_managed_node_groups_autoscaling_group_names
}
################################################################################
# Additional
################################################################################
output "aws_auth_configmap_yaml" {
description = "Formatted yaml output for base aws-auth configmap containing roles used in cluster node groups/fargate profiles"
value = module.eks.aws_auth_configmap_yaml
}
################################################################################
# IAM Role for Service Account (IRSA)
################################################################################
output "karpenter_irsa_name" {
description = "The name of the IAM role for service accounts"
value = module.karpenter.irsa_name
}
output "karpenter_irsa_arn" {
description = "The Amazon Resource Name (ARN) specifying the IAM role for service accounts"
value = module.karpenter.irsa_arn
}
output "karpenter_irsa_unique_id" {
description = "Stable and unique string identifying the IAM role for service accounts"
value = module.karpenter.irsa_unique_id
}
################################################################################
# Node Termination Queue
################################################################################
output "karpenter_queue_arn" {
description = "The ARN of the SQS queue"
value = module.karpenter.queue_arn
}
output "karpenter_queue_name" {
description = "The name of the created Amazon SQS queue"
value = module.karpenter.queue_name
}
output "karpenter_queue_url" {
description = "The URL for the created Amazon SQS queue"
value = module.karpenter.queue_url
}
################################################################################
# Node Termination Event Rules
################################################################################
output "karpenter_event_rules" {
description = "Map of the event rules created and their attributes"
value = module.karpenter.event_rules
}
################################################################################
# Node IAM Role
################################################################################
output "karpenter_role_name" {
description = "The name of the IAM role"
value = module.karpenter.role_name
}
output "karpenter_role_arn" {
description = "The Amazon Resource Name (ARN) specifying the IAM role"
value = module.karpenter.role_arn
}
output "karpenter_role_unique_id" {
description = "Stable and unique string identifying the IAM role"
value = module.karpenter.role_unique_id
}
################################################################################
# Node IAM Instance Profile
################################################################################
output "karpenter_instance_profile_arn" {
description = "ARN assigned by AWS to the instance profile"
value = module.karpenter.instance_profile_arn
}
output "karpenter_instance_profile_id" {
description = "Instance profile's ID"
value = module.karpenter.instance_profile_id
}
output "karpenter_instance_profile_name" {
description = "Name of the instance profile"
value = module.karpenter.instance_profile_name
}
output "karpenter_instance_profile_unique" {
description = "Stable and unique string identifying the IAM instance profile"
value = module.karpenter.instance_profile_unique
}
4
권한 주기
aws eks --region ap-northeast-2 update-kubeconfig --name $(terraform output -raw cluster_name)
5
k get nodes
NAME STATUS ROLES AGE VERSION
ip-10-110-46-96.ap-northeast-2.compute.internal Ready <none> 15m v1.26.10-eks-e71965b
6
콘솔 확인
EKS 클러스터 1.26으로 설치됨
버전 업은 1.27로 한단계만 가능함. 20여분 걸림
<3> 삭제 = NAT GW 삭제 필수!!!
다음 공부
https://brunch.co.kr/@topasvga/3589
감사합니다.
