8주차 - EKS IaC(Terraform) 실습

이전 Terraform 이론에서 배운 걸 바탕으로 실제 AWS에서 VPC 및 EC2 배포 / EKS 배포를 실습해보자

 

 Terraform / AWS VPC 및 EC2 배포

 

VPC 생성

• default VPC가 아닌 직접 VPC 생성
• vpc.tf파일에 VPC 관련 코드 작성

provider "aws" {
  region  = "ap-northeast-2"
}

resource "aws_vpc" "myvpc" {
  cidr_block       = "10.10.0.0/16"
  enable_dns_support   = true
  enable_dns_hostnames = true

  tags = {
    Name = "aews-study"
  }
}

resource "aws_subnet" "mysubnet1" {
  vpc_id     = aws_vpc.myvpc.id
  cidr_block = "10.10.1.0/24"

  availability_zone = "ap-northeast-2a"

  tags = {
    Name = "aews-subnet1"
  }
}

resource "aws_subnet" "mysubnet2" {
  vpc_id     = aws_vpc.myvpc.id
  cidr_block = "10.10.2.0/24"

  availability_zone = "ap-northeast-2c"

  tags = {
    Name = "aews-subnet2"
  }
}


resource "aws_internet_gateway" "myigw" {
  vpc_id = aws_vpc.myvpc.id

  tags = {
    Name = "aews-igw"
  }
}

resource "aws_route_table" "myrt" {
  vpc_id = aws_vpc.myvpc.id

  tags = {
    Name = "aews-rt"
  }
}

resource "aws_route_table_association" "myrtassociation1" {
  subnet_id      = aws_subnet.mysubnet1.id
  route_table_id = aws_route_table.myrt.id
}

resource "aws_route_table_association" "myrtassociation2" {
  subnet_id      = aws_subnet.mysubnet2.id
  route_table_id = aws_route_table.myrt.id
}

resource "aws_route" "mydefaultroute" {
  route_table_id         = aws_route_table.myrt.id
  destination_cidr_block = "0.0.0.0/0"
  gateway_id             = aws_internet_gateway.myigw.id
}

output "aws_vpc_id" {
  value = aws_vpc.myvpc.id
}

 

• provider에 aws 명시 / region = 서울리전
• resource로 vpc, subnet1, subnet2, internet-gateway, routing-table  생성
• subnet에 routing-table 설정 및routing-table에 igw 설정
• 생성 후 출력으로 vpc_ip 출력

terraform init && terraform plan && terraform apply -auto-approve

terraform state list
aws_internet_gateway.myigw
aws_route.mydefaultroute
aws_route_table.myrt
aws_route_table_association.myrtassociation1
aws_route_table_association.myrtassociation2
aws_subnet.mysubnet1
aws_subnet.mysubnet2
aws_vpc.myvpc

 

 

 

SG(보안그룹) 배포

• resource 블록에서 aws_security_group 내용 작성
  • 생성된 vpc_id 참조
  • inbound 0.0.0.0 / 80 정책 설정
  • outbound all 정책 설정

resource "aws_security_group" "mysg" {
  vpc_id      = aws_vpc.myvpc.id
  name        = "aews SG"
  description = "aews Study SG"
}

resource "aws_security_group_rule" "mysginbound" {
  type              = "ingress"
  from_port         = 80
  to_port           = 80
  protocol          = "tcp"
  cidr_blocks       = ["0.0.0.0/0"]
  security_group_id = aws_security_group.mysg.id
}

resource "aws_security_group_rule" "mysgoutbound" {
  type              = "egress"
  from_port         = 0
  to_port           = 0
  protocol          = "-1"
  cidr_blocks       = ["0.0.0.0/0"]
  security_group_id = aws_security_group.mysg.id
}

EC2 배포

• data 블록에서 aws 최신 ami정보 저장
• resourece 블록에서 aws_instance 내용 작성
  • data 블록에 ami 내용 참조
  • 생성된 subnet_id, sg_id 내용 참조

data "aws_ami" "my_amazonlinux2" {
  most_recent = true
  filter {
    name   = "owner-alias"
    values = ["amazon"]
  }

  filter {
    name   = "name"
    values = ["amzn2-ami-hvm-*-x86_64-ebs"]
  }

  owners = ["amazon"]
}

resource "aws_instance" "myec2" {

  depends_on = [
    aws_internet_gateway.myigw
  ]

  ami                         = data.aws_ami.my_amazonlinux2.id
  associate_public_ip_address = true
  instance_type               = "t2.micro"
  vpc_security_group_ids      = ["${aws_security_group.mysg.id}"]
  subnet_id                   = aws_subnet.mysubnet1.id

  user_data = <<-EOF
              #!/bin/bash
              wget https://busybox.net/downloads/binaries/1.31.0-defconfig-multiarch-musl/busybox-x86_64
              mv busybox-x86_64 busybox
              chmod +x busybox
              echo "Web Server</h1>" > index.html
              nohup ./busybox httpd -f -p 80 &
              EOF

  user_data_replace_on_change = true

  tags = {
    Name = "aews-myec2"
  }
}

output "myec2_public_ip" {
  value       = aws_instance.myec2.public_ip
  description = "The public IP of the Instance"
}

확인 및 삭제

# 출력된 EC2 퍼블릭IP로 cul 접속 확인
terraform output -raw myec2_public_ip
3.34.141.238

MYIP=$(terraform output -raw myec2_public_ip)
while true; do curl --connect-timeout 1  http://$MYIP/ ; echo "------------------------------"; date; sleep 1; done

terraform destroy -auto-approve

 

 

 

Terraform / AWS EKS 배포

• terraform에서 제공하는 eks 모듈 사용
• vpc.tf, eks.tf, var.tf, irsa,tf

# 코드 가져오기
git clone https://github.com/gasida/aews-cicd.git
cd aews-cicd/4

# terraform 환경 변수 저장

export TF_VAR_KeyName='2024-yjsong-aews'
echo $TF_VAR_KeyName

 

첫 번째 EKS 클러스터 배포

 

 vpc.tf

• 직접 vpc resource를 작성하지 않고 terraform에서 제공하는 vpc module 사용
• var.tf에 작성한 변수를 참조
  • TargetRegion, ClusterBaseName, VpcBlock, availability_zones, public_subnet_blocks, private_subnet_blocks

provider "aws" {
  region = var.TargetRegion
}

module "vpc" {
  source  = "terraform-aws-modules/vpc/aws"
  version = "~>5.7"

  name = "${var.ClusterBaseName}-VPC"
  cidr = var.VpcBlock
  azs  = var.availability_zones

  enable_dns_support   = true
  enable_dns_hostnames = true

  public_subnets  = var.public_subnet_blocks
  private_subnets = var.private_subnet_blocks

  enable_nat_gateway = true
  single_nat_gateway = true
  one_nat_gateway_per_az = false
  
  map_public_ip_on_launch = true
 
  igw_tags = {
    "Name" = "${var.ClusterBaseName}-IGW"
  }
 
  nat_gateway_tags = {
    "Name" = "${var.ClusterBaseName}-NAT"
  }
 
  public_subnet_tags = {
    "Name"                     = "${var.ClusterBaseName}-PublicSubnet"
    "kubernetes.io/role/elb"   = "1"
  }

  private_subnet_tags = {
    "Name"                             = "${var.ClusterBaseName}-PrivateSubnet"
    "kubernetes.io/role/internal-elb" = "1"
  }

  tags = {
    "Environment" = "${var.ClusterBaseName}-lab"
  }
}

eks.tf

• 직접 eks resource를 작성하지 않고 terraform에서 제공하는 eks module 사용
• aws_caller_identity내용 data 블록에 저장
• var.tf에 작성한 변수를 참조
  • ClusterBaseName, KubernetesVersion, WorkerNodeInstanceType, WorkerNodeCount, WorkerNodeVolumesize, KeyName

data "aws_caller_identity" "current" {}

resource "aws_iam_policy" "external_dns_policy" {
  name        = "${var.ClusterBaseName}ExternalDNSPolicy"
  description = "Policy for allowing ExternalDNS to modify Route 53 records"

  policy = jsonencode({
    "Version": "2012-10-17",
    "Statement": [
      {
        "Effect": "Allow",
        "Action": [
          "route53:ChangeResourceRecordSets"
        ],
        "Resource": [
          "arn:aws:route53:::hostedzone/*"
        ]
      },
      {
        "Effect": "Allow",
        "Action": [
          "route53:ListHostedZones",
          "route53:ListResourceRecordSets"
        ],
        "Resource": [
          "*"
        ]
      }
    ]
  })
}

resource "aws_iam_role_policy_attachment" "external_dns_policy_attach" {
  role       = "${var.ClusterBaseName}-node-group-eks-node-group"
  policy_arn = aws_iam_policy.external_dns_policy.arn

  depends_on = [module.eks]
}

resource "aws_security_group" "node_group_sg" {
  name        = "${var.ClusterBaseName}-node-group-sg"
  description = "Security group for EKS Node Group"
  vpc_id      = module.vpc.vpc_id

  tags = {
    Name = "${var.ClusterBaseName}-node-group-sg"
  }
}

module "eks" {
  source  = "terraform-aws-modules/eks/aws"
  version = "~>20.0"

  cluster_name = var.ClusterBaseName
  cluster_version = var.KubernetesVersion
  cluster_endpoint_private_access = false
  cluster_endpoint_public_access  = true

  cluster_addons = {
    coredns = {
      most_recent = true
    }
    kube-proxy = {
      most_recent = true
    }
    vpc-cni = {
      most_recent = true
    }
  }

  vpc_id = module.vpc.vpc_id
  enable_irsa = true
  subnet_ids = module.vpc.public_subnets

  eks_managed_node_groups = {
    default = {
      name             = "${var.ClusterBaseName}-node-group"
      use_name_prefix  = false
      instance_type    = var.WorkerNodeInstanceType
      desired_size     = var.WorkerNodeCount
      max_size         = var.WorkerNodeCount + 2
      min_size         = var.WorkerNodeCount - 1
      disk_size        = var.WorkerNodeVolumesize
      subnets          = module.vpc.public_subnets
      key_name         = var.KeyName
      vpc_security_group_ids = [aws_security_group.node_group_sg.id]
      iam_role_name    = "${var.ClusterBaseName}-node-group-eks-node-group"
      iam_role_use_name_prefix = false
      iam_role_additional_policies = {
        "${var.ClusterBaseName}ExternalDNSPolicy" = aws_iam_policy.external_dns_policy.arn
      }
   }
  }

  access_entries = {
    admin = {
      kubernetes_groups = []
      principal_arn     = "${data.aws_caller_identity.current.arn}"

      policy_associations = {
        myeks = {
          policy_arn = "arn:aws:eks::aws:cluster-access-policy/AmazonEKSClusterAdminPolicy"
          access_scope = {
            namespaces = []
            type       = "cluster"
          }
        }
      }
    }
  }

  tags = {
    Environment = "${var.ClusterBaseName}-lab"
    Terraform   = "true"
  }
}

var.tf

• vpc,tf와 eks.tf, irfs.tf에서 참조에서 사용 할 변수 설정

variable "KeyName" {
  description = "Name of an existing EC2 KeyPair to enable SSH access to the instances."
  type        = string
}

variable "ClusterBaseName" {
  description = "Base name of the cluster."
  type        = string
  default     = "myeks"
}

variable "KubernetesVersion" {
  description = "Kubernetes version for the EKS cluster."
  type        = string
  default     = "1.29"
}

variable "WorkerNodeInstanceType" {
  description = "EC2 instance type for the worker nodes."
  type        = string
  default     = "t3.medium"
}

variable "WorkerNodeCount" {
  description = "Number of worker nodes."
  type        = number
  default     = 3
}

variable "WorkerNodeVolumesize" {
  description = "Volume size for worker nodes (in GiB)."
  type        = number
  default     = 30
}

variable "TargetRegion" {
  description = "AWS region where the resources will be created."
  type        = string
  default     = "ap-northeast-2"
}

variable "availability_zones" {
  description = "List of availability zones."
  type        = list(string)
  default     = ["ap-northeast-2a", "ap-northeast-2b", "ap-northeast-2c"]
}

variable "VpcBlock" {
  description = "CIDR block for the VPC."
  type        = string
  default     = "192.168.0.0/16"
}

variable "public_subnet_blocks" {
  description = "List of CIDR blocks for the public subnets."
  type        = list(string)
  default     = ["192.168.1.0/24", "192.168.2.0/24", "192.168.3.0/24"]
}

variable "private_subnet_blocks" {
  description = "List of CIDR blocks for the private subnets."
  type        = list(string)
  default     = ["192.168.11.0/24", "192.168.12.0/24", "192.168.13.0/24"]
}

irsa.tf

locals {
  cluster_oidc_issuer_arn = "arn:aws:iam::${data.aws_caller_identity.current.account_id}:oidc-provider/${replace(module.eks.cluster_oidc_issuer_url, "https://", "")}"
}

module "eks-external-dns" {
  source = "DNXLabs/eks-external-dns/aws"
  version = "0.2.0"

  cluster_name                     = var.ClusterBaseName
  cluster_identity_oidc_issuer     = module.eks.cluster_oidc_issuer_url
  cluster_identity_oidc_issuer_arn = local.cluster_oidc_issuer_arn

  enabled = false

}

module "aws_load_balancer_controller_irsa_role" {
  source  = "terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks"
  version = "5.3.1"

  role_name = "${var.ClusterBaseName}-aws-load-balancer-controller"

  attach_load_balancer_controller_policy = true

  oidc_providers = {
    ex = {
      provider_arn               = module.eks.oidc_provider_arn
      namespace_service_accounts = ["kube-system:aws-load-balancer-controller"]
    }
  }
}

provider "kubernetes" {
  host                   = module.eks.cluster_endpoint
  token                  = data.aws_eks_cluster_auth.cluster.token
  cluster_ca_certificate = base64decode(module.eks.cluster_certificate_authority_data)
}

data "aws_eks_cluster" "cluster" {
  name = module.eks.cluster_name
  depends_on = [module.eks]
}

data "aws_eks_cluster_auth" "cluster" {
  name = module.eks.cluster_name
  depends_on = [module.eks]
}

resource "kubernetes_service_account" "aws_lb_controller" {
  metadata {
    name      = "aws-load-balancer-controller"
    namespace = "kube-system"
    annotations = {
      "eks.amazonaws.com/role-arn" = module.aws_load_balancer_controller_irsa_role.iam_role_arn
    }
  }
}

 

확인

# EKS 클러스터 인증 정보 업데이트
CLUSTER_NAME=myeks
aws eks update-kubeconfig --region ap-northeast-2 --name $CLUSTER_NAME

kubectl get node -v=6

kubectl get pod -A

 

 

두 번째 EKS 클러스터 배포

• 첫 번째 생성한 vpc.tf, eks.tf, var.tf, irsa.tf 파일 그대로 사용
• 입력 변수에 클러스터 명과 쿠버네티스 버전을 입력하여 다른 EKS 클러스터 생성

#위에서 설정한 vpc.tf, eks.tf, var.tf, irsa.tf를 새로운 디렉토리에 복사
mkdir 5
cd 5
cp ../4/*.tf .

terraform init

# apply시 입력변수를 사용하여 두 번째 클러스터 생성
terraform apply -auto-approve -var=ClusterBaseName=myeks2 -var=KubernetesVersion="1.28"

# EKS 클러스터 인증 정보 가져오기
CLUSTER_NAME2=myeks2
aws eks update-kubeconfig --region ap-northeast-2 --name $CLUSTER_NAME2 --kubeconfig ./myeks2config