홈서버 구축기 2 - Kubernetes 클러스터 구축과 애플리케이션 배포

ip link show eno1
# 2: eno1: <BROADCAST,MULTICAST,UP,LOWER_UP> ...
#    link/ether xx:xx:xx:xx:xx:xx brd ff:ff:ff:ff:ff:ff

sudo apt update
sudo apt install -y containerd
sudo mkdir -p /etc/containerd
containerd config default | sudo tee /etc/containerd/config.toml
# 생성된 파일에서 SystemdCgroup = true 로 수정
sudo systemctl restart containerd

sudo modprobe br_netfilter
echo "br_netfilter" | sudo tee /etc/modules-load.d/k8s.conf

cat <<EOF | sudo tee /etc/sysctl.d/k8s.conf
net.bridge.bridge-nf-call-iptables  = 1
net.bridge.bridge-nf-call-ip6tables = 1
net.ipv4.ip_forward                 = 1
EOF
sudo sysctl --system

sudo swapoff -a
sudo sed -i '/ swap / s/^/#/' /etc/fstab

sudo apt install -y apt-transport-https ca-certificates curl gpg
curl -fsSL https://pkgs.k8s.io/core:/stable:/v1.34/deb/Release.key | \
  sudo gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg
echo 'deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/v1.34/deb/ /' | \
  sudo tee /etc/apt/sources.list.d/kubernetes.list
sudo apt update
sudo apt install -y kubelet kubeadm kubectl
sudo apt-mark hold kubelet kubeadm kubectl

sudo kubeadm init \
  --apiserver-advertise-address=192.168.0.100 \
  --pod-network-cidr=10.244.0.0/16 \
  --service-cidr=10.96.0.0/12

mkdir -p $HOME/.kube
sudo cp /etc/kubernetes/admin.conf $HOME/.kube/config
sudo chown $(id -u):$(id -g) $HOME/.kube/config

kubectl taint nodes --all node-role.kubernetes.io/control-plane-

kubectl create -f https://raw.githubusercontent.com/projectcalico/calico/v3.31.3/manifests/tigera-operator.yaml

apiVersion: operator.tigera.io/v1
kind: Installation
metadata:
  name: default
spec:
  calicoNetwork:
    linuxDataplane: Iptables
    nodeAddressAutodetectionV4:
      interface: eth.*|ens.*|eno.*|enp.*
    ipPools:
      - name: default-ipv4-ippool
        blockSize: 26
        cidr: 10.244.0.0/16
        encapsulation: VXLANCrossSubnet
        natOutgoing: Enabled
        nodeSelector: all()

---
apiVersion: operator.tigera.io/v1
kind: APIServer
metadata:
  name: default
spec: {}
---
apiVersion: operator.tigera.io/v1
kind: Goldmane
metadata:
  name: default
---
apiVersion: operator.tigera.io/v1
kind: Whisker
metadata:
  name: default

apiVersion: projectcalico.org/v3
kind: GlobalNetworkPolicy
metadata:
  name: allow-internet-egress
spec:
  order: 10
  selector: all()
  types:
  - Egress
  egress:
  # 내부 통신 허용 (Pod 및 Service 대역)
  - action: Allow
    destination:
      nets:
      - 10.244.0.0/16
      - 10.96.0.0/12
  # 외부 인터넷 통신 허용 (Pod 대역 제외)
  - action: Allow
    destination:
      notNets:
      - 10.244.0.0/16
  # DNS 조회를 위한 UDP 53 포트 허용
  - action: Allow
    protocol: UDP
    destination:
      ports: [53]

calicoNetwork:
  linuxDataplane: BPF
  bpfNetworkBootstrap: Enabled
  kubeProxyManagement: Enabled
  # ...

chain nat-cali-nat-outgoing {
    meta mark & 0x03f00000 == 0x03800000 counter packets 0 bytes 0 masquerade
}

# Installation을 Iptables 데이터플레인으로 되돌리고 적용
kubectl apply -f custom-resources-iptables.yaml

# BPF 모드에서 비활성화했던 kube-proxy를 다시 활성화
kubectl patch ds -n kube-system kube-proxy --type=json \
  -p='[{"op":"remove","path":"/spec/template/spec/nodeSelector/non-calico"},
       {"op":"remove","path":"/spec/template/spec/nodeSelector/operator.tigera.io~1disable-kube-proxy"}]'

# BPF 부트스트랩용 ConfigMap 제거
kubectl delete cm -n tigera-operator kubernetes-service-endpoints

kubectl apply -f https://github.com/cert-manager/cert-manager/releases/latest/download/cert-manager.yaml

apiVersion: cert-manager.io/v1
kind: ClusterIssuer
metadata:
  name: letsencrypt-prod
spec:
  acme:
    server: https://acme-v02.api.letsencrypt.org/directory
    email: you@example.com
    privateKeySecretRef:
      name: letsencrypt-prod
    solvers:
    - dns01:
        cloudflare:
          email: you@example.com
          apiTokenSecretRef:
            name: cloudflare-api-token-secret
            key: api-token

kubectl create secret generic cloudflare-api-token-secret \
  --from-literal=api-token=YOUR_CLOUDFLARE_API_TOKEN \
  -n cert-manager

apiVersion: cert-manager.io/v1
kind: Certificate
metadata:
  name: movingzero-wildcard-cert
  namespace: nginx-gateway
spec:
  secretName: movingzero-wildcard-secret
  issuerRef:
    name: letsencrypt-prod
    kind: ClusterIssuer
  privateKey:
    rotationPolicy: Always
  dnsNames:
  - "movingzero.org"
  - "*.movingzero.org"

# Gateway API 표준 CRD
kubectl kustomize "https://github.com/nginx/nginx-gateway-fabric/config/crd/gateway-api/standard?ref=v2.2.0" | kubectl apply -f -

# NGINX Gateway Fabric 컨트롤러
helm install ngf oci://ghcr.io/nginx/charts/nginx-gateway-fabric \
  --create-namespace -n nginx-gateway

apiVersion: gateway.networking.k8s.io/v1
kind: Gateway
metadata:
  name: edge-gw
  namespace: nginx-gateway
spec:
  gatewayClassName: nginx
  listeners:
  - name: https
    protocol: HTTPS
    port: 443
    hostname: "*.movingzero.org"
    tls:
      mode: Terminate
      certificateRefs:
      - name: movingzero-wildcard-secret
    allowedRoutes:
      namespaces:
        from: All

apiVersion: gateway.networking.k8s.io/v1
kind: HTTPRoute
metadata:
  name: argocd-http
  namespace: argocd
spec:
  parentRefs:
  - name: edge-gw
    namespace: nginx-gateway
  hostnames:
  - argocd.movingzero.org
  rules:
  - matches:
    - path:
        type: PathPrefix
        value: /
    backendRefs:
    - name: argocd-server
      port: 80

day-to-us-backend/
├── secret.yaml                 # 시크릿 (Git 미추적)
├── base/
│   ├── kustomization.yaml      # 리소스 목록과 네임스페이스
│   ├── deployment.yaml         # 백엔드 Deployment
│   ├── service.yaml            # 백엔드 Service
│   ├── httproute.yaml          # API 라우팅 규칙
│   └── db.yaml                 # PostgreSQL PVC + Service + Deployment
└── overlays/
    └── prod/
        └── kustomization.yaml  # 프로덕션 오버레이

day-to-us-frontend/
├── base/
│   ├── kustomization.yaml
│   ├── deployment.yaml
│   ├── service.yaml
│   └── httproute.yaml
└── overlays/
    └── prod/
        └── kustomization.yaml

apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization

resources:
  - deployment.yaml
  - service.yaml
  - httproute.yaml
  - db.yaml

namespace: day-to-us

apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: postgres-pvc
  namespace: day-to-us
spec:
  accessModes:
    - ReadWriteOnce
  storageClassName: local-path
  resources:
    requests:
      storage: 5Gi
---
apiVersion: v1
kind: Service
metadata:
  name: db
  namespace: day-to-us
spec:
  selector:
    app: day-to-us-db
  ports:
    - port: 5432
      targetPort: 5432
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: day-to-us-db
  namespace: day-to-us
spec:
  replicas: 1
  selector:
    matchLabels:
      app: day-to-us-db
  template:
    metadata:
      labels:
        app: day-to-us-db
    spec:
      containers:
      - name: postgres
        image: postgres:16-alpine
        env:
        - name: POSTGRES_DB
          value: daytous
        - name: POSTGRES_USER
          valueFrom:
            secretKeyRef:
              name: day-to-us-secret
              key: POSTGRES_USER
        - name: POSTGRES_PASSWORD
          valueFrom:
            secretKeyRef:
              name: day-to-us-secret
              key: POSTGRES_PASSWORD
        volumeMounts:
        - name: postgres-storage
          mountPath: /var/lib/postgresql/data
        resources:
          requests:
            memory: "512Mi"
            cpu: "200m"
          limits:
            memory: "1024Mi"
            cpu: "500m"
        livenessProbe:
          tcpSocket:
            port: 5432
          initialDelaySeconds: 60
          periodSeconds: 10
          timeoutSeconds: 5
          successThreshold: 1
          failureThreshold: 3
        readinessProbe:
          tcpSocket:
            port: 5432
          initialDelaySeconds: 30
          periodSeconds: 10
          timeoutSeconds: 5
          successThreshold: 1
          failureThreshold: 3
      volumes:
      - name: postgres-storage
        persistentVolumeClaim:
          claimName: postgres-pvc

apiVersion: apps/v1
kind: Deployment
metadata:
  name: day-to-us-backend
  namespace: day-to-us
spec:
  replicas: 1
  selector:
    matchLabels:
      app: day-to-us-backend
  template:
    metadata:
      labels:
        app: day-to-us-backend
    spec:
      imagePullSecrets:
      - name: regcred
      containers:
      - name: backend
        image: dong5854/day-to-us-backend:latest
        ports:
        - containerPort: 8080
        env:
        - name: SPRING_PROFILES_ACTIVE
          value: "prod"
        - name: PROD_DB_URL
          value: "jdbc:postgresql://db:5432/daytous"
        - name: PROD_DB_USERNAME
          valueFrom: { secretKeyRef: { name: day-to-us-secret, key: POSTGRES_USER } }
        - name: PROD_DB_PASSWORD
          valueFrom: { secretKeyRef: { name: day-to-us-secret, key: POSTGRES_PASSWORD } }
        - name: SERVER_FORWARD_HEADERS_STRATEGY
          value: "native"
        - name: SERVER_SERVLET_SESSION_COOKIE_SAMESITE
          value: "none"
        - name: SERVER_SERVLET_SESSION_COOKIE_SECURE
          value: "true"
        # OAuth JWT 등 추가 환경 변수는 Secret에서 valueFrom으로 가져옵니다
        resources:
          requests:
            memory: "256Mi"
            cpu: "200m"
          limits:
            memory: "512Mi"
            cpu: "400m"

apiVersion: v1
kind: Service
metadata:
  name: day-to-us-backend-svc
  namespace: day-to-us
spec:
  selector:
    app: day-to-us-backend
  ports:
    - port: 8080
      targetPort: 8080

apiVersion: gateway.networking.k8s.io/v1
kind: HTTPRoute
metadata:
  name: day-to-us-route
  namespace: day-to-us
spec:
  parentRefs:
  - name: edge-gw
    namespace: nginx-gateway
  hostnames:
  - "day-to-us-api.movingzero.org"
  rules:
  - matches:
    - path:
        type: PathPrefix
        value: /
    backendRefs:
    - name: day-to-us-backend-svc
      port: 8080

apiVersion: apps/v1
kind: Deployment
metadata:
  name: day-to-us-frontend
  namespace: day-to-us
  labels:
    app: day-to-us-frontend
spec:
  replicas: 1
  selector:
    matchLabels:
      app: day-to-us-frontend
  template:
    metadata:
      labels:
        app: day-to-us-frontend
    spec:
      imagePullSecrets:
      - name: regcred
      containers:
        - name: frontend
          image: dong5854/day-to-us-frontend:latest
          ports:
            - containerPort: 80
          imagePullPolicy: Always
          resources:
            requests:
              memory: "128Mi"
              cpu: "100m"
            limits:
              memory: "256Mi"
              cpu: "200m"
          livenessProbe:
            httpGet:
              path: /
              port: 80
            initialDelaySeconds: 10
            periodSeconds: 10
          readinessProbe:
            httpGet:
              path: /
              port: 80
            initialDelaySeconds: 5
            periodSeconds: 10

apiVersion: v1
kind: Service
metadata:
  name: day-to-us-frontend-svc
  namespace: day-to-us
spec:
  selector:
    app: day-to-us-frontend
  ports:
    - protocol: TCP
      port: 80
      targetPort: 80
  type: ClusterIP

apiVersion: gateway.networking.k8s.io/v1
kind: HTTPRoute
metadata:
  name: day-to-us-frontend-route
  namespace: day-to-us
spec:
  parentRefs:
  - name: edge-gw
    namespace: nginx-gateway
  hostnames:
  - "daytous.movingzero.org"
  rules:
  - matches:
    - path:
        type: PathPrefix
        value: /
    backendRefs:
    - name: day-to-us-frontend-svc
      port: 80