Kubespray 분석 - 1(설치편)

mkdir k8s-kubespary
cd k8s-kubespary
 
# 파일 다운로드
curl https://raw.githubusercontent.com/gasida/vagrant-lab/refs/heads/main/k8s-kubespary/Vagrantfile > Vagrantfile
curl https://raw.githubusercontent.com/gasida/vagrant-lab/refs/heads/main/k8s-kubespary/init_cfg.sh > init_cfg.sh
 
# 실습 환경 배포
vagrant up
vagrant status

# user 확인
whoami
pwd
 
# Linux Kernel Requirements : 5.8+ 이상 권장
uname -a
Linux k8s-ctr 6.12.0-55.39.1.el10_0.aarch64 #1 SMP PREEMPT_DYNAMIC Wed Oct 15 11:18:23 EDT 2025 aarch64 GNU/Linux
 
# Python : 3.10 ~ 3.12 : (참고) bento/rockylinux-9 경우 3.9
which python  && python -V
which python3 && python3 -V
3.12.9
 
# pip , git 설치
dnf install -y python3-pip git
which pip  && pip -V
which pip3 && pip3 -V
pip 23.3.2 from /usr/lib/python3.12/site-packages/pip (python 3.12)
 
 
# /etc/hosts 확인
ip -br -c -4 addr
cat /etc/hosts
ping -c 1 k8s-ctr
 
# SSH 접속을 위한 설정
# -----------------
echo "root:qwe123" | chpasswd
 
cat << EOF >> /etc/ssh/sshd_config
PermitRootLogin yes
PasswordAuthentication yes
EOF
systemctl restart sshd
 
# Setting SSH Key
ssh-keygen -t rsa -N "" -f /root/.ssh/id_rsa
ls -l ~/.ssh
 
# ssh-copy-id
ssh-copy-id -o StrictHostKeyChecking=no root@192.168.10.10
root@192.168.10.10's password: qwe123
 
# ssh 접속 확인 : IP, hostname
cat /root/.ssh/authorized_keys
ssh root@192.168.10.10 hostname
ssh -o StrictHostKeyChecking=no root@k8s-ctr hostname
ssh root@k8s-ctr hostname
# -----------------
 
# Clone Kubespray Repository
git clone -b v2.29.1 https://github.com/kubernetes-sigs/kubespray.git /root/kubespray
cd /root/kubespray
 
# (옵션) IDE에서 VM SSH 접속(root/qwe123)해서 편집 창 열기
 
# 최상단 plybook 확인 -> 각각 import_playbook 확인
ls -l *.yml
-rw-r--r--. 1 root root  88 Jan 24 20:55 cluster.yml    # ansible.builtin.import_playbook: playbooks/cluster.yml
-rw-r--r--. 1 root root  30 Jan 24 20:55 _config.yml
-rw-r--r--. 1 root root 747 Jan 24 20:55 galaxy.yml
-rw-r--r--. 1 root root 105 Jan 24 20:55 recover-control-plane.yml
-rw-r--r--. 1 root root  85 Jan 24 20:55 remove-node.yml
-rw-r--r--. 1 root root  85 Jan 24 20:55 remove_node.yml
-rw-r--r--. 1 root root  85 Jan 24 20:55 reset.yml
-rw-r--r--. 1 root root  85 Jan 24 20:55 scale.yml      # ansible.builtin.import_playbook: playbooks/scale.yml
-rw-r--r--. 1 root root  93 Jan 24 20:55 upgrade-cluster.yml
-rw-r--r--. 1 root root  93 Jan 24 20:55 upgrade_cluster.yml
 
# 
tree -L 2
...
├── inventory
│   ├── local
│   └── sample
...
├── playbooks
│   ├── ansible_version.yml
│   ├── boilerplate.yml
│   ├── cluster.yml*
│   ├── facts.yml
│   ├── install_etcd.yml
│   ├── internal_facts.yml
│   ├── recover_control_plane.yml
│   ├── remove_node.yml
│   ├── reset.yml
│   ├── scale.yml
│   └── upgrade_cluster.yml
...
├── roles
│   ├── adduser
│   ├── bastion-ssh-config
│   ├── bootstrap-os
│   ├── bootstrap_os
│   ├── container-engine
│   ├── download
│   ├── dynamic_groups
│   ├── etcd
│   ├── etcdctl_etcdutl
│   ├── etcd_defaults
│   ├── helm-apps
│   ├── kubernetes
│   ├── kubernetes-apps
│   ├── kubespray-defaults
│   ├── kubespray_defaults
│   ├── network_facts
│   ├── network_plugin
│   ├── recover_control_plane
│   ├── remove-node
│   ├── remove_node
│   ├── reset
│   ├── system_packages
│   ├── upgrade
│   ├── validate_inventory
│   └── win_nodes
...
 
# Install Python Dependencies
cat requirements.txt
ansible==10.7.0
# Needed for community.crypto module
cryptography==46.0.3
# Needed for jinja2 json_query templating
jmespath==1.0.1
# Needed for ansible.utils.ipaddr
netaddr==1.3.0
 
pip3 install -r /root/kubespray/requirements.txt
Successfully installed MarkupSafe-3.0.3 ansible-10.7.0 ansible-core-2.17.14 cffi-2.0.0 cryptography-46.0.2 jinja2-3.1.6 jmespath-1.0.1 netaddr-1.3.0 pycparser-3.0 resolvelib-1.0.1
 
# ansible 버전 확인 : Ansible 2.17.3 이상
which ansible
ansible --version
ansible [core 2.17.14]
  config file = /root/kubespray/ansible.cfg
  ...
  python version = 3.12.9 (main, Aug 14 2025, 00:00:00) [GCC 14.2.1 20250110 (Red Hat 14.2.1-7)] (/usr/bin/python3)
  jinja version = 3.1.6
  libyaml = True
 
# pip list 확인
pip list
Package                   Version
------------------------- -----------
ansible                   10.7.0
ansible-core              2.17.14
...
Jinja2                    3.1.6
jmespath                  1.0.1
...
netaddr                   1.3.0
...
 
# 해당 폴더에서 ansible-playbook 실행 시 적용되는 ansible.cfg
cat ansible.cfg
[ssh_connection] # 통신 속도 및 안정성 최적화
pipelining=True  # SSH 세션을 여러 번 열지 않고 하나의 세션에서 여러 명령을 한꺼번에 실행
ssh_args = -o ControlMaster=auto -o ControlPersist=30m -o ConnectionAttempts=100 -o UserKnownHostsFile=/dev/null
## ControlMaster=auto -o ControlPersist=30m: 한 번 연결된 SSH 커넥션을 30분 동안 유지합니다. 매번 로그인할 필요가 없어 성능이 향상됩니다.
## ConnectionAttempts=100: 네트워크 불안정으로 연결 실패 시 100번까지 재시도합니다.
## UserKnownHostsFile=/dev/null: 접속 대상의 지문(fingerprint)을 저장하지 않아 관리가 편해집니다.
#control_path = ~/.ssh/ansible-%%r@%%h:%%p
 
[defaults]
# https://github.com/ansible/ansible/issues/56930 (to ignore group names with - and .)
force_valid_group_names = ignore  # Ansible은 원래 그룹 이름에 -나 . 사용을 제한하지만, 쿠버네티스 리소스 명칭 규칙상 이를 허용하도록 설정
host_key_checking=False # 새 서버 접속 시 "Are you sure you want to continue connecting?"이라는 확인 창이 뜨지 않게 합니다.
gathering = smart         # 대상 서버의 정보(Fact)를 한 번만 수집하고 /tmp에 JSON 파일로 저장합니다. (아래 설명 이어서)
fact_caching = jsonfile   # 재실행 시 서버 정보를 다시 수집하지 않아 시간이 단축됩니다. 86400(24시간) 동안 캐시를 유지합니다.
fact_caching_connection = /tmp
fact_caching_timeout = 86400
timeout = 300
stdout_callback = default
display_skipped_hosts = no
library = ./library
callbacks_enabled = profile_tasks # 각 Task가 실행되는 데 걸리는 시간을 표시해 줍니다. 어떤 단계에서 병목이 생기는지 확인할 때 매우 유용합니다.
roles_path = roles:$VIRTUAL_ENV/usr/local/share/kubespray/roles:$VIRTUAL_ENV/usr/local/share/ansible/roles:/usr/share/kubespray/roles
deprecation_warnings=False
inventory_ignore_extensions = ~, .orig, .bak, .ini, .cfg, .retry, .pyc, .pyo, .creds, .gpg # 백업용이나 임시 파일을 인벤토리로 인식하여 에러가 발생하는 것을 방지합니다.
 
[inventory]
ignore_patterns = artifacts, credentials # 배포 결과물(artifacts)이나 중요 정보(credentials) 폴더 내의 파일을 인벤토리 스캔 대상에서 제외합니다.
 
# (참고) Vagrantfile
cat Vagrantfile

# inventory 디렉터리 복사
cp -rfp /root/kubespray/inventory/sample /root/kubespray/inventory/mycluster
tree inventory/mycluster/
inventory/mycluster/
├── group_vars
│   ├── all
│   │   ├── all.yml
│   │   ├── aws.yml
│   │   ├── azure.yml
│   │   ├── containerd.yml
│   │   ├── coreos.yml
│   │   ├── cri-o.yml
│   │   ├── docker.yml
│   │   ├── etcd.yml
│   │   ├── gcp.yml
│   │   ├── hcloud.yml
│   │   ├── huaweicloud.yml
│   │   ├── oci.yml
│   │   ├── offline.yml
│   │   ├── openstack.yml
│   │   ├── upcloud.yml
│   │   └── vsphere.yml
│   └── k8s_cluster
│       ├── addons.yml
│       ├── k8s-cluster.yml
│       ├── k8s-net-calico.yml
│       ├── k8s-net-cilium.yml
│       ├── k8s-net-custom-cni.yml
│       ├── k8s-net-flannel.yml
│       ├── k8s-net-kube-ovn.yml
│       ├── k8s-net-kube-router.yml
│       ├── k8s-net-macvlan.yml
│       └── kube_control_plane.yml
└── inventory.ini
 
# inventory.ini 작성
cat << EOF > /root/kubespray/inventory/mycluster/inventory.ini
k8s-ctr ansible_host=192.168.10.10 ip=192.168.10.10
 
[kube_control_plane]
k8s-ctr
 
[etcd:children]
kube_control_plane
 
[kube_node]
k8s-ctr
EOF
cat /root/kubespray/inventory/mycluster/inventory.ini
 
 
# https://github.com/kubernetes-sigs/kubespray/blob/master/docs/ansible/vars.md
## <your-favorite-editor> inventory/mycluster/group_vars/all.yml # for every node, including etcd
grep "^[^#]" inventory/mycluster/group_vars/all/all.yml
---
bin_dir: /usr/local/bin
loadbalancer_apiserver_port: 6443
loadbalancer_apiserver_healthcheck_port: 8081
no_proxy_exclude_workers: false
kube_webhook_token_auth: false
kube_webhook_token_auth_url_skip_tls_verify: false
ntp_enabled: false
ntp_manage_config: false
ntp_servers:
  - "0.pool.ntp.org iburst"
  - "1.pool.ntp.org iburst"
  - "2.pool.ntp.org iburst"
  - "3.pool.ntp.org iburst"
unsafe_show_logs: false
allow_unsupported_distribution_setup: false
 
## <your-favorite-editor> inventory/mycluster/group_vars/k8s_cluster.yml # for every node in the cluster (not etcd when it's separate)
grep "^[^#]" inventory/mycluster/group_vars/k8s_cluster/k8s-cluster.yml
---
kube_config_dir: /etc/kubernetes
kube_script_dir: "{{ bin_dir }}/kubernetes-scripts"
kube_manifest_dir: "{{ kube_config_dir }}/manifests"
kube_cert_dir: "{{ kube_config_dir }}/ssl"
kube_token_dir: "{{ kube_config_dir }}/tokens"
kube_api_anonymous_auth: true
local_release_dir: "/tmp/releases"
retry_stagger: 5
kube_owner: kube
kube_cert_group: kube-cert
kube_log_level: 2
credentials_dir: "{{ inventory_dir }}/credentials"
kube_network_plugin: calico
kube_network_plugin_multus: false
kube_service_addresses: 10.233.0.0/18
kube_pods_subnet: 10.233.64.0/18
kube_network_node_prefix: 24
kube_service_addresses_ipv6: fd85:ee78:d8a6:8607::1000/116
kube_pods_subnet_ipv6: fd85:ee78:d8a6:8607::1:0000/112
kube_network_node_prefix_ipv6: 120
kube_apiserver_ip: "{{ kube_service_subnets.split(',') | first | ansible.utils.ipaddr('net') | ansible.utils.ipaddr(1) | ansible.utils.ipaddr('address') }}"
kube_apiserver_port: 6443  # (https)
kube_proxy_mode: ipvs
kube_proxy_strict_arp: false
kube_proxy_nodeport_addresses: >-
  {%- if kube_proxy_nodeport_addresses_cidr is defined -%}
  [{{ kube_proxy_nodeport_addresses_cidr }}]
  {%- else -%}
  []
  {%- endif -%}
kube_encrypt_secret_data: false
cluster_name: cluster.local
ndots: 2
dns_mode: coredns
enable_nodelocaldns: true
enable_nodelocaldns_secondary: false
nodelocaldns_ip: 169.254.25.10
nodelocaldns_health_port: 9254
nodelocaldns_second_health_port: 9256
nodelocaldns_bind_metrics_host_ip: false
nodelocaldns_secondary_skew_seconds: 5
enable_coredns_k8s_external: false
coredns_k8s_external_zone: k8s_external.local
enable_coredns_k8s_endpoint_pod_names: false
resolvconf_mode: host_resolvconf
deploy_netchecker: false
skydns_server: "{{ kube_service_subnets.split(',') | first | ansible.utils.ipaddr('net') | ansible.utils.ipaddr(3) | ansible.utils.ipaddr('address') }}"
skydns_server_secondary: "{{ kube_service_subnets.split(',') | first | ansible.utils.ipaddr('net') | ansible.utils.ipaddr(4) | ansible.utils.ipaddr('address') }}"
dns_domain: "{{ cluster_name }}"
container_manager: containerd
kata_containers_enabled: false
kubeadm_certificate_key: "{{ lookup('password', credentials_dir + '/kubeadm_certificate_key.creds length=64 chars=hexdigits') | lower }}"
k8s_image_pull_policy: IfNotPresent
kubernetes_audit: false
default_kubelet_config_dir: "{{ kube_config_dir }}/dynamic_kubelet_dir"
volume_cross_zone_attachment: false
persistent_volumes_enabled: false
event_ttl_duration: "1h0m0s"
auto_renew_certificates: false
# auto_renew_certificates_systemd_calendar: "Mon *-*-1,2,3,4,5,6,7 03:00:00"  # First Monday of each month
kubeadm_patches_dir: "{{ kube_config_dir }}/patches"
kubeadm_patches: []
remove_anonymous_access: false
 
# 테스트할 기능 관련 수정
sed -i 's|kube_network_plugin: calico|kube_network_plugin: flannel|g' inventory/mycluster/group_vars/k8s_cluster/k8s-cluster.yml
sed -i 's|kube_proxy_mode: ipvs|kube_proxy_mode: iptables|g' inventory/mycluster/group_vars/k8s_cluster/k8s-cluster.yml
sed -i 's|enable_nodelocaldns: true|enable_nodelocaldns: false|g' inventory/mycluster/group_vars/k8s_cluster/k8s-cluster.yml
sed -i 's|auto_renew_certificates: false|auto_renew_certificates: true|g' inventory/mycluster/group_vars/k8s_cluster/k8s-cluster.yml
sed -i 's|# auto_renew_certificates_systemd_calendar|auto_renew_certificates_systemd_calendar|g' inventory/mycluster/group_vars/k8s_cluster/k8s-cluster.yml
grep -iE 'kube_network_plugin:|kube_proxy_mode|enable_nodelocaldns:|^auto_renew_certificates' inventory/mycluster/group_vars/k8s_cluster/k8s-cluster.yml
 
 
## flannel 설정 수정  inventory/mycluster/group_vars/k8s_cluster/k8s-net-flannel.yml
cat inventory/mycluster/group_vars/k8s_cluster/k8s-net-flannel.yml
echo "flannel_interface: enp0s9" >> inventory/mycluster/group_vars/k8s_cluster/k8s-net-flannel.yml
grep "^[^#]" inventory/mycluster/group_vars/k8s_cluster/k8s-net-flannel.yml
 
 
## <your-favorite-editor> inventory/mycluster/group_vars/kube_control_plane.yml # for the control plane
cat inventory/mycluster/group_vars/k8s_cluster/kube_control_plane.yml 
# Reservation for control plane kubernetes components
# kube_memory_reserved: 512Mi
# kube_cpu_reserved: 200m
# kube_ephemeral_storage_reserved: 2Gi
# kube_pid_reserved: "1000"
 
# Reservation for control plane host system
# system_memory_reserved: 256Mi
# system_cpu_reserved: 250m
# system_ephemeral_storage_reserved: 2Gi
# system_pid_reserved: "1000"
 
## <your-favorite-editor> addons.yml
grep "^[^#]" inventory/mycluster/group_vars/k8s_cluster/addons.yml
---
helm_enabled: false
registry_enabled: false
metrics_server_enabled: false
local_path_provisioner_enabled: false
local_volume_provisioner_enabled: false
gateway_api_enabled: false
ingress_nginx_enabled: false
ingress_publish_status_address: ""
ingress_alb_enabled: false
cert_manager_enabled: false
metallb_enabled: false
metallb_speaker_enabled: "{{ metallb_enabled }}"
metallb_namespace: "metallb-system"
argocd_enabled: false
kube_vip_enabled: false
node_feature_discovery_enabled: false
 
# 테스트할 기능 관련 수정
sed -i 's|helm_enabled: false|helm_enabled: true|g' inventory/mycluster/group_vars/k8s_cluster/addons.yml
sed -i 's|metrics_server_enabled: false|metrics_server_enabled: true|g' inventory/mycluster/group_vars/k8s_cluster/addons.yml
sed -i 's|node_feature_discovery_enabled: false|node_feature_discovery_enabled: true|g' inventory/mycluster/group_vars/k8s_cluster/addons.yml
grep -iE 'helm_enabled:|metrics_server_enabled:|node_feature_discovery_enabled:' inventory/mycluster/group_vars/k8s_cluster/addons.yml
 
 
# etcd.yml : 파드가 아닌 systemd unit
grep "^[^#]" inventory/mycluster/group_vars/all/etcd.yml
---
etcd_data_dir: /var/lib/etcd
etcd_deployment_type: host
 
# containerd.yml 
cat inventory/mycluster/group_vars/all/containerd.yml
---
# Please see roles/container-engine/containerd/defaults/main.yml for more configuration options
 
# containerd_storage_dir: "/var/lib/containerd"
# containerd_state_dir: "/run/containerd"
# containerd_oom_score: 0
 
# containerd_default_runtime: "runc"
# containerd_snapshotter: "native"
 
# containerd_runc_runtime:
#   name: runc
#   type: "io.containerd.runc.v2"
#   engine: ""
...(생략)...
 
# 기본 환경 정보 출력 저장
ip addr | tee -a ip_addr-1.txt 
ss -tnlp | tee -a ss-1.txt
df -hT | tee -a df-1.txt
findmnt | tee -a findmnt-1.txt
sysctl -a | tee -a sysctl-1.txt
 
# 지원 버전 정보 확인
cat roles/kubespray_defaults/vars/main/checksums.yml | grep -i kube -A40
 
# 배포: 아래처럼 반드시 ~/kubespray 디렉토리에서 ansible-playbook 를 실행하자!
# Deploy Kubespray with Ansible Playbook - run the playbook as root
# The option `--become` is required, as for example writing SSL keys in /etc/,
# installing packages and interacting with various systemd daemons.
ansible-playbook -i inventory/mycluster/inventory.ini -v cluster.yml -e kube_version="1.33.3" --list-tasks # 배포 전, Task 목록 확인
ANSIBLE_FORCE_COLOR=true ansible-playbook -i inventory/mycluster/inventory.ini -v cluster.yml -e kube_version="1.33.3" | tee kubespray_install.log
 
# 설치 확인 : /root/.kube/config
more kubespray_install.log
kubectl get node -v=6
cat /root/.kube/config
 
# k8s
kubectl get node -owide
NAME      STATUS   ROLES           AGE    VERSION   INTERNAL-IP     EXTERNAL-IP   OS-IMAGE                        KERNEL-VERSION                  CONTAINER-RUNTIME
k8s-ctr   Ready    control-plane   113s   v1.33.3   192.168.10.10   <none>        Rocky Linux 10.0 (Red Quartz)   6.12.0-55.39.1.el10_0.aarch64   containerd://2.1.5
 
kubectl get pod -A
...
 
# 기본 환경 정보 출력 저장
ip addr | tee -a ip_addr-2.txt 
ss -tnlp | tee -a ss-2.txt
df -hT | tee -a df-2.txt
findmnt | tee -a findmnt-2.txt
sysctl -a | tee -a sysctl-2.txt
 
# 파일 출력 비교 : 빠져나오기 ':q' -> ':q' => 변경된 부분이 어떤 동작과 역할인지 조사해보기! , ctrl + f / b
vi -d ip_addr-1.txt ip_addr-2.txt
vi -d ss-1.txt ss-2.txt
vi -d df-1.txt df-2.txt
vi -d findmnt-1.txt findmnt-2.txt
vi -d sysctl-1.txt sysctl-2.txt

# Source the completion
source <(kubectl completion bash)
source <(kubeadm completion bash)
 
# Alias kubectl to k
alias k=kubectl
complete -o default -F __start_kubectl k
 
# k9s 설치 : https://github.com/derailed/k9s
CLI_ARCH=amd64
if [ "$(uname -m)" = "aarch64" ]; then CLI_ARCH=arm64; fi
wget https://github.com/derailed/k9s/releases/latest/download/k9s_linux_${CLI_ARCH}.tar.gz
tar -xzf k9s_linux_*.tar.gz
ls -al k9s
chown root:root k9s
mv k9s /usr/local/bin/
chmod +x /usr/local/bin/k9s
k9s