# rancher + openstack
#### 오픈스택 랜처용 프로젝트 및 리소스 생성
```bash
openstack project create --domain admin_domain --description "RKE1 Cluster" rke1
openstack role add --project bd5fd0d84eb044369f6f422ca9ff8d13 --user admin admin
export OS_PROJECT_ID=bd5fd0d84eb044369f6f422ca9ff8d13
export OS_PROJECT_NAME=rke1
openstack network create \
--project bd5fd0d84eb044369f6f422ca9ff8d13 \
--internal \
rke1
openstack subnet create --network rke1 --dns-nameserver 8.8.8.8 \
--project bd5fd0d84eb044369f6f422ca9ff8d13 \
--subnet-range 172.31.0.0/24 \
--allocation-pool start=172.31.0.10,end=172.31.0.249 \
rke1-subnet
openstack router create \
rke1-router \
--project bd5fd0d84eb044369f6f422ca9ff8d13
openstack router add subnet rke1-router rke1-subnet
openstack router set rke1-router --external-gateway ext_net
openstack security group create rke1 --project bd5fd0d84eb044369f6f422ca9ff8d13
openstack security group rule create --protocol icmp --dst-port 1:65535 rke1
openstack security group rule create --protocol tcp --dst-port 22:22 rke1
openstack security group rule create --protocol tcp --dst-port 53:53 rke1
openstack security group rule create --protocol tcp --dst-port 179:179 rke1
openstack security group rule create --protocol tcp --dst-port 6443:6443 rke1
openstack security group rule create --protocol tcp --dst-port 2380:2380 rke1
openstack security group rule create --protocol tcp --dst-port 7080:7080 rke1
openstack security group rule create --protocol tcp --dst-port 8472:8472 rke1
openstack security group rule create --protocol tcp --dst-port 8080:8080 rke1
openstack security group rule create --protocol tcp --dst-port 9100:9100 rke1
openstack security group rule create --protocol tcp --dst-port 10250:10250 rke1
openstack security group rule create --protocol udp --dst-port 8472:8472 rke1
openstack security group rule create --protocol tcp --dst-port 30000:32767 rke1
```
#### 랜처 노드(Openstack VM) 추가
```bash
cat <
./install_docker.sh
#!/bin/bash
curl https://releases.rancher.com/install-docker/20.10.sh | sh
sudo usermod -aG docker ubuntu
EOF
openstack server create --flavor 8c-32g-200disk \
--image="focal-amd64" \
--network rke1 \
--key-name user1 \
--security-group rke1 \
--user-data ./install_docker.sh \
--max 3 \
rke1
openstack server create --flavor m1.medium \
--image="focal-amd64" \
--network rke1 \
--key-name user1 \
--security-group rke1 \
rke1-launcher2
openstack server add floating ip rke1-launcher2 192.168.3.69
scp -i ~/cloud-keys/user1-key ~/cloud-keys/user1-key ubuntu@192.168.3.69:/home/ubuntu/.ssh/id_rsa
```
#### rke 명령어 및 콘솔 서버 설정
```bash
# launcher2 에서 패키지 설치
curl -OL https://github.com/rancher/rke/releases/download/v1.6.2/rke_linux-amd64
chmod +x rke_linux-amd64 && sudo mv rke_linux-amd64 /usr/local/bin/rke
curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl"
sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl
```
### **OpenStack Cloud Controller Manager**
```bash
openstack application credential create --description "Kubernetes" kubernetes
# 여기서 나오는 id, secret 저장
```
- cloud.conf
```bash
[Global]
verify=false
auth-url=https://192.168.2.125:5000/v3
application-credential-id=85036858110f4d5aa9a391d251896a34
application-credential-secret=Gv0su4n6s6sxWNAxtZK2bFvoJ1shTLUpjLiAjc3YK-HN-gZmfX6rHWEWEb8g98774gboE84P0Ow1CiOb8YifwQ
tls-insecure=true
domain-name=admin_domain
[LoadBalancer]
use-octavia=true
floating-network-id=a3594572-5a10-4e1c-8165-7707a432a0b4
subnet-id=372affb3-bcea-429e-9fb0-15c86af786e5 (user1net)
```
```bash
kubectl create secret -n kube-system generic cloud-config --from-file=cloud.conf
kubectl apply -f https://raw.githubusercontent.com/kubernetes/cloud-provider-openstack/master/manifests/controller-manager/cloud-controller-manager-roles.yaml
kubectl apply -f https://raw.githubusercontent.com/kubernetes/cloud-provider-openstack/master/manifests/controller-manager/cloud-controller-manager-role-bindings.yaml
kubectl apply -f https://raw.githubusercontent.com/kubernetes/cloud-provider-openstack/master/manifests/controller-manager/openstack-cloud-controller-manager-ds.yaml
```
- Cinder Storage Class
```bash
kubectl apply -f - << EOF
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
name: csi-sc-cinder
annotations:
storageclass.kubernetes.io/is-default-class: "true"
provisioner: cinder.csi.openstack.org
parameters:
availability: nova
type: multi_attach #멀티 볼륨 타입은 지정 필요
allowVolumeExpansion: true
volumeBindingMode: Immediate
EOF
```
- install_ccm.sh (cloud controller manager 설치 스크립트)
```bash
#!/bin/bash
if [ $# -eq 0 ]; then
echo "Usage: $0 "
exit 1
fi
KUBECONFIG_PATH=$1
kubectl create secret -n kube-system generic cloud-config --from-file=cloud.conf --kubeconfig=${KUBECONFIG_PATH}
kubectl apply -f ccm/cloud-controller-manager-roles.yaml --kubeconfig=${KUBECONFIG_PATH}
kubectl apply -f ccm/cloud-controller-manager-role-bindings.yaml --kubeconfig=${KUBECONFIG_PATH}
kubectl apply -f ccm/openstack-cloud-controller-manager-ds.yaml --kubeconfig=${KUBECONFIG_PATH}
kubectl apply -f cloud-provider-openstack/manifests/cinder-csi-plugin/ --kubeconfig=${KUBECONFIG_PATH}
kubectl apply -f csi-sc-cinder.yaml --kubeconfig=${KUBECONFIG_PATH}
echo "Success!"
```
- ccm/openstack-cloud-controller-manager-ds.yaml (wget으로 가져와서 수정한 것)
```bash
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: cloud-controller-manager
namespace: kube-system
---
apiVersion: apps/v1
kind: DaemonSet
metadata:
name: openstack-cloud-controller-manager
namespace: kube-system
labels:
k8s-app: openstack-cloud-controller-manager
spec:
selector:
matchLabels:
k8s-app: openstack-cloud-controller-manager
updateStrategy:
type: RollingUpdate
template:
metadata:
labels:
k8s-app: openstack-cloud-controller-manager
spec:
nodeSelector:
node-role.kubernetes.io/controlplane: "true"
securityContext:
runAsUser: 1001
tolerations:
- key: node.cloudprovider.kubernetes.io/uninitialized
value: "true"
effect: NoSchedule
- key: node-role.kubernetes.io/master
effect: NoSchedule
- key: node-role.kubernetes.io/controlplane
effect: NoSchedule
value: 'true'
- key: node-role.kubernetes.io/etcd
effect: NoExecute
value: 'true'
serviceAccountName: cloud-controller-manager
containers:
- name: openstack-cloud-controller-manager
image: registry.k8s.io/provider-os/openstack-cloud-controller-manager:v1.31.0
args:
- /bin/openstack-cloud-controller-manager
- --v=1
- --cluster-name=$(CLUSTER_NAME)
- --cloud-config=$(CLOUD_CONFIG)
- --cloud-provider=openstack
- --use-service-account-credentials=false
- --bind-address=127.0.0.1
volumeMounts:
- mountPath: /etc/kubernetes/pki
name: k8s-certs
readOnly: true
- mountPath: /etc/ssl/certs
name: ca-certs
readOnly: true
- mountPath: /etc/config
name: cloud-config-volume
readOnly: true
resources:
requests:
cpu: 200m
env:
- name: CLOUD_CONFIG
value: /etc/config/cloud.conf
- name: CLUSTER_NAME
value: kubernetes
dnsPolicy: ClusterFirst
hostNetwork: true
volumes:
- hostPath:
path: /etc/kubernetes/pki
type: DirectoryOrCreate
name: k8s-certs
- hostPath:
path: /etc/ssl/certs
type: DirectoryOrCreate
name: ca-certs
- name: cloud-config-volume
secret:
secretName: cloud-config
```
- sudo vim /etc/systemd/resolved.conf (dns 문제)
```bash
# 오픈스택 subnet에 dns 설정 안 해주면 NS 서버 설정이 안 되어 있음
[Resolve]
DNS=8.8.8.8
sudo systemctl restart systemd-resolved
```
- cloud.conf 변경시
```bash
k delete secret cloud-config -n kube-system
k create secret -n kube-system generic cloud-config --from-file=cloud.conf
k delete po -l k8s-app=openstack-cloud-controller-manager -n kube-system
# 관련 클러스터 롤도 다시 배포해야함
```
- ETC (오픈스택에서만의 문제) https://hkpark130.p-e.kr:8100/posts/76
```bash
kubectl edit cm -n kube-system canal-config
kubectl delete pod -n kube-system -l k8s-app=canal
kubectl edit felixconfiguration default
노드 들어가서
sudo tee /etc/docker/daemon.json > /dev/null << EOF
{
"mtu": 1442
}
EOF
sudo systemctl daemon-reload
sudo systemctl restart docker
kubectl delete validatingwebhookconfigurations rancher.cattle.io
kubectl delete -n cattle-system MutatingWebhookConfiguration rancher.cattle.io
kubectl delete validatingwebhookconfigurations rancher.cattle.io
```
---
### 실제 대시보드
