Python项目容器化实践(九) - 将lyanna应用部署到线上的Kubernetes上运行啦！

# 先安装Docker
➜  ~ sudo apt install -y apt-transport-https ca-certificates curl software-properties-common
➜  ~ curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
➜  ~ sudo apt-key fingerprint 0EBFCD88
# 由于我在服务器安装了Python3.7，/usr/bin/python3链接不再不再是原来系统自带的Python3.5，需要明确指定
➜  ~  sudo sed -i 's/python3/python3.5/g' /usr/bin/add-apt-repository
➜  ~ sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable"
➜  ~ sudo apt-get update
➜  ~ sudo apt install docker-ce
➜  ~ sudo systemctl enable docker
# 安装k8s用到的三个包
➜  ~ cat <<EOF | sudo tee -a /etc/apt/sources.list.d/kubernetes.list
deb http://mirrors.ustc.edu.cn/kubernetes/apt kubernetes-xenial main
EOF
➜  ~ sudo apt-get update
W: GPG error: http://mirrors.ustc.edu.cn/kubernetes/apt kubernetes-xenial InRelease: The following signatures couldn't be verified because the public key is not available: NO_PUBKEY 6A030B21BA07F4FB
W: The repository 'http://mirrors.ustc.edu.cn/kubernetes/apt kubernetes-xenial InRelease' is not signed.
N: Data from such a repository can't be authenticated and is therefore potentially dangerous to use.
N: See apt-secure(8) manpage for repository creation and user configuration details.
# 由于没有对应的公钥所以不能验证这个源包的签名，我这里给大家看墙内对应解决方案，否则可以用官网提供的名字添加
# 上面的输出有个 6A030B21BA07F4FB 下面2句都要用到它:
➜  ~ gpg --keyserver keyserver.ubuntu.com --recv-keys 6A030B21BA07F4FB
➜  ~ gpg --export --armor 6A030B21BA07F4FB | sudo apt-key add -
OK
➜  ~ sudo apt-get update  # 这次就正常了
➜  ~ sudo apt-get install -y kubelet kubeadm kubectl
➜  ~ sudo apt-mark hold kubelet kubeadm kubectl  # 停止它们的自动更新
➜  ~ kubelet --version
Kubernetes v1.16.1

# 下面这部分每个服务器都要执行:
➜  ~ sudo swapoff -a  # 从v1.8开始要求关闭系统的Swap，但永久关闭需要编辑/etc/fstab注释掉交换分区所在行
➜  ~ cat <<EOF | sudo tee -a /etc/docker/daemon.json  # 统一cgroup驱动为systemd, 并且使用国内源
pipe heredoc> {
  "exec-opts": ["native.cgroupdriver=systemd"],
  "log-driver": "json-file",
  "log-opts": {
    "max-size": "100m"
  },
  "storage-driver": "overlay2",
  "registry-mirrors": [
    "https://registry.docker-cn.com",
    "http://hub-mirror.c.163.com",
    "http://docker.mirrors.ustc.edu.cn",
    "http://dockerhub.azk8s.cn"
  ]
}
pipe heredoc> EOF
➜  ~ sudo systemctl daemon-reload
➜  ~ sudo systemctl restart docker

# 下面这部分只在Master执行:
# 初始化，使用阿里云源，指定Pod网络地址(10.244.0.0/16网段) ,要下载image，需要花费几分钟
➜  ~ sudo kubeadm init --image-repository registry.aliyuncs.com/google_containers -v 5 --pod-network-cidr 10.244.0.0/16
...  #  省略输出
Your Kubernetes control-plane has initialized successfully!

To start using your cluster, you need to run the following as a regular user:

  mkdir -p $HOME/.kube
  sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
  sudo chown $(id -u):$(id -g) $HOME/.kube/config

You should now deploy a pod network to the cluster.
Run "kubectl apply -f [podnetwork].yaml" with one of the options listed at:
  https://kubernetes.io/docs/concepts/cluster-administration/addons/

Then you can join any number of worker nodes by running the following on each as root:

kubeadm join 172.17.87.14:6443 --token r4aa9m.n3xtkspn48x4j0s4 \
    --discovery-token-ca-cert-hash sha256:047fbe9327aae75856433d95d298b97ca6de2f3efe253cfed618369448e49ff5
➜  ~ mkdir -p $HOME/.kube
➜  ~ sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
➜  ~ sudo chown $(id -u):$(id -g) $HOME/.kube/config

➜  ~ kubectl apply -f https://docs.projectcalico.org/v3.10/manifests/calico.yaml
# 等待1-2分钟，查看kube-system命名空间下的pod状态都是Running的了:
➜  ~ kubectl get pod -n kube-system
NAME                             READY   STATUS    RESTARTS   AGE
calico-kube-controllers-6b64bcd855-mf2nw   1/1     Running   0          2m18s
calico-node-fttbw                          1/1     Running   0          2m18s
coredns-58cc8c89f4-px2kz                   1/1     Running   0          3m17s
coredns-58cc8c89f4-xhxsr                   1/1     Running   0          3m17s
etcd-master                                1/1     Running   0          2m35s
kube-apiserver-master                      1/1     Running   0          2m33s
kube-controller-manager-master             1/1     Running   0          2m16s
kube-flannel-ds-amd64-jmbpf                1/1     Running   0          2m57s
kube-proxy-7hl5l                           1/1     Running   0          3m17s
kube-scheduler-master                      1/1     Running   0          2m28s

# 我这个博客应用没有其他Node节点，所以会主节点上安排Pod，但出于安全原因考虑需要主节点隔离:
➜  ~ kubectl taint nodes --all node-role.kubernetes.io/master-
node/vm-17-14-ubuntu untainted

# 按之前的kubeadm init输出提示，每个节点执行join加入集群
➜  ~ sudo kubeadm join 172.17.87.14:6443 --token r4aa9m.n3xtkspn48x4j0s4 --discovery-token-ca-cert-hash sha256:047fbe9327aae75856433d95d298b97ca6de2f3efe253cfed618369448e49ff5

# 等待几分钟后，就可以查看集群状态了:
➜  ~ kubectl get nodes
NAME     STATUS     ROLES    AGE     VERSION
master   Ready      master   9m      v1.16.1
...

➜  ~ kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/master/deploy/static/mandatory.yaml
➜  ~ kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/master/deploy/static/provider/baremetal/service-nodeport.yaml

# 稍等1分钟:
➜  ~ kubectl get pods --all-namespaces -l app.kubernetes.io/name=ingress-nginx
NAMESPACE       NAME                                        READY   STATUS    RESTARTS   AGE
ingress-nginx   nginx-ingress-controller-568867bf56-7b4qb   1/1     Running   0          87s

➜  ~ kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/master/deploy/static/provider/cloud-generic.yaml

➜  ~ kubectl apply -f k8s
➜  ~ kubectl get pods -l app.kubernetes.io/name=lyanna
NAME                                 READY   STATUS    RESTARTS   AGE
lyanna-deployment-579f79ff47-q84gm   1/1     Running   0          5m31s
lyanna-deployment-579f79ff47-rfvgv   1/1     Running   0          5m31s
lyanna-deployment-579f79ff47-w7dgh   1/1     Running   0          5m31s
➜  ~ kubectl exec -it lyanna-deployment-579f79ff47-q84gm -- sh -c ./setup.sh  # 在其中一个Pod上执行

resources:
  limits:
    cpu: 300m  # 1m等于千分之一个CPU
    memory: 200Mi
  requests:
    cpu: 200m
    memory: 100Mi

➜  ~ kubectl exec nginx-ingress-controller-568867bf56-7b4qb -n ingress-nginx cat /etc/nginx/nginx.conf|less  # 生成的nginx.conf，可以测试配置是否生效
# 本地试验缓存效果:
❯ curl --head http://lyanna.local/static/css/index.min.css
...
X-Cache-Status: MISS

❯ curl --head http://lyanna.local/static/css/index.min.css
...
X-Cache-Status: HIT

❯ curl --head http://lyanna.local/static/css/index.min.css
...
X-Cache-Status: HIT

# 172.17.17.14是主节点IP，172.17.17.13是MySQL服务IP
➜  ~ sudo iptables -t nat -A OUTPUT -o lo -d 172.17.17.14 -p tcp --dport 3306 -j DNAT  --to-destination 172.17.17.13:3306
➜  ~ sudo iptables -t nat -A POSTROUTING -d 172.17.17.13 -p tcp --dport 3306 -j SNAT --to 172.17.17.14
# 我的服务器是Ubuntu，所以用下面方法让策略生效
➜  ~ sudo sh -c "iptables-save > /etc/iptables/rules.v4"
➜  ~ sudo sh -c "iptables-restore < /etc/iptables/rules.v4"

➜  ~ kubectl get svc -n ingress-nginx
NAME            TYPE       CLUSTER-IP      EXTERNAL-IP   PORT(S)                      AGE
ingress-nginx   NodePort   10.105.221.35   <none>        80:31255/TCP,443:30786/TCP   1h16m
➜  ~ ping 10.105.221.35  # ping不同
PING 10.105.221.35 (10.105.221.35) 56(84) bytes of data.
^C
--- 10.105.221.35 ping statistics ---
3 packets transmitted, 0 received, 100% packet loss, time 2011ms
➜  ~ curl 10.105.221.35  # 但是curl可以
<html>
<head><title>404 Not Found</title></head>
<body>
<center><h1>404 Not Found</h1></center>
<hr><center>openresty/1.15.8.2</center>
</body>
</html>
➜  ~ curl localhost:31255  # 使用Node的对应随机端口也可以
<html>
<head><title>404 Not Found</title></head>
<body>
<center><h1>404 Not Found</h1></center>
<hr><center>openresty/1.15.8.2</center>
</body>
</html>

# 新建/etc/nginx/sites-enabled/lyanna.conf
server {
    server_name blog.dongwm.com;

    location / {
        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
        proxy_set_header X-Forwarded-Proto $scheme;
        proxy_set_header Host $host;
        proxy_set_header X-Real-IP $remote_addr;
        proxy_set_header X-Scheme $scheme;
        proxy_set_header X-NginX-Proxy true;
        proxy_redirect off;
        proxy_pass http://10.105.221.35;
        proxy_http_version 1.1;
    }
}

➜  ~ sudo certbot -d blog.dongwm.com
...  # 省略输出
Congratulations! You have successfully enabled https://blog.dongwm.com
...