Ubuntu 18.04 下部署k8s

一 、更新Ubuntu源

mv /etc/apt/sources.list /etc/apt/sources.list.bakcat  /etc/apt/sources.list.bak |grep -v "#" |grep -v "^#34; >sources.listsed -i s/archive.ubuntu.com/mirrors.ustc.edu.cn/g /etc/apt/sources.listsed -i s/security.ubuntu.com/mirrors.ustc.edu.cn/g /etc/apt/sources.listapt -y update && apt -y upgrade# 2
、timedatectlsed -i s/en_US/C/g /etc/default/localetimedatectl set-timezone Asia/Shanghai# 3

、bash-completionsed -i 97,99s/#//g /root/.bashrc# 4
、sshecho "PermitRootLogin yes" >>/etc/ssh/sshd_configpasswd root << "EOF"passwordpasswordEOFsystemctl reload ssh# 5
	、hostsvim /etc/hosts10.0.0.20 k8s-master0010.0.0.21 k8s-master0110.0.0.22 k8s-master0210.0.0.23 k8s-node0110.0.0.24 k8s-node0210.0.0.25 k8s-bl-master# 6
、ssh-keygenssh-keygen -t rsafor i in `cat /root/*.txt`;do echo $i;ssh-copy-id -i .ssh/id_rsa.pub $i;done# 7

、swapswapoff -ased -i '/swap/s/^(.*)$/#1/g' /etc/fstab# 8、networknet=`cat /etc/netplan/00-installer-config.yaml |awk 'NR==4{ print $1}'`sed -i "s/${ net}/eth0:/g" /etc/netplan/00-installer-config.yamlsed -i '11s/""/"net.ifnames=0 biosdevname=0"/g' /etc/default/grubupdate-grubreboot

二 
、安裝ipvs

apt -y install ipvsadm ipset sysstat conntrack libseccomp2 libseccomp-devcat >/etc/modules-load.d/ipvs.conf << EOFip_vsip_vs_lcip_vs_wlcip_vs_rrip_vs_wrrip_vs_lblcip_vs_lblcrip_vs_dhip_vs_ship_vs_foip_vs_nqip_vs_sedip_vs_ftpnf_conntrackip_tablesip_setxt_setipt_setipt_rpfilteript_REJECTipipEOFsystemctl restart systemd-modules-load.servicelsmod |grep -e ip_vs -e nf_conntrack_ipv4

三 、下載安裝containerd

wget https://github.com/containerd/containerd/releases/download/v1.6.1/cri-containerd-cni-1.6.1-linux-amd64.tar.gztar --no-overwrite-dir -C / -xzf cri-containerd-cni-1.6.1-linux-amd64.tar.gzsystemctl daemon-reloadsystemctl enable --now containerd修改 config.tomlcontainerd config default >/etc/containerd/config.toml---sed -i "s#k8s.gcr.io#registry.aliyuncs.com/google_containers#g" /etc/containerd/config.tomlsed -i "s#SystemdCgroup = false#SystemdCgroup = true#g" /etc/containerd/config.tomlsed -i '153a        [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"]' /etc/containerd/config.toml  # 8個(gè)空格 # endpoint 10個(gè)空格sed -i '154a          endpoint = ["https://registry.aliyuncs.com"]' /etc/containerd/config.toml修改crictl.yamlmv /etc/crictl.yaml /etc/crictl.yaml.bakcat >/etc/crictl.yaml << "EOF"runtime-endpoint: unix:///run/containerd/containerd.sockimage-endpoint: unix:///run/containerd/containerd.socktimeout: 0debug: falsepull-image-on-create: falsedisable-pull-on-run: falseEOF

四、安裝nginx 做四層代理

apt -y install nginxcp /etc/nginx/nginx.conf /etc/nginx/nginx.conf.bakvim /etc/nginx/nginx.conf---......stream {                   log_format main '$remote_addr $upstream_addr - [$time_local] $status $upstream_bytes_sent';                  access_log /var/log/nginx/k8s-access.log main;                upstream k8s-apiserver {                 server 10.0.0.20:6443;                server 10.0.0.21:6443;                server 10.0.0.22:6443;        }            server {                 listen 6444;                 proxy_pass k8s-apiserver;        }}http { 		log_format main '$remote_addr - $remote_user [$time_local] "$request" '                        '$status $body_bytes_sent "$http_referer" '                        '"$http_user_agent" "$http_x_forwarded_for"';        ...        ...}---systemctl enable --now nginx.servicesystemctl status nginx.service

五
、安裝keepalive 做高可用

apt -y install keepalived#keepalived configcat >/etc/keepalived/keepalived.conf << "EOF"global_defs {     notification_email {       acassen@firewall.loc      failover@firewall.loc      sysadmin@firewall.loc    }    notification_email_from Alexandre.Cassen@firewall.loc    smtp_server 127.0.0.1    smtp_connect_timeout 30     router_id NGINX_MASTER}vrrp_script check_nginx {   script "/etc/keepalived/check_nginx.sh"  interval 5  weight -1  fall 2  rise 1}vrrp_instance VI_1 {     state MASTER    interface eth0 # 修改為實(shí)際網(wǎng)卡名    virtual_router_id 51 # VRRP 路由 ID 實(shí)例，每個(gè)實(shí)例是唯一的    priority 100 # 優(yōu)先級(jí)，備服務(wù)器設(shè)置 90    advert_int 1 # 指定 VRRP 心跳包通告間隔時(shí)間，默認(rèn) 1 秒    authentication {         auth_type PASS        auth_pass K8SHA_KA_AUTH    }    # 虛擬 IP    virtual_ipaddress {         10.0.0.25/24    }    track_script {         check_nginx    }}EOF#health configcat >/etc/keepalived/check_nginx.sh << "EOF"#!/bin/bash count=$(ps -ef |grep nginx | grep sbin | egrep -cv "grep|$") if [ "$count" -eq 0 ];then   systemctl stop keepalived fiEOF---systemctl enable --now keepalived.servicesystemctl status keepalived.service

六 、master端部署cfssl 、etcd、ca certificate、etcd certificate

6.1、下載cfssl

wget https://github.com/cloudflare/cfssl/releases/download/v1.6.1/cfssl_1.6.1_linux_amd64 -O /usr/local/bin/cfsslwget https://github.com/cloudflare/cfssl/releases/download/v1.6.1/cfssljson_1.6.1_linux_amd64 -O /usr/local/bin/cfssljsonwget https://github.com/cloudflare/cfssl/releases/download/v1.6.1/cfssl-certinfo_1.6.1_linux_amd64 -O /usr/local/bin/cfssl-certinfochmod +x /usr/local/bin/cfssl*chown -Rf root:root /usr/local/bin/cfssl*

6.2、etcd目錄規(guī)劃

# all Master# 1、etcd-sslmkdir -p /etc/etcd/ssl/# 2、etcd-WorkingDirectorymkdir -p /var/lib/etcd/default.etcd# 3、kubernetes-sslmkdir -p /etc/kubernetes/ssl# 4、kubernetes-logmkdir -p /var/log/kubernetes

6.3 、ca 證書生成

mkdir -p ~/workcd ~/work/---cat >ca-csr.json << "EOF"{   "CN": "kubernetes",  "key": {     "algo": "rsa",    "size": 2048  },  "names": [    {       "C": "CN",      "ST": "Shanghai",      "L": "Shanghai",      "O": "k8s",      "OU": "system"    }  ]}EOF---cat >ca-config.json << "EOF"{   "signing": {     "default": {       "expiry": "87600h"    },    "profiles": {       "kubernetes": {         "usages": [            "signing",            "key encipherment",            "server auth",            "client auth"        ],        "expiry": "87600h"      }    }  }}EOF---cfssl gencert -initca ca-csr.json | cfssljson -bare cacp ca*.pem /etc/etcd/ssl/---# send to other masterfor i in `cat ~/MasterNodes.txt`;do echo $i;scp /etc/etcd/ssl/ca*.pem $i:/etc/etcd/ssl;done

6.4 配置etcd證書

cat >etcd-csr.json << "EOF"{   "CN": "etcd",  "hosts": [    "127.0.0.1",    "10.0.0.20",    "10.0.0.21",    "10.0.0.22",    "10.0.0.25"  ],  "key": {     "algo": "rsa",    "size": 2048  },  "names": [    {       "C": "CN",      "ST": "Shanghai",      "L": "Shanghai",      "O": "k8s",      "OU": "system"    }  ]}EOF---cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=kubernetes etcd-csr.json | cfssljson -bare etcdcp etcd*.pem /etc/etcd/ssl/---# send to otherfor i in `cat ~/MasterNodes.txt`;do echo $i;scp /etc/etcd/ssl/etcd*.pem $i:/etc/etcd/ssl;done

6.5、下載及配置etcd

# download etcdwget https://github.com/etcd-io/etcd/releases/download/v3.5.0/etcd-v3.5.0-linux-amd64.tar.gz# tar etcd-*.tar.gztar -xf etcd-v3.5.0-linux-amd64.tar.gz --strip-components=1 -C ~/work/ etcd-v3.5.0-linux-amd64/etcd{ ,ctl}chown -Rf root:root etcd*cp -arp etcd* /usr/local/bin/# send to otherfor i in `cat ~/MasterNodes.txt`;do echo $i;scp /usr/local/bin/etcd{ ,ctl} $i:/usr/local/bin/;donecat >/etc/etcd/etcd.conf << "EOF"ETCD_NAME='etcd1'ETCD_DATA_DIR="/var/lib/etcd/default.etcd"ETCD_LISTEN_PEER_URLS="https://10.0.0.20:2380" # change ipETCD_LISTEN_CLIENT_URLS="https://10.0.0.20:2379,http://127.0.0.1:2379" # change ipETCD_INITIAL_ADVERTISE_PEER_URLS="https://10.0.0.20:2380" # change ipETCD_ADVERTISE_CLIENT_URLS="https://10.0.0.20:2379" # change ipETCD_INITIAL_CLUSTER="etcd1=https://10.0.0.20:2380,etcd2=https://10.0.0.21:2380,etcd3=https://10.0.0.22:2380"ETCD_INITIAL_CLUSTER_TOKEN="etcd-cluster"ETCD_INITIAL_CLUSTER_STATE="new"EOF

6.6  、添加etcd systemd啟動(dòng)

cat >/usr/lib/systemd/system/etcd.service << "EOF"[Unit]Description=Etcd ServiceAfter=network.targetAfter=network-online.targetWants=network-online.target[Service]Type=notifyEnvironmentFile=-/etc/etcd/etcd.confWorkingDirectory=/var/lib/etcd/ExecStart=/usr/local/bin/etcd  --cert-file=/etc/etcd/ssl/etcd.pem  --key-file=/etc/etcd/ssl/etcd-key.pem  --trusted-ca-file=/etc/etcd/ssl/ca.pem  --peer-cert-file=/etc/etcd/ssl/etcd.pem  --peer-key-file=/etc/etcd/ssl/etcd-key.pem  --peer-trusted-ca-file=/etc/etcd/ssl/ca.pem  --peer-client-cert-auth  --client-cert-authRestart=on-failureRestartSec=10LimitNOFILE=65536[Install]WantedBy=multi-user.targetEOF---# send to otherfor i in `cat ~/MasterNodes.txt`;do echo $i;scp /usr/lib/systemd/system/etcd.service $i:/usr/lib/systemd/system/;done

啟動(dòng)etcd

# 1	、start etcdsystemctl daemon-reloadsystemctl enable --now etcd.servicesystemctl status etcd.service# 2、check etcdETCDCTL_API=3etcdctl --endpoints=https://10.0.0.20:2379,https://10.0.0.21:2379,https://10.0.0.22:2379 --write-out=table --cacert=/etc/etcd/ssl/ca.pem --cert=/etc/etcd/ssl/etcd.pem --key=/etc/etcd/ssl/etcd-key.pem endpoint health+----------------------------+--------+-------------+-------+|          ENDPOINT          | HEALTH |    TOOK     | ERROR |+----------------------------+--------+-------------+-------+| https://10.0.0.20:2379     |   true | 16.188005ms |       || https://10.0.0.21:2379     |   true | 16.693314ms |       || https://10.0.0.22:2379     |   true | 16.089367ms |       |+----------------------------+--------+-------------+-------+

七
、安裝 k8s-master

# 1、downloadwget https://dl.k8s.io/v1.23.5/kubernetes-server-linux-amd64.tar.gz# 2

、tartar -xf kubernetes-server-linux-amd64.tar.gz --strip-components=3 -C ~/work kubernetes/server/bin/kube{ let,ctl,-apiserver,-controller-manager,-scheduler,-proxy}scp kube{ ctl,-apiserver,-controller-manager,-scheduler} /usr/local/bin/# 3、kube{ let,ctl,-apiserver,-controller-manager,-scheduler,-proxy}for i in `cat ~/MasterNodes.txt`;do echo $i;scp ~/work/kube{ ctl,-apiserver,-controller-manager,-scheduler} $i:/usr/local/bin/;done# 4
、kube{ let,-proxy}for i in `cat ~/WorkNodes.txt`;do echo $i;scp ~/work/kube{ let,-proxy} $i:/usr/local/bin/;done# 5、send pemcp /etc/etcd/ssl/ca*.pem /etc/kubernetes/ssl/for i in `cat ~/WorkNodes.txt`;do echo $i;scp /etc/etcd/ssl/ca*.pem $i:/etc/kubernetes/ssl/;done

# 添加kube-apiserver  tokencat  >/etc/kubernetes/token.csv <

7.2、添加kube-apiserver 證書

cat >kube-apiserver-csr.json << "EOF"{   "CN": "kubernetes",  "hosts": [    "127.0.0.1",    "10.0.0.20",    "10.0.0.21",    "10.0.0.22",    "10.0.0.23",    "10.0.0.24",    "10.0.0.25",    "10.96.0.1",    "kubernetes",    "kubernetes.default",    "kubernetes.default.svc",    "kubernetes.default.svc.cluster",    "kubernetes.default.svc.cluster.local"  ],  "key": {     "algo": "rsa",    "size": 2048  },  "names": [    {       "C": "CN",      "ST": "Shanghai",      "L": "Shanghai",      "O": "k8s",      "OU": "system"    }  ]}EOF---cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=kubernetes kube-apiserver-csr.json | cfssljson -bare kube-apiservercp kube-apiserver*.pem /etc/kubernetes/ssl/for i in `cat ~/MasterNodes.txt`;do echo $i;scp ~/work/kube-apiserver*.pem $i:/etc/kubernetes/ssl/;done

7.3 、天kube-apiserver 配置文件

# change --bind-address= and --advertise-address=---cat >/etc/kubernetes/kube-apiserver.conf << "EOF"KUBE_APISERVER_OPTS="--enable-admission-plugins=NamespaceLifecycle,NodeRestriction,LimitRanger,ServiceAccount,DefaultStorageClass,ResourceQuota  --anonymous-auth=false  --bind-address=10.0.0.20  --secure-port=6443  --advertise-address=10.0.0.20  --insecure-port=0  --authorization-mode=Node,RBAC  --runtime-config=api/all=true  --enable-bootstrap-token-auth  --service-cluster-ip-range=10.96.0.0/16  --token-auth-file=/etc/kubernetes/token.csv  --service-node-port-range=30000-50000  --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem  --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem  --client-ca-file=/etc/kubernetes/ssl/ca.pem  --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem  --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem  --service-account-key-file=/etc/kubernetes/ssl/ca-key.pem  --service-account-signing-key-file=/etc/kubernetes/ssl/ca-key.pem  --service-account-issuer=https://kubernetes.default.svc.cluster.local  --etcd-cafile=/etc/etcd/ssl/ca.pem  --etcd-certfile=/etc/etcd/ssl/etcd.pem  --etcd-keyfile=/etc/etcd/ssl/etcd-key.pem  --etcd-servers=https://10.0.0.20:2379,https://10.0.0.21:2379,https://10.0.0.22:2379  --enable-swagger-ui=true  --allow-privileged=true  --apiserver-count=3  --audit-log-maxage=30  --audit-log-maxbackup=3  --audit-log-maxsize=100  --audit-log-path=/var/log/kube-apiserver-audit.log  --event-ttl=1h  --alsologtostderr=true  --logtostderr=false  --log-dir=/var/log/kubernetes  --v=4"EOF

7.4 、添加kube-apiserver systemd啟動(dòng)

cat >/usr/lib/systemd/system/kube-apiserver.service << "EOF"[Unit]Description=Kubernetes API ServerDocumentation=https://github.com/kubernetes/kubernetesAfter=etcd.serviceWants=etcd.service[Service]EnvironmentFile=-/etc/kubernetes/kube-apiserver.confExecStart=/usr/local/bin/kube-apiserver $KUBE_APISERVER_OPTSRestart=on-failureRestartSec=5Type=notifyLimitNOFILE=65536[Install]WantedBy=multi-user.targetEOF---for i in `cat ~/MasterNodes.txt`;do echo $i;scp /usr/lib/systemd/system/kube-apiserver.service $i:/usr/lib/systemd/system/;done

啟動(dòng)kube-apiserver

systemctl daemon-reloadsystemctl enable --now kube-apiserver.servicesystemctl status kube-apiserver.service---# checkcurl --insecure https://10.0.0.20:6443---{   "kind": "Status",  "apiVersion": "v1",  "metadata": { },  "status": "Failure",  "message": "Unauthorized",  "reason": "Unauthorized",  "code": 401

7.5