二进制部署Kubernetes v1.13.4 HA可选

kernel:unregister_netdevice: waiting for lo to become free. Usage count = 1

[ ! -f /usr/bin/perl ] && yum install perl -y

grubby --default-kernel

sudo apt-get install -y wget git conntrack ipvsadm ipset jq sysstat curl iptables libseccomp

curl https://raw.githubusercontent.com/docker/docker/master/contrib/check-config.sh > check-config.sh
bash ./check-config.sh

yum install -y epel-release bash-completion && cp /usr/share/bash-completion/completions/docker /etc/bash_completion.d/
systemctl enable --now docker

# 声明集群成员信息
declare -A MasterArray otherMaster NodeArray AllNode Other
MasterArray=(['k8s-m1']=10.0.6.166 ['k8s-m2']=10.0.6.167 ['k8s-m3']=10.0.6.168)
otherMaster=(['k8s-m2']=10.0.6.167 ['k8s-m3']=10.0.6.168)
NodeArray=(['k8s-n1']=10.0.6.169 ['k8s-n2']=10.0.6.170)
# 下面复制上面的信息粘贴即可
AllNode=(['k8s-m1']=10.0.6.166 ['k8s-m2']=10.0.6.167 ['k8s-m3']=10.0.6.168 ['k8s-n1']=10.0.6.169 ['k8s-n2']=10.0.6.170)
Other=(['k8s-m2']=10.0.6.167 ['k8s-m3']=10.0.6.168 ['k8s-n1']=10.0.6.169 ['k8s-n2']=10.0.6.170)

export         VIP=10.0.6.155

export INGRESS_VIP=10.0.6.156
[ "${#MasterArray[@]}" -eq 1 ]  && export VIP=${MasterArray[@]} || export API_PORT=8443
export KUBE_APISERVER=https://${VIP}:${API_PORT:-6443}

#声明需要安装的的k8s版本
export KUBE_VERSION=v1.13.4

# 网卡名
export interface=eth0

# cni
export CNI_URL="https://github.com/containernetworking/plugins/releases/download"
export CNI_VERSION=v0.7.4
# etcd
export ETCD_version=v3.3.12

yum install sshpass -y
alias ssh='sshpass -p zhangguanzhang ssh -o StrictHostKeyChecking=no'
alias scp='sshpass -p zhangguanzhang scp -o StrictHostKeyChecking=no'

git clone https://github.com/zhangguanzhang/k8s-manual-files.git ~/k8s-manual-files -b v1.13.4
cd ~/k8s-manual-files/

$ curl -s https://zhangguanzhang.github.io/bash/pull.sh | bash -s search gcr.io/google_containers/kube-apiserver-amd64/ | 
    grep -P 'v[\d.]+$' | sort -t '.' -n -k 2

curl  https://storage.googleapis.com/kubernetes-release/release/${KUBE_VERSION}/kubernetes-server-linux-amd64.tar.gz > kubernetes-server-linux-amd64.tar.gz
tar -zxvf kubernetes-server-linux-amd64.tar.gz  --strip-components=3 -C /usr/local/bin kubernetes/server/bin/kube{let,ctl,-apiserver,-controller-manager,-scheduler,-proxy}

for NODE in "${!otherMaster[@]}"; do
    echo "--- $NODE ${otherMaster[$NODE]} ---"
    scp /usr/local/bin/kube{let,ctl,-apiserver,-controller-manager,-scheduler,-proxy} ${otherMaster[$NODE]}:/usr/local/bin/ 
done

for NODE in "${!NodeArray[@]}"; do
    echo "--- $NODE ${NodeArray[$NODE]} ---"
    scp /usr/local/bin/kube{let,-proxy} ${NodeArray[$NODE]}:/usr/local/bin/ 
done

mkdir -p /opt/cni/bin
wget  "${CNI_URL}/${CNI_VERSION}/cni-plugins-amd64-${CNI_VERSION}.tgz" 
tar -zxf cni-plugins-amd64-${CNI_VERSION}.tgz -C /opt/cni/bin

# 分发cni文件
for NODE in "${!Other[@]}"; do
    echo "--- $NODE ${Other[$NODE]} ---"
    ssh ${Other[$NODE]} 'mkdir -p /opt/cni/bin'
    scp /opt/cni/bin/* ${Other[$NODE]}:/opt/cni/bin/
done

mkdir -p /etc/kubernetes/pki/etcd
sed -i "/IP.2/a IP.3 = $VIP" ~/k8s-manual-files/pki/openssl.cnf
sed -ri '/IP.3/r '<( paste -d '' <(seq -f 'IP.%g = ' 4 $[${#AllNode[@]}+3])  <(xargs -n1<<<${AllNode[@]} | sort) ) ~/k8s-manual-files/pki/openssl.cnf
sed -ri '$r '<( paste -d '' <(seq -f 'IP.%g = ' 2 $[${#MasterArray[@]}+1])  <(xargs -n1<<<${MasterArray[@]} | sort) ) ~/k8s-manual-files/pki/openssl.cnf
cp ~/k8s-manual-files/pki/openssl.cnf /etc/kubernetes/pki/
cd /etc/kubernetes/pki

openssl genrsa -out ca.key 2048
openssl req -x509 -new -nodes -key ca.key -config openssl.cnf -subj "/CN=kubernetes-ca" -extensions v3_ca -out ca.crt -days 10000

openssl genrsa -out etcd/ca.key 2048
openssl req -x509 -new -nodes -key etcd/ca.key -config openssl.cnf -subj "/CN=etcd-ca" -extensions v3_ca -out etcd/ca.crt -days 10000

openssl genrsa -out front-proxy-ca.key 2048
openssl req -x509 -new -nodes -key front-proxy-ca.key -config openssl.cnf -subj "/CN=kubernetes-ca" -extensions v3_ca -out front-proxy-ca.crt -days 10000

openssl genrsa -out apiserver-etcd-client.key 2048
openssl req -new -key apiserver-etcd-client.key -subj "/CN=apiserver-etcd-client/O=system:masters" -out apiserver-etcd-client.csr
openssl x509 -in apiserver-etcd-client.csr -req -CA etcd/ca.crt -CAkey etcd/ca.key -CAcreateserial -extensions v3_req_etcd -extfile openssl.cnf -out apiserver-etcd-client.crt -days 10000

openssl genrsa -out etcd/server.key 2048
openssl req -new -key etcd/server.key -subj "/CN=etcd-server" -out etcd/server.csr
openssl x509 -in etcd/server.csr -req -CA etcd/ca.crt -CAkey etcd/ca.key -CAcreateserial -extensions v3_req_etcd -extfile openssl.cnf -out etcd/server.crt -days 10000

openssl genrsa -out etcd/peer.key 2048
openssl req -new -key etcd/peer.key -subj "/CN=etcd-peer" -out etcd/peer.csr
openssl x509 -in etcd/peer.csr -req -CA etcd/ca.crt -CAkey etcd/ca.key -CAcreateserial -extensions v3_req_etcd -extfile openssl.cnf -out etcd/peer.crt -days 10000

openssl genrsa -out etcd/healthcheck-client.key 2048
openssl req -new -key etcd/healthcheck-client.key -subj "/CN=etcd-client" -out etcd/healthcheck-client.csr
openssl x509 -in etcd/healthcheck-client.csr -req -CA etcd/ca.crt -CAkey etcd/ca.key -CAcreateserial -extensions v3_req_etcd -extfile openssl.cnf -out etcd/healthcheck-client.crt -days 10000

openssl genrsa -out apiserver.key 2048
openssl req -new -key apiserver.key -subj "/CN=kube-apiserver" -config openssl.cnf -out apiserver.csr
openssl x509 -req -in apiserver.csr -CA ca.crt -CAkey ca.key -CAcreateserial -days 10000 -extensions v3_req_apiserver -extfile openssl.cnf -out apiserver.crt

openssl genrsa -out  apiserver-kubelet-client.key 2048
openssl req -new -key apiserver-kubelet-client.key -subj "/CN=apiserver-kubelet-client/O=system:masters" -out apiserver-kubelet-client.csr
openssl x509 -req -in apiserver-kubelet-client.csr -CA ca.crt -CAkey ca.key -CAcreateserial -days 10000 -extensions v3_req_client -extfile openssl.cnf -out apiserver-kubelet-client.crt

openssl genrsa -out  front-proxy-client.key 2048
openssl req -new -key front-proxy-client.key -subj "/CN=front-proxy-client" -out front-proxy-client.csr
openssl x509 -req -in front-proxy-client.csr -CA front-proxy-ca.crt -CAkey front-proxy-ca.key -CAcreateserial -days 10000 -extensions v3_req_client -extfile openssl.cnf -out front-proxy-client.crt

openssl genrsa -out  kube-scheduler.key 2048
openssl req -new -key kube-scheduler.key -subj "/CN=system:kube-scheduler" -out kube-scheduler.csr
openssl x509 -req -in kube-scheduler.csr -CA ca.crt -CAkey ca.key -CAcreateserial -days 10000 -extensions v3_req_client -extfile openssl.cnf -out kube-scheduler.crt

openssl genrsa -out  sa.key 2048
openssl ecparam -name secp521r1 -genkey -noout -out sa.key
openssl ec -in sa.key -outform PEM -pubout -out sa.pub
openssl req -new -sha256 -key sa.key -subj "/CN=system:kube-controller-manager" -out sa.csr
openssl x509 -req -in sa.csr -CA ca.crt -CAkey ca.key -CAcreateserial -days 10000 -extensions v3_req_client -extfile openssl.cnf -out sa.crt

openssl genrsa -out  admin.key 2048
openssl req -new -key admin.key -subj "/CN=kubernetes-admin/O=system:masters" -out admin.csr
openssl x509 -req -in admin.csr -CA ca.crt -CAkey ca.key -CAcreateserial -days 10000 -extensions v3_req_client -extfile openssl.cnf -out admin.crt

find . -name "*.csr" -o -name "*.srl"|xargs  rm -f

CLUSTER_NAME="kubernetes"
KUBE_USER="system:kube-controller-manager"
KUBE_CERT="sa"
KUBE_CONFIG="controller-manager.kubeconfig"

# 设置集群参数
kubectl config set-cluster ${CLUSTER_NAME} \
  --certificate-authority=/etc/kubernetes/pki/ca.crt \
  --embed-certs=true \
  --server=${KUBE_APISERVER} \
  --kubeconfig=/etc/kubernetes/${KUBE_CONFIG}

# 设置客户端认证参数
kubectl config set-credentials ${KUBE_USER} \
  --client-certificate=/etc/kubernetes/pki/${KUBE_CERT}.crt \
  --client-key=/etc/kubernetes/pki/${KUBE_CERT}.key \
  --embed-certs=true \
  --kubeconfig=/etc/kubernetes/${KUBE_CONFIG}

# 设置上下文参数
kubectl config set-context ${KUBE_USER}@${CLUSTER_NAME} \
  --cluster=${CLUSTER_NAME} \
  --user=${KUBE_USER} \
  --kubeconfig=/etc/kubernetes/${KUBE_CONFIG}

# 设置当前使用的上下文
kubectl config use-context ${KUBE_USER}@${CLUSTER_NAME} --kubeconfig=/etc/kubernetes/${KUBE_CONFIG}

# 查看生成的配置文件
kubectl config view --kubeconfig=/etc/kubernetes/${KUBE_CONFIG}

CLUSTER_NAME="kubernetes"
KUBE_USER="system:kube-scheduler"
KUBE_CERT="kube-scheduler"
KUBE_CONFIG="scheduler.kubeconfig"

# 设置集群参数
kubectl config set-cluster ${CLUSTER_NAME} \
  --certificate-authority=/etc/kubernetes/pki/ca.crt \
  --embed-certs=true \
  --server=${KUBE_APISERVER} \
  --kubeconfig=/etc/kubernetes/${KUBE_CONFIG}

# 设置客户端认证参数
kubectl config set-credentials ${KUBE_USER} \
  --client-certificate=/etc/kubernetes/pki/${KUBE_CERT}.crt \
  --client-key=/etc/kubernetes/pki/${KUBE_CERT}.key \
  --embed-certs=true \
  --kubeconfig=/etc/kubernetes/${KUBE_CONFIG}

# 设置上下文参数
kubectl config set-context ${KUBE_USER}@${CLUSTER_NAME} \
  --cluster=${CLUSTER_NAME} \
  --user=${KUBE_USER} \
  --kubeconfig=/etc/kubernetes/${KUBE_CONFIG}

# 设置当前使用的上下文
kubectl config use-context ${KUBE_USER}@${CLUSTER_NAME} --kubeconfig=/etc/kubernetes/${KUBE_CONFIG}

# 查看生成的配置文件
kubectl config view --kubeconfig=/etc/kubernetes/${KUBE_CONFIG}

CLUSTER_NAME="kubernetes"
KUBE_USER="kubernetes-admin"
KUBE_CERT="admin"
KUBE_CONFIG="admin.kubeconfig"

# 设置集群参数
kubectl config set-cluster ${CLUSTER_NAME} \
  --certificate-authority=/etc/kubernetes/pki/ca.crt \
  --embed-certs=true \
  --server=${KUBE_APISERVER} \
  --kubeconfig=/etc/kubernetes/${KUBE_CONFIG}

# 设置客户端认证参数
kubectl config set-credentials ${KUBE_USER} \
  --client-certificate=/etc/kubernetes/pki/${KUBE_CERT}.crt \
  --client-key=/etc/kubernetes/pki/${KUBE_CERT}.key \
  --embed-certs=true \
  --kubeconfig=/etc/kubernetes/${KUBE_CONFIG}

# 设置上下文参数
kubectl config set-context ${KUBE_USER}@${CLUSTER_NAME} \
  --cluster=${CLUSTER_NAME} \
  --user=${KUBE_USER} \
  --kubeconfig=/etc/kubernetes/${KUBE_CONFIG}

# 设置当前使用的上下文
kubectl config use-context ${KUBE_USER}@${CLUSTER_NAME} --kubeconfig=/etc/kubernetes/${KUBE_CONFIG}

# 查看生成的配置文件
kubectl config view --kubeconfig=/etc/kubernetes/${KUBE_CONFIG}

for NODE in "${!otherMaster[@]}"; do
    echo "--- $NODE ${otherMaster[$NODE]} ---"
    scp -r /etc/kubernetes ${otherMaster[$NODE]}:/etc
done

https://github.com/etcd-io/etcd/releases

[ "${#MasterArray[@]}" -eq 1 ] && ETCD_version=v3.1.9 || :
cd ~/k8s-manual-files

wget https://github.com/etcd-io/etcd/releases/download/${ETCD_version}/etcd-${ETCD_version}-linux-amd64.tar.gz

tar -zxvf etcd-${ETCD_version}-linux-amd64.tar.gz --strip-components=1 -C /usr/local/bin etcd-${ETCD_version}-linux-amd64/etcd{,ctl}
#-------
#上面被墙了可以使用骚套路
docker pull quay.io/coreos/etcd:$ETCD_version
docker run --rm -d --name temp quay.io/coreos/etcd:$ETCD_version sleep 10
docker cp temp:/usr/local/bin/etcd /usr/local/bin
docker cp temp:/usr/local/bin/etcdctl /usr/local/bin

for NODE in "${!otherMaster[@]}"; do
    echo "--- $NODE ${otherMaster[$NODE]} ---"
    scp /usr/local/bin/etcd* ${otherMaster[$NODE]}:/usr/local/bin/
done

cd ~/k8s-manual-files/master/
etcd_servers=$( xargs -n1<<<${MasterArray[@]} | sort | sed 's#^#https://#;s#$#:2379#;$s#\n##' | paste -d, -s - )
etcd_initial_cluster=$( for i in ${!MasterArray[@]};do  echo $i=https://${MasterArray[$i]}:2380; done | sort | paste -d, -s - )
sed -ri "/initial-cluster:/s#'.+'#'${etcd_initial_cluster}'#" etc/etcd/config.yml

for NODE in "${!MasterArray[@]}"; do
    echo "--- $NODE ${MasterArray[$NODE]} ---"
    ssh ${MasterArray[$NODE]} "mkdir -p $MANIFESTS_DIR /etc/etcd /var/lib/etcd"
    scp systemd/etcd.service ${MasterArray[$NODE]}:/usr/lib/systemd/system/etcd.service
    scp etc/etcd/config.yml ${MasterArray[$NODE]}:/etc/etcd/etcd.config.yml
    ssh ${MasterArray[$NODE]} "sed -i "s/{HOSTNAME}/$NODE/g" /etc/etcd/etcd.config.yml"
    ssh ${MasterArray[$NODE]} "sed -i "s/{PUBLIC_IP}/${MasterArray[$NODE]}/g" /etc/etcd/etcd.config.yml"
    ssh ${MasterArray[$NODE]} 'systemctl daemon-reload'
done

for NODE in "${!MasterArray[@]}"; do
    echo "--- $NODE ${MasterArray[$NODE]} ---"
    ssh ${MasterArray[$NODE]} 'systemctl enable --now etcd' &
done
wait

etcdctl \
  --cert-file /etc/kubernetes/pki/etcd/healthcheck-client.crt \
  --key-file /etc/kubernetes/pki/etcd/healthcheck-client.key \
  --ca-file /etc/kubernetes/pki/etcd/ca.crt \
   --endpoints $etcd_servers cluster-health

...下面是输出
member 4f15324b6756581c is healthy: got healthy result from https://10.0.6.166:2379
member cce1303a6b6dd443 is healthy: got healthy result from https://10.0.6.167:2379
member ead42f3e6c9bb295 is healthy: got healthy result from https://10.0.6.168:2379
cluster is healthy


ETCDCTL_API=3 \
    etcdctl   \
   --cert /etc/kubernetes/pki/etcd/healthcheck-client.crt \
   --key /etc/kubernetes/pki/etcd/healthcheck-client.key \
   --cacert /etc/kubernetes/pki/etcd/ca.crt \
    --endpoints $etcd_servers get / --prefix --keys-only

for NODE in "${!MasterArray[@]}"; do
    echo "--- $NODE ${MasterArray[$NODE]} ---"
    ssh ${MasterArray[$NODE]} 'yum install haproxy keepalived -y' &
done
wait

cd ~/k8s-manual-files/master/etc

# 修改haproxy.cfg配置文件
sed -i '$r '<(paste <( seq -f'  server k8s-api-%g'  ${#MasterArray[@]} ) <( xargs -n1<<<${MasterArray[@]} | sort | sed 's#$#:6443  check#')) haproxy/haproxy.cfg

# 修改keepalived(网卡和VIP写进去,使用下面命令)

sed -ri "s#\{\{ VIP \}\}#${VIP}#" keepalived/*
sed -ri "s#\{\{ interface \}\}#${interface}#" keepalived/keepalived.conf 
sed -i '/unicast_peer/r '<(xargs -n1<<<${MasterArray[@]} | sort | sed 's#^#\t#') keepalived/keepalived.conf

# 分发文件
for NODE in "${!MasterArray[@]}"; do
    echo "--- $NODE ${MasterArray[$NODE]} ---"
    scp -r haproxy/ ${MasterArray[$NODE]}:/etc
    scp -r keepalived/ ${MasterArray[$NODE]}:/etc
    ssh ${MasterArray[$NODE]} 'systemctl enable --now haproxy keepalived'
done

ping $VIP

for NODE in "${!MasterArray[@]}"; do
    echo "--- $NODE ${MasterArray[$NODE]} ---"
    ssh ${MasterArray[$NODE]}  'systemctl restart haproxy keepalived'
done

cd ~/k8s-manual-files/master/
etcd_servers=$( xargs -n1<<<${MasterArray[@]} | sort | sed 's#^#https://#;s#$#:2379#;$s#\n##' | paste -d, -s - )

# 注入VIP和etcd_servers,apiserver数量
sed -ri '/--advertise-address/s#=.+#='"$VIP"' \\#' systemd/kube-apiserver.service
sed -ri '/--etcd-servers/s#=.+#='"$etcd_servers"' \\#' systemd/kube-apiserver.service
sed -ri '/apiserver-count/s#=[^\]+#='"${#MasterArray[@]}"' #' systemd/kube-apiserver.service

# 分发文件
for NODE in "${!MasterArray[@]}"; do
    echo "--- $NODE ${MasterArray[$NODE]} ---"
    ssh ${MasterArray[$NODE]} 'mkdir -p /etc/kubernetes/manifests /etc/systemd/system/kubelet.service.d /var/lib/kubelet /var/log/kubernetes'
    scp systemd/kube-*.service ${MasterArray[$NODE]}:/usr/lib/systemd/system/
    #注入网卡ip
    ssh ${MasterArray[$NODE]} "sed -ri '/bind-address/s#=[^\]+#=${MasterArray[$NODE]} #' /usr/lib/systemd/system/kube-apiserver.service"
done

for NODE in "${!MasterArray[@]}"; do
    echo "--- $NODE ${MasterArray[$NODE]} ---"
    ssh ${MasterArray[$NODE]} 'systemctl enable --now  kube-apiserver kube-controller-manager kube-scheduler;
    mkdir -p ~/.kube/
    cp /etc/kubernetes/admin.kubeconfig ~/.kube/config;
    kubectl completion bash > /etc/bash_completion.d/kubectl'
done

$ kubectl get cs
NAME                 STATUS    MESSAGE              ERROR
scheduler            Healthy   ok                   
controller-manager   Healthy   ok                   
etcd-2               Healthy   {"health": "true"}   
etcd-0               Healthy   {"health": "true"}   
etcd-1               Healthy   {"health": "true"} 

$ kubectl get svc
NAME         TYPE        CLUSTER-IP   EXTERNAL-IP   PORT(S)   AGE
kubernetes   ClusterIP   10.96.0.1    <none>        443/TCP   36s

TOKEN_PUB=$(openssl rand -hex 3)
TOKEN_SECRET=$(openssl rand -hex 8)
BOOTSTRAP_TOKEN="${TOKEN_PUB}.${TOKEN_SECRET}"

kubectl -n kube-system create secret generic bootstrap-token-${TOKEN_PUB} \
        --type 'bootstrap.kubernetes.io/token' \
        --from-literal description="cluster bootstrap token" \
        --from-literal token-id=${TOKEN_PUB} \
        --from-literal token-secret=${TOKEN_SECRET} \
        --from-literal usage-bootstrap-authentication=true \
        --from-literal usage-bootstrap-signing=true

CLUSTER_NAME="kubernetes"
KUBE_USER="kubelet-bootstrap"
KUBE_CONFIG="bootstrap.kubeconfig"

# 设置集群参数
kubectl config set-cluster ${CLUSTER_NAME} \
  --certificate-authority=/etc/kubernetes/pki/ca.crt \
  --embed-certs=true \
  --server=${KUBE_APISERVER} \
  --kubeconfig=/etc/kubernetes/${KUBE_CONFIG}

# 设置上下文参数
kubectl config set-context ${KUBE_USER}@${CLUSTER_NAME} \
  --cluster=${CLUSTER_NAME} \
  --user=${KUBE_USER} \
  --kubeconfig=/etc/kubernetes/${KUBE_CONFIG}

# 设置客户端认证参数
kubectl config set-credentials ${KUBE_USER} \
  --token=${BOOTSTRAP_TOKEN} \
  --kubeconfig=/etc/kubernetes/${KUBE_CONFIG}

# 设置当前使用的上下文
kubectl config use-context ${KUBE_USER}@${CLUSTER_NAME} --kubeconfig=/etc/kubernetes/${KUBE_CONFIG}

# 查看生成的配置文件
kubectl config view --kubeconfig=/etc/kubernetes/${KUBE_CONFIG}

kubectl create clusterrolebinding kubeadm:kubelet-bootstrap \
        --clusterrole system:node-bootstrapper --group system:bootstrappers

cat <<EOF | kubectl apply -f -
# Approve all CSRs for the group "system:bootstrappers"
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: auto-approve-csrs-for-group
subjects:
- kind: Group
  name: system:bootstrappers
  apiGroup: rbac.authorization.k8s.io
roleRef:
  kind: ClusterRole
  name: system:certificates.k8s.io:certificatesigningrequests:nodeclient
  apiGroup: rbac.authorization.k8s.io
EOF

cat <<EOF | kubectl apply -f -
# Approve renewal CSRs for the group "system:nodes"
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: auto-approve-renewals-for-nodes
subjects:
- kind: Group
  name: system:nodes
  apiGroup: rbac.authorization.k8s.io
roleRef:
  kind: ClusterRole
  name: system:certificates.k8s.io:certificatesigningrequests:selfnodeclient
  apiGroup: rbac.authorization.k8s.io
EOF

cat <<EOF | kubectl apply -f -
# A ClusterRole which instructs the CSR approver to approve a user requesting
# node client credentials.
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: system:certificates.k8s.io:certificatesigningrequests:nodeclient
rules:
- apiGroups: ["certificates.k8s.io"]
  resources: ["certificatesigningrequests/nodeclient"]
  verbs: ["create"]
---
# A ClusterRole which instructs the CSR approver to approve a node renewing its
# own client credentials.
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: system:certificates.k8s.io:certificatesigningrequests:selfnodeclient
rules:
- apiGroups: ["certificates.k8s.io"]
  resources: ["certificatesigningrequests/selfnodeclient"]
  verbs: ["create"]
EOF

for NODE in "${!Other[@]}"; do
    echo "--- $NODE ${Other[$NODE]} ---"
    ssh ${Other[$NODE]} "mkdir -p /etc/kubernetes/pki /etc/kubernetes/manifests /var/lib/kubelet/"
    for FILE in /etc/kubernetes/pki/ca.crt /etc/kubernetes/bootstrap.kubeconfig; do
      scp ${FILE} ${Other[$NODE]}:${FILE}
    done
done

cd ~/k8s-manual-files/
for NODE in "${!AllNode[@]}"; do
    echo "--- $NODE ${AllNode[$NODE]} ---"
    scp master/systemd/kubelet.service ${AllNode[$NODE]}:/lib/systemd/system/kubelet.service
    scp master/etc/kubelet/kubelet-conf.yml ${AllNode[$NODE]}:/etc/kubernetes/kubelet-conf.yml
    ssh ${AllNode[$NODE]} "sed -ri '/0.0.0.0/s#\S+\$#${MasterArray[$NODE]}#' /etc/kubernetes/kubelet-conf.yml"
done

for NODE in "${!AllNode[@]}"; do
    echo "--- $NODE ${AllNode[$NODE]} ---"
    ssh ${AllNode[$NODE]} 'systemctl enable --now kubelet.service'
done

$ kubectl get nodes
NAME     STATUS     ROLES    AGE     VERSION   INTERNAL-IP   EXTERNAL-IP   OS-IMAGE                KERNEL-VERSION                CONTAINER-RUNTIME
k8s-m1   NotReady   <none>   22s     v1.13.4   10.0.6.166    <none>        CentOS Linux 7 (Core)   4.20.13-1.el7.elrepo.x86_64   docker://18.6.3
k8s-m2   NotReady   <none>   24s     v1.13.4   10.0.6.167    <none>        CentOS Linux 7 (Core)   4.20.13-1.el7.elrepo.x86_64   docker://18.6.3
k8s-m3   NotReady   <none>   21s     v1.13.4   10.0.6.168    <none>        CentOS Linux 7 (Core)   4.20.13-1.el7.elrepo.x86_64   docker://18.6.3
k8s-n1   NotReady   <none>   22s     v1.13.4   10.0.6.169    <none>        CentOS Linux 7 (Core)   4.20.13-1.el7.elrepo.x86_64   docker://18.6.3
k8s-n2   NotReady   <none>   22s     v1.13.4   10.0.6.170    <none>        CentOS Linux 7 (Core)   4.20.13-1.el7.elrepo.x86_64   docker://18.6.3
# csr自动被授权
$ kubectl get csr
NAME                                                   AGE   REQUESTOR                 CONDITION
node-csr-4fCDrNulc_btdBiRgev0JO4EorZ0rMuyJ756wrn9NqQ   27s   system:bootstrap:e860ec   Approved,Issued
node-csr-P3Y_knryQNaQWDDYFObtcdfXB4XAl9IB2Be2YJ-b-dA   27s   system:bootstrap:e860ec   Approved,Issued
node-csr-r_4ZDnanqBw2HPTSn6bSL50r-kJkTPjix6SY1n9UmjY   28s   system:bootstrap:e860ec   Approved,Issued
node-csr-vy-6tgMI9vUiIzR3Ogv6bPVGA2_gZrd7aMIWMSuHrME   27s   system:bootstrap:e860ec   Approved,Issued
node-csr-zOvVxSaY1iMco2LnOHmwqiBDwPUaLix7cSqUfZWTGFo   26s   system:bootstrap:e860ec   Approved,Issued

kubectl taint nodes ${!MasterArray[@]} node-role.kubernetes.io/master="":NoSchedule
# 下面是输出
node "k8s-m1" tainted
node "k8s-m2" tainted
node "k8s-m3" tainted

kubectl label node ${!MasterArray[@]} node-role.kubernetes.io/master=""
kubectl label node ${!NodeArray[@]} node-role.kubernetes.io/worker=worker

kubectl -n kube-system create serviceaccount kube-proxy

kubectl create clusterrolebinding kubeadm:kube-proxy \
        --clusterrole system:node-proxier \
        --serviceaccount kube-system:kube-proxy

CLUSTER_NAME="kubernetes"
KUBE_CONFIG="kube-proxy.kubeconfig"

SECRET=$(kubectl -n kube-system get sa/kube-proxy \
    --output=jsonpath='{.secrets[0].name}')

JWT_TOKEN=$(kubectl -n kube-system get secret/$SECRET \
    --output=jsonpath='{.data.token}' | base64 -d)

kubectl config set-cluster ${CLUSTER_NAME} \
  --certificate-authority=/etc/kubernetes/pki/ca.crt \
  --embed-certs=true \
  --server=${KUBE_APISERVER} \
  --kubeconfig=/etc/kubernetes/${KUBE_CONFIG}

kubectl config set-context ${CLUSTER_NAME} \
  --cluster=${CLUSTER_NAME} \
  --user=${CLUSTER_NAME} \
  --kubeconfig=/etc/kubernetes/${KUBE_CONFIG}

kubectl config set-credentials ${CLUSTER_NAME} \
  --token=${JWT_TOKEN} \
  --kubeconfig=/etc/kubernetes/${KUBE_CONFIG}

kubectl config use-context ${CLUSTER_NAME} --kubeconfig=/etc/kubernetes/${KUBE_CONFIG}
kubectl config view --kubeconfig=/etc/kubernetes/${KUBE_CONFIG}

cd ~/k8s-manual-files/
for NODE in "${!Other[@]}"; do
    echo "--- $NODE ${Other[$NODE]} ---"
    scp /etc/kubernetes/kube-proxy.kubeconfig ${Other[$NODE]}:/etc/kubernetes/kube-proxy.kubeconfig
done

for NODE in "${!AllNode[@]}"; do
    echo "--- $NODE ${AllNode[$NODE]} ---"
    scp addons/kube-proxy/kube-proxy.conf ${AllNode[$NODE]}:/etc/kubernetes/kube-proxy.conf
    scp addons/kube-proxy/kube-proxy.service ${AllNode[$NODE]}:/usr/lib/systemd/system/kube-proxy.service
    ssh ${AllNode[$NODE]} "sed -ri '/0.0.0.0/s#\S+\$#${MasterArray[$NODE]}#' /etc/kubernetes/kube-proxy.conf"
done

for NODE in "${!AllNode[@]}"; do
    echo "--- $NODE ${AllNode[$NODE]} ---"
    ssh ${AllNode[$NODE]} 'systemctl enable --now kube-proxy'
done

cd ~/k8s-manual-files
# 注入变量
sed -ri "/server:/s#(: ).+#\1${KUBE_APISERVER}#" addons/kube-proxy/kube-proxy.yml
sed -ri "/image:.+kube-proxy/s#:[^:]+\$#:$KUBE_VERSION#" addons/kube-proxy/kube-proxy.yml
kubectl apply -f addons/kube-proxy/kube-proxy.yml
# 下面是输出
serviceaccount "kube-proxy" created
clusterrolebinding.rbac.authorization.k8s.io "system:kube-proxy" created
configmap "kube-proxy" created
daemonset.apps "kube-proxy" created

$ kubectl -n kube-system get po -l k8s-app=kube-proxy
NAME               READY     STATUS    RESTARTS   AGE
kube-proxy-dd2m7   1/1       Running   0          8m
kube-proxy-fwgx8   1/1       Running   0          8m
kube-proxy-kjn57   1/1       Running   0          8m
kube-proxy-vp47w   1/1       Running   0          8m

$ ipvsadm -ln
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
  -> RemoteAddress:Port           Forward Weight ActiveConn InActConn
TCP  10.96.0.1:443 rr
  -> 10.0.6.155:6443            Masq    1      0          0

$ curl localhost:10249/proxyMode
ipvs

$ grep -Pom1 'image:\s+\K\S+' addons/flannel/kube-flannel.yml 
quay.io/coreos/flannel:v0.11.0-amd64
$ curl -s https://zhangguanzhang.github.io/bash/pull.sh | bash -s -- quay.io/coreos/flannel:v0.11.0-amd64

sed -ri "s#\{\{ interface \}\}#${interface}#" addons/flannel/kube-flannel.yml

kubectl apply -f addons/flannel/kube-flannel.yml
...

E0304 07:49:51.171741       1 main.go:241] Failed to create SubnetManager: error retrieving pod spec for 'kube-system/kube-flannel-ds-amd64-4lpqx': Get https://10.96.0.1:443/api/v1/namespaces/kube-system/pods/kube-flannel-ds-amd64-4lpqx: dial tcp 10.96.0.1:443: connect: connection refused

$ curl -v  -sk https://10.96.0.1:443/healthz
* About to connect() to 10.96.0.1 port 443 (#0)
*   Trying 10.96.0.1...
* Connection refused
* Failed connect to 10.96.0.1:443; Connection refused
* Closing connection 0

$ ipvsadm -ln
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
  -> RemoteAddress:Port           Forward Weight ActiveConn InActConn
TCP  10.96.0.1:443 rr
  -> 10.0.6.155:6443            Masq    1      0          0

$ curl -v  -sk https://$VIP:6443/healthz

* About to connect() to 10.96.0.1 port 443 (#0)
*   Trying 10.96.0.1...
* Connection refused
* Failed connect to 10.0.6.155:6443; Connection refused
* Closing connection 0

$ curl -v  -sk https://${MasterArray[@]:1:1}:6443/healthz

* About to connect() to 10.0.6.166 port 6443 (#0)
*   Trying 10.0.6.166...
* Connected to 10.0.6.166 (10.0.6.166) port 6443 (#0)
* Initializing NSS with certpath: sql:/etc/pki/nssdb
* skipping SSL peer certificate verification
* NSS: client certificate not found (nickname not specified)
* SSL connection using TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256
* Server certificate:
* 	subject: CN=kube-apiserver
* 	start date: Mar 04 07:22:05 2019 GMT
* 	expire date: Jul 20 07:22:05 2046 GMT
* 	common name: kube-apiserver
* 	issuer: CN=kubernetes-ca
> GET /healthz HTTP/1.1
> User-Agent: curl/7.29.0
> Host: 10.0.6.166:6443
> Accept: */*
> 
< HTTP/1.1 200 OK
< Date: Mon, 04 Mar 2019 07:57:01 GMT
< Content-Length: 2
< Content-Type: text/plain; charset=utf-8
< 
* Connection #0 to host 10.0.6.166 left intact
ok

$ curl -v  -sk https://$VIP:8443/healthz

ok

kubectl patch ep kubernetes -p '{"subsets": [{"addresses":[{"ip":"'$VIP'"}],"ports":[{"name":"https","port":8443,"protocol":"TCP"}]}] }'

$ kubectl -n kube-system get po -l k8s-app=flannel
NAME                READY     STATUS    RESTARTS   AGE
kube-flannel-ds-27jwl   2/2       Running   0          59s
kube-flannel-ds-4fgv6   2/2       Running   0          59s
kube-flannel-ds-mvrt7   2/2       Running   0          59s
kube-flannel-ds-p2q9g   2/2       Running   0          59s
kube-flannel-ds-zchsz   2/2       Running   0          59s

$ grep -Po 'image:\s+\K\S+' addons/calico/v3.1/calico.yml 
quay.io/calico/typha:v0.7.4
quay.io/calico/node:v3.1.3
quay.io/calico/cni:v3.1.3

curl -s https://zhangguanzhang.github.io/bash/pull.sh | bash -s -- quay.io/calico/node:v3.1.3
curl -s https://zhangguanzhang.github.io/bash/pull.sh | bash -s -- quay.io/calico/cni:v3.1.3

sed -ri "s#\{\{ interface \}\}#${interface}#" addons/calico/v3.1/calico.yml
kubectl apply -f addons/calico/v3.1

$ kubectl -n kube-system get pod --all-namespaces
NAMESPACE     NAME                              READY     STATUS              RESTARTS   AGE
kube-system   calico-node-2hdqf                 0/2       ContainerCreating   0          4m
kube-system   calico-node-456fh                 0/2       ContainerCreating   0          4m
kube-system   calico-node-jh6vd                 0/2       ContainerCreating   0          4m
kube-system   calico-node-sp6w9                 0/2       ContainerCreating   0          4m
kube-system   calicoctl-6dfc585667-24s9h        0/1       Pending             0          4m
kube-system   kube-proxy-46hr5                  1/1       Running             0          7m
kube-system   kube-proxy-l42sk                  1/1       Running             0          7m
kube-system   kube-proxy-p2nbf                  1/1       Running             0          7m
kube-system   kube-proxy-q6qn9                  1/1       Running             0          7m

$ kubectl get pod --all-namespaces
NAMESPACE     NAME                              READY     STATUS    RESTARTS   AGE
kube-system   calico-node-2hdqf                 2/2       Running   0          4m
kube-system   calico-node-456fh                 2/2       Running   2          4m
kube-system   calico-node-jh6vd                 2/2       Running   0          4m
kube-system   calico-node-sp6w9                 2/2       Running   0          4m
kube-system   calicoctl-6dfc585667-24s9h        1/1       Running   0          4m
kube-system   kube-proxy-46hr5                  1/1       Running   0          8m
kube-system   kube-proxy-l42sk                  1/1       Running   0          8m
kube-system   kube-proxy-p2nbf                  1/1       Running   0          8m
kube-system   kube-proxy-q6qn9                  1/1       Running   0          8m

kubectl -n kube-system get po -l k8s-app=calico-node
NAME                READY     STATUS    RESTARTS   AGE
calico-node-bv7r9   2/2       Running   4          5m
calico-node-cmh2w   2/2       Running   3          5m
calico-node-klzrz   2/2       Running   4          5m
calico-node-n4c9j   2/2       Running   4          5m

kubectl -n kube-system get po -l k8s-app=calicoctl
NAME                         READY     STATUS    RESTARTS   AGE
calicoctl-6b5bf7cb74-d9gv8   1/1       Running   0          5m

$ kubectl -n kube-system exec calicoctl-6b5bf7cb74-d9gv8 -- calicoctl get profiles -o wide
NAME              LABELS   
kns.default       map[]    
kns.kube-public   map[]    
kns.kube-system   map[]    

$ kubectl -n kube-system exec calicoctl-6b5bf7cb74-d9gv8 -- calicoctl get node -o wide
NAME     ASN         IPV4                 IPV6   
k8s-m1   (unknown)   192.168.88.111/24          
k8s-m2   (unknown)   192.168.88.112/24          
k8s-m3   (unknown)   192.168.88.113/24          
k8s-n1   (unknown)   10.244.3.1/24

data:
  Corefile: |
    .:53 {
        errors
        health
        kubernetes cluster.local in-addr.arpa ip6.arpa {
          pods insecure
          upstream
          fallthrough in-addr.arpa ip6.arpa
        }
        hosts {
          192.168.88.111 k8s-m1
          192.168.88.112 k8s-m2
          192.168.88.113 k8s-m3
          192.168.88.114 k8s-n1
        }
        prometheus :9153
        proxy . /etc/resolv.conf
        cache 30
        reload
        loadbalance
    }

sed  -i '57r '<(echo '        hosts {';for NODE in "${!MasterArray[@]}";do  echo "          ${MasterArray[$NODE]} $NODE"; done;for NODE in "${!NodeArray[@]}";do echo "          ${NodeArray[$NODE]} $NODE";done;echo '        }';) addons/coredns/coredns.yml

kubectl exec coredns-xxxxxx -- kill -SIGUSR1 1

kubectl apply -f addons/coredns/coredns.yml
# 下面是输出
serviceaccount/coredns created
clusterrole.rbac.authorization.k8s.io/system:coredns created
clusterrolebinding.rbac.authorization.k8s.io/system:coredns created
configmap/coredns created
deployment.extensions/coredns created
service/kube-dns created

$ kubectl -n kube-system get po -l k8s-app=kube-dns
NAMESPACE     NAME                              READY     STATUS              RESTARTS   AGE
kube-system   coredns-6975654877-jjqkg          1/1       Running   0          1m
kube-system   coredns-6975654877-ztqjh          1/1       Running   0          1m

$ kubectl get nodes
NAME      STATUS     ROLES     AGE       VERSION
k8s-m1    Ready      master    17m       v1.11.1
k8s-m2    Ready      master    16m       v1.11.1
k8s-m3    Ready      master    16m       v1.11.1
k8s-n1    Ready      node      6m        v1.11.1

$ cat<<EOF | kubectl apply -f -
apiVersion: v1
kind: Pod
metadata:
  name: busybox
  namespace: default
spec:
  containers:
  - name: busybox
    image: busybox:1.28
    command:
      - sleep
      - "3600"
    imagePullPolicy: IfNotPresent
  restartPolicy: Always
EOF

$ kubectl exec -ti busybox -- nslookup kubernetes
Server:    10.96.0.10
Address 1: 10.96.0.10 kube-dns.kube-system.svc.cluster.local

Name:      kubernetes
Address 1: 10.96.0.1 kubernetes.default.svc.cluster.local

Server:    10.96.0.10
Address 1: 10.96.0.10 kube-dns.kube-system.svc.cluster.local

nslookup: can't resolve 'kubernetes'
command terminated with exit code 1

$ kubectl delete -f addons/coredns/coredns.yml
$ kubectl -n kube-system get pod,svc -l k8s-app=kube-dns
No resources found.

$ kubectl apply -f addons/Kubedns/kubedns.yml 

serviceaccount/kube-dns created
service/kube-dns created
deployment.extensions/kube-dns create

$ kubectl -n kube-system get pod,svc -l k8s-app=kube-dns

NAME                            READY   STATUS    RESTARTS   AGE
pod/kube-dns-59c677cb95-pxcbc   3/3     Running   0          3m
pod/kube-dns-59c677cb95-wlprb   3/3     Running   0          3m

NAME               TYPE        CLUSTER-IP   EXTERNAL-IP   PORT(S)         AGE
service/kube-dns   ClusterIP   10.96.0.10   <none>        53/UDP,53/TCP   3m

$ kubectl exec -ti busybox -- nslookup kubernetes
Server:    10.96.0.10
Address 1: 10.96.0.10 kube-dns.kube-system.svc.cluster.local

Name:      kubernetes
Address 1: 10.96.0.1 kubernetes.default.svc.cluster.local

设置apiserver相关参数
      --requestheader-client-ca-file=/etc/kubernetes/pki/front-proxy-ca.pem  \
      --proxy-client-cert-file=/etc/kubernetes/pki/front-proxy-client.pem  \
      --proxy-client-key-file=/etc/kubernetes/pki/front-proxy-client-key.pem  \
      --requestheader-allowed-names=aggregator  \
      --requestheader-group-headers=X-Remote-Group  \
      --requestheader-extra-headers-prefix=X-Remote-Extra-  \
      --requestheader-username-headers=X-Remote-User  \

apiVersion: apiregistration.k8s.io/v1beta1
kind: APIService
metadata:
  name: v1beta1.metrics.k8s.io
spec:
  service:
    name: metrics-server
    namespace: kube-system
  group: metrics.k8s.io
  version: v1beta1
  insecureSkipTLSVerify: true
  groupPriorityMinimum: 100
  versionPriority: 10

$ kubectl top node
Error from server (NotFound): the server could not find the requested resource (get services http:heapster:)

kubectl create -f addons/metric-server/metrics-server.yml

kubectl create -f addons/metric-server/metrics-server-1.12+.yml

kubectl -n kube-system get po -l k8s-app=metrics-server
NAME                                  READY     STATUS    RESTARTS   AGE
pod/metrics-server-86bd9d7667-5hbn6   1/1       Running   0          1m

$ kubectl get --raw /apis/metrics.k8s.io/v1beta1
{"kind":"APIResourceList","apiVersion":"v1","groupVersion":"metrics.k8s.io/v1beta1","resources":[{"name":"nodes","singularName":"","namespaced":false,"kind":"NodeMetrics","verbs":["get","list"]},{"name":"pods","singularName":"","namespaced":true,"kind":"PodMetrics","verbs":["get","list"]}]}

$ kubectl get apiservice|grep metrics
v1beta1.metrics.k8s.io                 kube-system/metrics-server   True        2m

$ kubectl top node
NAME      CPU(cores)   CPU%      MEMORY(bytes)   MEMORY%   
k8s-m1    113m         2%        1080Mi          14%       
k8s-m2    133m         3%        1086Mi          14%       
k8s-m3    100m         2%        1029Mi          13%       
k8s-n1    146m         3%        403Mi           5%

$ kubectl run nginx --image nginx --restart=Never --port 80
$ kubectl get po
NAME      READY     STATUS    RESTARTS   AGE
nginx     1/1       Running   0          22s

$ cd ~/k8s-manual-files

$ kubectl apply -f ExtraAddons/dashboard
$ kubectl -n kube-system get po,svc -l k8s-app=kubernetes-dashboard
NAME                                    READY     STATUS    RESTARTS   AGE
kubernetes-dashboard-7d5dcdb6d9-j492l   1/1       Running   0          12s

NAME                   TYPE        CLUSTER-IP      EXTERNAL-IP   PORT(S)   AGE
kubernetes-dashboard   ClusterIP   10.111.22.111   <none>        443/TCP   12s

$ kubectl -n kube-system describe secrets | sed -rn '/\sdashboard-token-/,/^token/{/^token/s#\S+\s+##p}'
eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJrdWJlcm5ldGVzL3NlcnZpY2VhY2NvdW50Iiwia3ViZXJuZXRlcy5pby9zZXJ2aWNlYWNjb3VudC9uYW1lc3BhY2UiOiJrdWJlLXN5c3RlbSIsImt1YmVybmV0ZXMuaW8vc2VydmljZWFjY291bnQvc2VjcmV0Lm5hbWUiOiJkYXNoYm9hcmQtdG9rZW4tdzVocmgiLCJrdWJlcm5ldGVzLmlvL3NlcnZpY2VhY2NvdW50L3NlcnZpY2UtYWNjb3VudC5uYW1lIjoiZGFzaGJvYXJkIiwia3ViZXJuZXRlcy5pby9zZXJ2aWNlYWNjb3VudC9zZXJ2aWNlLWFjY291bnQudWlkIjoiYWJmMTFjYzMtZjRlYi0xMWU3LTgzYWUtMDgwMDI3NjdkOWI5Iiwic3ViIjoic3lzdGVtOnNlcnZpY2VhY2NvdW50Omt1YmUtc3lzdGVtOmRhc2hib2FyZCJ9.Xuyq34ci7Mk8bI97o4IldDyKySOOqRXRsxVWIJkPNiVUxKT4wpQZtikNJe2mfUBBD-JvoXTzwqyeSSTsAy2CiKQhekW8QgPLYelkBPBibySjBhJpiCD38J1u7yru4P0Pww2ZQJDjIxY4vqT46ywBklReGVqY3ogtUQg-eXueBmz-o7lJYMjw8L14692OJuhBjzTRSaKW8U2MPluBVnD7M2SOekDff7KpSxgOwXHsLVQoMrVNbspUCvtIiEI1EiXkyCNRGwfnd2my3uzUABIHFhm0_RZSmGwExPbxflr8Fc6bxmuz-_jSdOtUidYkFIzvEWw2vRovPgs3MXTv59RwUw

sed -ri 's#\{\{ INGRESS_VIP \}\}#'"${INGRESS_VIP}"'#' ExtraAddons/ingress-controller/ingress-controller-svc.yml
kubectl create ns ingress-nginx
kubectl apply -f ExtraAddons/ingress-controller/
kubectl -n ingress-nginx get po,svc
# 下面是输出
NAME                                           READY     STATUS    RESTARTS   AGE
pod/default-http-backend-846b65fb5f-l5hrc      1/1       Running   0          2m
pod/nginx-ingress-controller-5db8d65fb-z2lf9   1/1       Running   0          2m

NAME                           TYPE           CLUSTER-IP      EXTERNAL-IP      PORT(S)        AGE
service/default-http-backend   ClusterIP      10.99.105.112   <none>           80/TCP         2m
service/ingress-nginx          LoadBalancer   10.106.18.106   192.168.88.109  80:31197/TCP   2m

$ curl ${INGRESS_VIP}:80
default backend - 404

kubectl apply -f apps/nginx/
# 下面是输出
deployment.extensions/nginx created
ingress.extensions/nginx-ingress created
service/nginx created

$ kubectl get po,svc,ing
NAME                        READY     STATUS    RESTARTS   AGE
pod/nginx-966857787-78kth   1/1       Running   0          32s

NAME                 TYPE        CLUSTER-IP       EXTERNAL-IP   PORT(S)   AGE
service/kubernetes   ClusterIP   10.96.0.1        <none>        443/TCP   2d
service/nginx        ClusterIP   10.104.180.119   <none>        80/TCP    32s

NAME                               HOSTS             ADDRESS          PORTS     AGE
ingress.extensions/nginx-ingress   nginx.k8s.local   192.168.88.109  80        33s

$ curl ${INGRESS_VIP} -H 'Host: nginx.k8s.local'
<!DOCTYPE html>
<html> 
<head>
<title>Welcome to nginx!</title>
...

# 测试其他 domain name 是否会回返回 404
$ curl ${INGRESS_VIP} -H 'Host: nginx1.k8s.local'
default backend - 404

sed -ri 's#\{\{ INGRESS_VIP \}\}#'"${INGRESS_VIP}"'#' ExtraAddons/external-dns/coredns/coredns-svc-tcp.yml
sed -ri 's#\{\{ INGRESS_VIP \}\}#'"${INGRESS_VIP}"'#' ExtraAddons/external-dns/coredns/coredns-svc-udp.yml
kubectl create ns external-dns
kubectl create -f ExtraAddons/external-dns/coredns/
kubectl -n external-dns get po,svc
# 下面是输出
NAME                                READY     STATUS    RESTARTS   AGE
pod/coredns-54bcfcbd5b-5grb5        1/1       Running   0          2m
pod/coredns-etcd-6c9c68fd76-n8rhj   1/1       Running   0          2m

NAME                   TYPE           CLUSTER-IP       EXTERNAL-IP       PORT(S)                       AGE
service/coredns-etcd   ClusterIP      10.110.186.83    <none>            2379/TCP,2380/TCP             2m
service/coredns-tcp    LoadBalancer   10.109.105.166   192.168.88.109   53:32169/TCP,9153:32150/TCP   2m
service/coredns-udp    LoadBalancer   10.110.242.185   192.168.88.109   53:31210/UDP

$ dig @${INGRESS_VIP} SOA nginx.k8s.local +noall +answer +time=2 +tries=1
...
; (1 server found)
;; global options: +cmd
k8s.local.        300    IN    SOA    ns.dns.k8s.local. hostmaster.k8s.local. 1531299150 7200 1800 86400 30

kubectl apply -f ExtraAddons/external-dns/external-dns/
kubectl -n external-dns get po -l k8s-app=external-dns
NAME                            READY     STATUS    RESTARTS   AGE
external-dns-86f67f6df8-ljnhj   1/1       Running   0          1m

$ dig @${INGRESS_VIP} A nginx.k8s.local +noall +answer +time=2 +tries=1
...
; (1 server found)
;; global options: +cmd
nginx.k8s.local.    300    IN    A    192.168.88.109

$ nslookup nginx.k8s.local
Server:        192.168.88.109
Address:    192.168.88.109#53

** server can't find nginx.k8s.local: NXDOMAIN

$ nslookup nginx.k8s.local
Server:        192.168.88.109
Address:    192.168.88.109#53

Name:    nginx.k8s.local
Address: 192.168.88.109

$ nslookup www.baidu.com
Server:   192.168.88.110
Address:  192.168.88.110#53

Non-authoritative answer:
www.baidu.com canonical name = www.a.shifen.com.
Name: www.a.shifen.com
Address: 61.135.169.125
Name: www.a.shifen.com
Address: 61.135.169.121

$ curl nginx.k8s.local
<!DOCTYPE html>
<html>
<head>
<title>Welcome to nginx!</title>
...

kubectl apply -f ExtraAddons/prometheus/
kubectl apply -f ExtraAddons/prometheus/operator/

# 这边要等 operator 起來并建立好 CRDs 才能进行
kubectl apply -f ExtraAddons/prometheus/alertmanater/
kubectl apply -f ExtraAddons/prometheus/node-exporter/
kubectl apply -f ExtraAddons/prometheus/kube-state-metrics/
kubectl apply -f ExtraAddons/prometheus/grafana/
kubectl apply -f ExtraAddons/prometheus/kube-service-discovery/

kubectl apply -f ExtraAddons/prometheus/prometheus/
kubectl apply -f ExtraAddons/prometheus/servicemonitor/

$ kubectl -n monitoring get po,svc,ing
NAME                                      READY     STATUS    RESTARTS   AGE
pod/alertmanager-main-0                   1/2       Running   0          1m
pod/grafana-6d495c46d5-jpf6r              1/1       Running   0          43s
pod/kube-state-metrics-b84cfb86-4b8qg     4/4       Running   0          37s
pod/node-exporter-2f4lh                   2/2       Running   0          59s
pod/node-exporter-7cz5s                   2/2       Running   0          59s
pod/node-exporter-djdtk                   2/2       Running   0          59s
pod/node-exporter-kfpzt                   2/2       Running   0          59s
pod/node-exporter-qp2jf                   2/2       Running   0          59s
pod/prometheus-k8s-0                      3/3       Running   0          28s
pod/prometheus-k8s-1                      3/3       Running   0          15s
pod/prometheus-operator-9ffd6bdd9-rvqsz   1/1       Running   0          1m

NAME                            TYPE        CLUSTER-IP       EXTERNAL-IP   PORT(S)             AGE
service/alertmanager-main       ClusterIP   10.110.188.2     <none>        9093/TCP            1m
service/alertmanager-operated   ClusterIP   None             <none>        9093/TCP,6783/TCP   1m
service/grafana                 ClusterIP   10.104.147.154   <none>        3000/TCP            43s
service/kube-state-metrics      ClusterIP   None             <none>        8443/TCP,9443/TCP   51s
service/node-exporter           ClusterIP   None             <none>        9100/TCP            1m
service/prometheus-k8s          ClusterIP   10.96.78.58      <none>        9090/TCP            28s
service/prometheus-operated     ClusterIP   None             <none>        9090/TCP            33s
service/prometheus-operator     ClusterIP   10.99.251.16     <none>        8080/TCP 

NAME                                HOSTS                             ADDRESS           PORTS     AGE
ingress.extensions/grafana-ing      grafana.monitoring.k8s.local      192.168.88.109   80        45s
ingress.extensions/prometheus-ing   prometheus.monitoring.k8s.local   192.168.88.109   80        34s

cd ExtraAddons/prometheus
ll servicemonitor/kube-*
...下面是输出
-rw-r--r-- 1 root root 567 Dec 23 20:20 servicemonitor/kube-apiserver-sm.yml
-rw-r--r-- 1 root root 374 Dec 23 20:20 servicemonitor/kube-controller-manager-sm.yml
-rw-r--r-- 1 root root 347 Dec 23 20:20 servicemonitor/kube-scheduler-sm.yml

$ ll kube-service-discovery/
total 8
-rw-r--r-- 1 root root 343 Dec 23 20:20 kube-controller-manager-svc.yml
-rw-r--r-- 1 root root 316 Dec 23 20:20 kube-scheduler-svc.yml

$ grep -P '^\s+name:' kube-service-discovery/kube-*
kube-service-discovery/kube-controller-manager-svc.yml:  name: kube-controller-manager-prometheus-discovery
kube-service-discovery/kube-scheduler-svc.yml:  name: kube-scheduler-prometheus-discovery

$ kubectl -n kube-system get ep 
NAME                                           ENDPOINTS                                                                    AGE
kube-controller-manager                        <none>                                                                       5m
kube-controller-manager-prometheus-discovery   <none>                                                                       5m
kube-dns                                       10.244.1.2:53,10.244.8.2:53,10.244.1.2:53 + 1 more...                        5m
kube-scheduler                                 <none>                                                                       5m
kube-scheduler-prometheus-discovery            <none>                                                                       5m
kubelet                                        192.168.88.111:10255,192.168.88.112:10255,192.168.88.113:10255 + 1 more...   5m
metrics-server                                 10.244.7.3:443                                                               5m

kubectl -n kube-system patch ep kube-controller-manager-prometheus-discovery -p "$(cat<<EOF
{"subsets": [
        {
            "addresses": [
$(for i in ${!MasterArray[@]};do echo '{"ip": "'${MasterArray[$i]}'"}'; done | paste -d',' -s -)
            ],
            "ports": [
                {
                    "name": "http-metrics",
                    "port": 10252,
                    "protocol": "TCP"
                }
            ]
        }
    ]
  }
EOF
)"

kubectl -n kube-system patch ep kube-scheduler-prometheus-discovery -p "$(cat<<EOF
{"subsets": [
        {
            "addresses": [
$(for i in ${!MasterArray[@]};do echo '{"ip": "'${MasterArray[$i]}'"}'; done | paste -d',' -s -)
            ],
            "ports": [
                {
                    "name": "http-metrics",
                    "port": 10251,
                    "protocol": "TCP"
                }
            ]
        }
    ]
  }
EOF
)"

sed -ri '/^\s+"date"/s#"[^"]+"\s*$#"'"$(date "+%a %b %d %Y %T GMT%z (%Z)")"'"#;/^\s+"url"/s#=[^"]+#='$(kubectl version | base64 | tr -d '\n')'#' ExtraAddons/WeaveScope/scope.yml
kubectl apply -f ExtraAddons/WeaveScope/scope.yml

[root@k8s-m1 weave-scope]# kubectl -n weave get pod,svc,ing
NAME                                   READY   STATUS    RESTARTS   AGE
pod/weave-scope-agent-2srz9            1/1     Running   0          111s
pod/weave-scope-agent-9t847            1/1     Running   0          111s
pod/weave-scope-agent-dmjkp            1/1     Running   0          111s
pod/weave-scope-agent-j26rb            1/1     Running   0          111s
pod/weave-scope-app-5989b9c6d7-t28fs   1/1     Running   0          112s

NAME                      TYPE        CLUSTER-IP       EXTERNAL-IP   PORT(S)   AGE
service/weave-scope-app   ClusterIP   10.103.161.235   <none>        80/TCP    112s

NAME                               HOSTS                   ADDRESS      PORTS   AGE
ingress.extensions/weave-ingress   scope.weave.k8s.local   192.168.88.109   80      111s

$ wget -qO- https://kubernetes-helm.storage.googleapis.com/helm-v2.9.1-linux-amd64.tar.gz | tar -zx
$ sudo mv linux-amd64/helm /usr/local/bin/

yum install -y socat

$ sudo apt-get install -y socat

$ kubectl -n kube-system create sa tiller
$ kubectl create clusterrolebinding tiller --clusterrole cluster-admin --serviceaccount=kube-system:tiller
$ helm init --service-account tiller
...
Tiller (the Helm server-side component) has been installed into your Kubernetes Cluster.
Happy Helming!

kubectl -n kube-system patch deploy  tiller-deploy -p '{"spec":{"template":{"spec":{"containers":[{"name":"tiller","image":"zhangguanzhang/gcr.io.kubernetes-helm.tiller:v2.9.1"}]}}}}'

$ kubectl -n kube-system get po -l app=helm
NAME                             READY     STATUS    RESTARTS   AGE
tiller-deploy-5f789bd9f7-tzss6   1/1       Running   0          29s

$ helm version
Client: &version.Version{SemVer:"v2.9.1", GitCommit:"20adb27c7c5868466912eebdf6664e7390ebe710", GitTreeState:"clean"}
Server: &version.Version{SemVer:"v2.9.1", GitCommit:"20adb27c7c5868466912eebdf6664e7390ebe710", GitTreeState:"clean"}

$ helm install --name demo --set Persistence.Enabled=false stable/jenkins

$ kubectl get po,svc  -l app=demo-jenkins
NAME                           READY     STATUS    RESTARTS   AGE
demo-jenkins-7bf4bfcff-q74nt   1/1       Running   0          2m

NAME                 TYPE           CLUSTER-IP       EXTERNAL-IP   PORT(S)          AGE
demo-jenkins         LoadBalancer   10.103.15.129    <pending>     8080:31161/TCP   2m
demo-jenkins-agent   ClusterIP      10.103.160.88   <none>        50000/TCP        2m

$ printf $(kubectl get secret --namespace default demo-jenkins -o jsonpath="{.data.jenkins-admin-password}" | base64 --decode);echo
r6y9FMuF2u

$ helm ls
NAME    REVISION    UPDATED                     STATUS      CHART             NAMESPACE
demo    1           Tue Apr 10 07:29:51 2018    DEPLOYED    jenkins-0.14.4    default

$ helm delete demo --purge
release "demo" deleted

$ sudo poweroff

# 先检查 etcd 状态,可以发现 etcd-0 因為关机而中断
$ kubectl get cs
NAME                 STATUS      MESSAGE                                                                                                                                          ERROR
scheduler            Healthy     ok
controller-manager   Healthy     ok
etcd-1               Healthy     {"health": "true"}
etcd-2               Healthy     {"health": "true"}
etcd-0               Unhealthy   Get https://192.168.88.111:2379/health: net/http: request canceled while waiting for connection (Client.Timeout exceeded while awaiting headers)